From 894ee62fc185ee2db9052bce2b0371703825b857 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 25 Apr 2024 07:18:50 -0400 Subject: [PATCH] WIP: file source templates --- client/src/api/configTemplates.ts | 21 + client/src/api/fileSources.ts | 6 + client/src/api/schema/schema.ts | 252 ++++++++++- .../ConfigTemplates/CreateInstance.test.ts | 32 ++ .../ConfigTemplates/CreateInstance.vue | 21 + .../ConfigTemplates/EditSecretsForm.test.ts | 36 ++ .../ConfigTemplates/EditSecretsForm.vue | 34 ++ .../ConfigTemplates/InstanceDropdown.test.ts | 40 ++ .../ConfigTemplates/InstanceDropdown.vue | 50 +++ .../ConfigTemplates/InstanceForm.test.ts | 42 ++ .../InstanceForm.vue | 9 +- .../ConfigTemplates/ManageIndexHeader.vue | 35 ++ .../ConfigTemplates/SelectTemplate.test.js | 33 ++ .../ConfigTemplates/SelectTemplate.vue | 44 ++ .../TemplateSummaryPopover.test.ts | 22 + .../TemplateSummaryPopover.vue | 21 + .../VaultSecret.vue | 2 +- .../src/components/ConfigTemplates/fields.ts | 25 ++ .../ConfigTemplates/formUtil.test.ts | 45 ++ .../components/ConfigTemplates/formUtil.ts | 158 +++++++ .../Instances => ConfigTemplates}/icons.ts | 0 .../ConfigTemplates/test_fixtures.ts | 82 ++++ .../FileSources/FileSourceTypeSpan.vue | 23 + .../FileSources/Instances/CreateForm.vue | 53 +++ .../FileSources/Instances/CreateInstance.vue | 34 ++ .../FileSources/Instances/EditInstance.vue | 77 ++++ .../FileSources/Instances/EditSecrets.vue | 28 ++ .../Instances/InstanceDropdown.vue | 30 ++ .../FileSources/Instances/ManageIndex.vue | 68 +++ .../FileSources/Instances/UpgradeForm.vue | 63 +++ .../FileSources/Instances/UpgradeInstance.vue | 27 ++ .../FileSources/Instances/instance.ts | 23 + .../FileSources/Instances/routing.ts | 16 + .../FileSources/Instances/services.ts | 4 + .../Templates/CreateUserFileSource.vue | 30 ++ .../FileSources/Templates/SelectTemplate.vue | 39 ++ .../FileSources/Templates/TemplateSummary.vue | 22 + .../Templates/TemplateSummaryPopover.vue | 19 + .../Templates/TemplateSummarySpan.vue | 32 ++ client/src/components/FileSources/style.css | 4 + .../ObjectStore/Instances/CreateForm.vue | 61 +-- .../ObjectStore/Instances/EditInstance.vue | 79 ++-- .../ObjectStore/Instances/EditSecrets.vue | 14 +- .../Instances/InstanceDropdown.vue | 42 +- .../ObjectStore/Instances/ManageIndex.vue | 65 +-- .../ObjectStore/Instances/UpgradeForm.vue | 52 +-- .../components/ObjectStore/Instances/types.ts | 5 - .../components/ObjectStore/Instances/util.ts | 41 -- .../Templates/CreateUserObjectStore.vue | 19 +- .../ObjectStore/Templates/SelectTemplate.vue | 27 +- .../ObjectStore/Templates/TemplateSummary.vue | 2 +- .../Templates/TemplateSummaryPopover.vue | 12 +- .../src/components/User/UserPreferences.vue | 28 +- client/src/entry/analysis/router.js | 47 +- client/src/stores/configTemplatesUtil.ts | 56 +++ client/src/stores/fileSourceInstancesStore.ts | 51 +++ client/src/stores/fileSourceTemplatesStore.ts | 70 +++ .../src/stores/objectStoreTemplatesStore.ts | 49 +- client/src/utils/navigation/navigation.yml | 1 + lib/galaxy/app.py | 39 +- lib/galaxy/app_unittest_utils/galaxy_mock.py | 23 + lib/galaxy/config/schemas/config_schema.yml | 15 + lib/galaxy/files/__init__.py | 136 +++++- lib/galaxy/files/plugins.py | 5 + lib/galaxy/files/sources/__init__.py | 11 +- lib/galaxy/files/templates/__init__.py | 17 + lib/galaxy/files/templates/manager.py | 86 ++++ lib/galaxy/files/templates/models.py | 125 ++++++ lib/galaxy/files/unittest_utils/__init__.py | 7 +- lib/galaxy/managers/file_source_instances.py | 421 ++++++++++++++++++ lib/galaxy/managers/object_store_instances.py | 36 +- lib/galaxy/model/__init__.py | 86 +++- ...a3c93d66b_add_user_defined_file_sources.py | 57 +++ lib/galaxy/objectstore/templates/manager.py | 92 +--- lib/galaxy/objectstore/templates/models.py | 59 +-- lib/galaxy/util/config_templates.py | 168 +++++++ lib/galaxy/webapps/galaxy/api/file_sources.py | 95 ++++ lib/galaxy/webapps/galaxy/buildapp.py | 5 + packages/util/setup.cfg | 2 + .../app/managers/test_user_file_sources.py | 108 +++++ test/unit/files/_util.py | 3 +- test/unit/files/test_posix.py | 29 +- test/unit/files/test_template_manager.py | 58 +++ test/unit/files/test_template_models.py | 91 ++++ 84 files changed, 3525 insertions(+), 572 deletions(-) create mode 100644 client/src/api/configTemplates.ts create mode 100644 client/src/api/fileSources.ts create mode 100644 client/src/components/ConfigTemplates/CreateInstance.test.ts create mode 100644 client/src/components/ConfigTemplates/CreateInstance.vue create mode 100644 client/src/components/ConfigTemplates/EditSecretsForm.test.ts create mode 100644 client/src/components/ConfigTemplates/EditSecretsForm.vue create mode 100644 client/src/components/ConfigTemplates/InstanceDropdown.test.ts create mode 100644 client/src/components/ConfigTemplates/InstanceDropdown.vue create mode 100644 client/src/components/ConfigTemplates/InstanceForm.test.ts rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/InstanceForm.vue (79%) create mode 100644 client/src/components/ConfigTemplates/ManageIndexHeader.vue create mode 100644 client/src/components/ConfigTemplates/SelectTemplate.test.js create mode 100644 client/src/components/ConfigTemplates/SelectTemplate.vue create mode 100644 client/src/components/ConfigTemplates/TemplateSummaryPopover.test.ts create mode 100644 client/src/components/ConfigTemplates/TemplateSummaryPopover.vue rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/VaultSecret.vue (97%) create mode 100644 client/src/components/ConfigTemplates/fields.ts create mode 100644 client/src/components/ConfigTemplates/formUtil.test.ts create mode 100644 client/src/components/ConfigTemplates/formUtil.ts rename client/src/components/{ObjectStore/Instances => ConfigTemplates}/icons.ts (100%) create mode 100644 client/src/components/ConfigTemplates/test_fixtures.ts create mode 100644 client/src/components/FileSources/FileSourceTypeSpan.vue create mode 100644 client/src/components/FileSources/Instances/CreateForm.vue create mode 100644 client/src/components/FileSources/Instances/CreateInstance.vue create mode 100644 client/src/components/FileSources/Instances/EditInstance.vue create mode 100644 client/src/components/FileSources/Instances/EditSecrets.vue create mode 100644 client/src/components/FileSources/Instances/InstanceDropdown.vue create mode 100644 client/src/components/FileSources/Instances/ManageIndex.vue create mode 100644 client/src/components/FileSources/Instances/UpgradeForm.vue create mode 100644 client/src/components/FileSources/Instances/UpgradeInstance.vue create mode 100644 client/src/components/FileSources/Instances/instance.ts create mode 100644 client/src/components/FileSources/Instances/routing.ts create mode 100644 client/src/components/FileSources/Instances/services.ts create mode 100644 client/src/components/FileSources/Templates/CreateUserFileSource.vue create mode 100644 client/src/components/FileSources/Templates/SelectTemplate.vue create mode 100644 client/src/components/FileSources/Templates/TemplateSummary.vue create mode 100644 client/src/components/FileSources/Templates/TemplateSummaryPopover.vue create mode 100644 client/src/components/FileSources/Templates/TemplateSummarySpan.vue create mode 100644 client/src/components/FileSources/style.css delete mode 100644 client/src/components/ObjectStore/Instances/util.ts create mode 100644 client/src/stores/configTemplatesUtil.ts create mode 100644 client/src/stores/fileSourceInstancesStore.ts create mode 100644 client/src/stores/fileSourceTemplatesStore.ts create mode 100644 lib/galaxy/files/templates/__init__.py create mode 100644 lib/galaxy/files/templates/manager.py create mode 100644 lib/galaxy/files/templates/models.py create mode 100644 lib/galaxy/managers/file_source_instances.py create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py create mode 100644 lib/galaxy/util/config_templates.py create mode 100644 lib/galaxy/webapps/galaxy/api/file_sources.py create mode 100644 test/unit/app/managers/test_user_file_sources.py create mode 100644 test/unit/files/test_template_manager.py create mode 100644 test/unit/files/test_template_models.py diff --git a/client/src/api/configTemplates.ts b/client/src/api/configTemplates.ts new file mode 100644 index 000000000000..c9679f5876fa --- /dev/null +++ b/client/src/api/configTemplates.ts @@ -0,0 +1,21 @@ +import type { components } from "@/api/schema/schema"; + +export type Instance = + | components["schemas"]["UserFileSourceModel"] + | components["schemas"]["UserConcreteObjectStoreModel"]; + +export type TemplateVariable = components["schemas"]["TemplateVariable"]; +export type TemplateSecret = components["schemas"]["TemplateSecret"]; +export type VariableValueType = (string | boolean | number) | undefined; +export type VariableData = { [key: string]: VariableValueType }; +export type SecretData = { [key: string]: string }; + +export interface TemplateSummary { + description: string | null; + hidden?: boolean; + id: string; + name: string | null; + secrets?: TemplateSecret[] | null; + variables?: TemplateVariable[] | null; + version?: number; +} diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts new file mode 100644 index 000000000000..68935db4573f --- /dev/null +++ b/client/src/api/fileSources.ts @@ -0,0 +1,6 @@ +import { type components } from "@/api/schema"; + +export type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplateSummary"]; +export type FileSourceTemplateSummaries = FileSourceTemplateSummary[]; + +export type UserFileSourceModel = components["schemas"]["UserFileSourceModel"]; diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 6d606da14212..fae040a726a1 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -318,6 +318,22 @@ export interface paths { /** Download */ get: operations["download_api_drs_download__object_id__get"]; }; + "/api/file_source_instances": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_index"]; + /** Create a user-bound object store. */ + post: operations["file_sources__create_instance"]; + }; + "/api/file_source_instances/{user_file_source_id}": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_get"]; + /** Update or upgrade user file source instance. */ + put: operations["file_sources__instances_update"]; + }; + "/api/file_source_templates": { + /** Get a list of file source templates available to build user defined file sources from */ + get: operations["file_sources__templates_index"]; + }; "/api/folders/{folder_id}/contents": { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. @@ -5234,6 +5250,36 @@ export interface components { */ update_time: string; }; + /** FileSourceTemplateSummaries */ + FileSourceTemplateSummaries: components["schemas"]["FileSourceTemplateSummary"][]; + /** FileSourceTemplateSummary */ + FileSourceTemplateSummary: { + /** Description */ + description: string | null; + /** + * Hidden + * @default false + */ + hidden?: boolean; + /** Id */ + id: string; + /** Name */ + name: string | null; + /** Secrets */ + secrets?: components["schemas"]["TemplateSecret"][] | null; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Variables */ + variables?: components["schemas"]["TemplateVariable"][] | null; + /** + * Version + * @default 0 + */ + version?: number; + }; /** FilesSourcePlugin */ FilesSourcePlugin: { /** @@ -9915,13 +9961,6 @@ export interface components { */ up_to_date: boolean; }; - /** ObjectStoreTemplateSecret */ - ObjectStoreTemplateSecret: { - /** Help */ - help: string | null; - /** Name */ - name: string; - }; /** ObjectStoreTemplateSummaries */ ObjectStoreTemplateSummaries: components["schemas"]["ObjectStoreTemplateSummary"][]; /** ObjectStoreTemplateSummary */ @@ -9940,32 +9979,20 @@ export interface components { /** Name */ name: string | null; /** Secrets */ - secrets?: components["schemas"]["ObjectStoreTemplateSecret"][] | null; + secrets?: components["schemas"]["TemplateSecret"][] | null; /** * Type * @enum {string} */ type: "s3" | "azure_blob" | "disk" | "generic_s3"; /** Variables */ - variables?: components["schemas"]["ObjectStoreTemplateVariable"][] | null; + variables?: components["schemas"]["TemplateVariable"][] | null; /** * Version * @default 0 */ version?: number; }; - /** ObjectStoreTemplateVariable */ - ObjectStoreTemplateVariable: { - /** Help */ - help: string | null; - /** Name */ - name: string; - /** - * Type - * @enum {string} - */ - type: "string" | "boolean" | "integer"; - }; /** OutputReferenceByLabel */ OutputReferenceByLabel: { /** @@ -11842,6 +11869,25 @@ export interface components { * @enum {string} */ TaskState: "PENDING" | "STARTED" | "RETRY" | "FAILURE" | "SUCCESS"; + /** TemplateSecret */ + TemplateSecret: { + /** Help */ + help: string | null; + /** Name */ + name: string; + }; + /** TemplateVariable */ + TemplateVariable: { + /** Help */ + help: string | null; + /** Name */ + name: string; + /** + * Type + * @enum {string} + */ + type: "string" | "boolean" | "integer"; + }; /** ToolDataDetails */ ToolDataDetails: { /** @@ -12572,6 +12618,34 @@ export interface components { */ id: string; }; + /** UserFileSourceModel */ + UserFileSourceModel: { + /** Description */ + description: string | null; + /** Id */ + id: string | number; + /** Name */ + name: string; + /** Secrets */ + secrets: string[]; + /** Template Id */ + template_id: string; + /** Template Version */ + template_version: number; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Uri Root */ + uri_root: string; + /** Uuid */ + uuid: string; + /** Variables */ + variables: { + [key: string]: (string | boolean | number) | undefined; + } | null; + }; /** * UserModel * @description User in a transaction context. @@ -14761,6 +14835,142 @@ export interface operations { }; }; }; + file_sources__instances_index: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__create_instance: { + /** Create a user-bound object store. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_get: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_update: { + /** Update or upgrade user file source instance. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + requestBody: { + content: { + "application/json": + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__templates_index: { + /** Get a list of file source templates available to build user defined file sources from */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description A list of the configured file source templates. */ + 200: { + content: { + "application/json": components["schemas"]["FileSourceTemplateSummaries"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_folders__folder_id__contents_get: { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. diff --git a/client/src/components/ConfigTemplates/CreateInstance.test.ts b/client/src/components/ConfigTemplates/CreateInstance.test.ts new file mode 100644 index 000000000000..fe4e59ad4a59 --- /dev/null +++ b/client/src/components/ConfigTemplates/CreateInstance.test.ts @@ -0,0 +1,32 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import CreateInstance from "./CreateInstance.vue"; + +const localVue = getLocalVue(true); + +describe("CreateInstance", () => { + it("should render a loading message during loading", async () => { + const wrapper = shallowMount(CreateInstance, { + propsData: { + loading: true, + loadingMessage: "component loading...", + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeTruthy(); + }); + + it("should hide a loading message after loading", async () => { + const wrapper = shallowMount(CreateInstance, { + propsData: { + loading: false, + loadingMessage: "component loading...", + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeFalsy(); + }); +}); diff --git a/client/src/components/ConfigTemplates/CreateInstance.vue b/client/src/components/ConfigTemplates/CreateInstance.vue new file mode 100644 index 000000000000..334d341bc590 --- /dev/null +++ b/client/src/components/ConfigTemplates/CreateInstance.vue @@ -0,0 +1,21 @@ + + + diff --git a/client/src/components/ConfigTemplates/EditSecretsForm.test.ts b/client/src/components/ConfigTemplates/EditSecretsForm.test.ts new file mode 100644 index 000000000000..3a6252da0e29 --- /dev/null +++ b/client/src/components/ConfigTemplates/EditSecretsForm.test.ts @@ -0,0 +1,36 @@ +import { mount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import { STANDARD_FILE_SOURCE_TEMPLATE, STANDARD_OBJECT_STORE_TEMPLATE } from "./test_fixtures"; + +import EditSecretsForm from "./EditSecretsForm.vue"; + +const localVue = getLocalVue(true); + +describe("EditSecretsForm", () => { + it("should render a secrets for for file source templates", async () => { + const wrapper = mount(EditSecretsForm, { + propsData: { + template: STANDARD_FILE_SOURCE_TEMPLATE, + title: "Secrets FORM for file source", + }, + localVue, + }); + const titleText = wrapper.find(".portlet-title-text"); + expect(titleText.exists()).toBeTruthy(); + expect(titleText.text()).toEqual("Secrets FORM for file source"); + }); + + it("should render a secrets for for object store templates", async () => { + const wrapper = mount(EditSecretsForm, { + propsData: { + template: STANDARD_OBJECT_STORE_TEMPLATE, + title: "Secrets FORM for object store", + }, + localVue, + }); + const titleText = wrapper.find(".portlet-title-text"); + expect(titleText.exists()).toBeTruthy(); + expect(titleText.text()).toEqual("Secrets FORM for object store"); + }); +}); diff --git a/client/src/components/ConfigTemplates/EditSecretsForm.vue b/client/src/components/ConfigTemplates/EditSecretsForm.vue new file mode 100644 index 000000000000..e2a904bfa932 --- /dev/null +++ b/client/src/components/ConfigTemplates/EditSecretsForm.vue @@ -0,0 +1,34 @@ + + + diff --git a/client/src/components/ConfigTemplates/InstanceDropdown.test.ts b/client/src/components/ConfigTemplates/InstanceDropdown.test.ts new file mode 100644 index 000000000000..e0a54fd53489 --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceDropdown.test.ts @@ -0,0 +1,40 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import InstanceDropdown from "./InstanceDropdown.vue"; + +const localVue = getLocalVue(true); + +describe("InstanceDropdown", () => { + it("should render a drop down without upgrade if upgrade unavailable as an option", async () => { + const wrapper = shallowMount(InstanceDropdown, { + propsData: { + prefix: "file-source", + name: "my cool instance", + routeEdit: "/object_store_instance/edit", + routeUpgrade: "/object_store_instance/upgrade", + isUpgradable: false, + }, + localVue, + }); + const menu = wrapper.find(".dropdown-menu"); + const links = menu.findAll("a"); + expect(links.length).toBe(1); + }); + + it("should render a drop down with upgrade if upgrade available as an option", async () => { + const wrapper = shallowMount(InstanceDropdown, { + propsData: { + prefix: "file-source", + name: "my cool instance", + routeEdit: "/object_store_instance/edit", + routeUpgrade: "/object_store_instance/upgrade", + isUpgradable: true, + }, + localVue, + }); + const menu = wrapper.find(".dropdown-menu"); + const links = menu.findAll("a"); + expect(links.length).toBe(2); + }); +}); diff --git a/client/src/components/ConfigTemplates/InstanceDropdown.vue b/client/src/components/ConfigTemplates/InstanceDropdown.vue new file mode 100644 index 000000000000..e825479bc3e3 --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceDropdown.vue @@ -0,0 +1,50 @@ + + + diff --git a/client/src/components/ConfigTemplates/InstanceForm.test.ts b/client/src/components/ConfigTemplates/InstanceForm.test.ts new file mode 100644 index 000000000000..546a6670d68f --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceForm.test.ts @@ -0,0 +1,42 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import InstanceForm from "./InstanceForm.vue"; + +const localVue = getLocalVue(true); + +const inputs: any[] = []; +const SUBMIT_TITLE = "Submit the form!"; + +describe("InstanceForm", () => { + it("should render a loading message and not submit button if inputs is null", async () => { + const wrapper = shallowMount(InstanceForm, { + propsData: { + title: "MY FORM", + loading: true, + inputs: null, + submitTitle: SUBMIT_TITLE, + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeTruthy(); + expect(wrapper.find("#submit").exists()).toBeFalsy(); + }); + + it("should hide a loading message after loading", async () => { + const wrapper = shallowMount(InstanceForm, { + propsData: { + title: "MY FORM", + loading: false, + inputs: inputs, + submitTitle: SUBMIT_TITLE, + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeFalsy(); + expect(wrapper.find("#submit").exists()).toBeTruthy(); + expect(wrapper.find("#submit").text()).toEqual(SUBMIT_TITLE); + }); +}); diff --git a/client/src/components/ObjectStore/Instances/InstanceForm.vue b/client/src/components/ConfigTemplates/InstanceForm.vue similarity index 79% rename from client/src/components/ObjectStore/Instances/InstanceForm.vue rename to client/src/components/ConfigTemplates/InstanceForm.vue index 97d4525aa4cb..7e9bba13e4aa 100644 --- a/client/src/components/ObjectStore/Instances/InstanceForm.vue +++ b/client/src/components/ConfigTemplates/InstanceForm.vue @@ -1,4 +1,6 @@ diff --git a/client/src/components/ConfigTemplates/fields.ts b/client/src/components/ConfigTemplates/fields.ts new file mode 100644 index 000000000000..33a00f7e69f4 --- /dev/null +++ b/client/src/components/ConfigTemplates/fields.ts @@ -0,0 +1,25 @@ +import _l from "@/utils/localization"; + +export const NAME_FIELD = { + key: "name", + label: _l("Name"), + sortable: true, +}; + +export const DESCRIPTION_FIELD = { + key: "description", + label: _l("Description"), + sortable: true, +}; + +export const TYPE_FIELD = { + key: "type", + label: _l("Type"), + sortable: true, +}; + +export const TEMPLATE_FIELD = { + key: "template", + label: _l("From Template"), + sortable: true, +}; diff --git a/client/src/components/ConfigTemplates/formUtil.test.ts b/client/src/components/ConfigTemplates/formUtil.test.ts new file mode 100644 index 000000000000..42d498d62640 --- /dev/null +++ b/client/src/components/ConfigTemplates/formUtil.test.ts @@ -0,0 +1,45 @@ +import { createTemplateForm, upgradeForm } from "./formUtil"; +import { OBJECT_STORE_INSTANCE, STANDARD_FILE_SOURCE_TEMPLATE, STANDARD_OBJECT_STORE_TEMPLATE } from "./test_fixtures"; + +describe("formUtils", () => { + describe("createTemplateForm", () => { + it("should create a form from an object store templates", () => { + const form = createTemplateForm(STANDARD_OBJECT_STORE_TEMPLATE, "storage location"); + expect(form.length).toBe(6); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("_meta_name"); + expect(formEl0?.help).toEqual("Label this new storage location with a name."); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("_meta_description"); + }); + + it("should create a form from a file source templates", () => { + const form = createTemplateForm(STANDARD_FILE_SOURCE_TEMPLATE, "file source"); + expect(form.length).toBe(6); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("_meta_name"); + expect(formEl0?.help).toEqual("Label this new file source with a name."); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("_meta_description"); + }); + }); + + describe("upgradeForm", () => { + it("should create a form from an object store templates", () => { + const form = upgradeForm(STANDARD_OBJECT_STORE_TEMPLATE, OBJECT_STORE_INSTANCE); + expect(form.length).toBe(3); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("oldvar"); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("newvar"); + }); + + it("should only ask for new secrets during upgrade", () => { + const form = upgradeForm(STANDARD_OBJECT_STORE_TEMPLATE, OBJECT_STORE_INSTANCE); + expect(form.length).toBe(3); + const formEl0 = form[2]; + expect(formEl0?.name).toEqual("newsecret"); + expect(formEl0?.type).toEqual("password"); + }); + }); +}); diff --git a/client/src/components/ConfigTemplates/formUtil.ts b/client/src/components/ConfigTemplates/formUtil.ts new file mode 100644 index 000000000000..47ad9fc89096 --- /dev/null +++ b/client/src/components/ConfigTemplates/formUtil.ts @@ -0,0 +1,158 @@ +import type { + Instance, + SecretData, + TemplateSecret, + TemplateSummary, + TemplateVariable, + VariableData, + VariableValueType, +} from "@/api/configTemplates"; +import { markup } from "@/components/ObjectStore/configurationMarkdown"; + +export function metadataFormEntryName(what: string) { + return { + name: "_meta_name", + label: "Name", + type: "text", + optional: false, + help: `Label this new ${what} with a name.`, + }; +} + +export function metadataFormEntryDescription(what: string) { + return { + name: "_meta_description", + label: "Description", + optional: true, + type: "textarea", + help: `Provide some notes to yourself about this ${what} - perhaps to remind you how it is configured, where it stores the data, etc..`, + }; +} + +export function templateVariableFormEntry(variable: TemplateVariable, variableValue: VariableValueType) { + return { + name: variable.name, + type: "text", + help: markup(variable.help || "", true), + value: variableValue, + }; +} + +export function templateSecretFormEntry(secret: TemplateSecret) { + return { + name: secret.name, + type: "password", + help: markup(secret.help || "", true), + value: "", + }; +} + +export function editTemplateForm(template: TemplateSummary, what: string, instance: Instance) { + const form = []; + const nameInput = metadataFormEntryName(what); + form.push({ value: instance.name ?? "", ...nameInput }); + + const descriptionInput = metadataFormEntryDescription(what); + form.push({ value: instance.description ?? "", ...descriptionInput }); + + const variables = template.variables ?? []; + const variableValues: VariableData = instance.variables || {}; + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, variableValues[variable.name])); + } + return form; +} + +export function editFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const name = formData["_meta_name"]; + const description = formData["_meta_description"]; + const variableData: VariableData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + const payload = { + name: name, + description: description, + variables: variableData, + }; + return payload; +} + +export function createTemplateForm(template: TemplateSummary, what: string) { + const form = []; + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + form.push(metadataFormEntryName(what)); + form.push(metadataFormEntryDescription(what)); + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, undefined)); + } + for (const secret of secrets) { + form.push(templateSecretFormEntry(secret)); + } + return form; +} + +export function createFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + const variableData: VariableData = {}; + const secretData: SecretData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + for (const secret of secrets) { + secretData[secret.name] = formData[secret.name]; + } + const name: string = formData._meta_name; + const description: string = formData._meta_description; + const payload = { + name: name, + description: description, + secrets: secretData, + variables: variableData, + template_id: template.id, + template_version: template.version ?? 0, + }; + return payload; +} + +export function upgradeForm(template: TemplateSummary, instance: Instance): Array { + const form = []; + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + const variableValues: VariableData = instance.variables || {}; + const secretsSet = instance.secrets || []; + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, variableValues[variable.name])); + } + for (const secret of secrets) { + const secretName = secret.name; + if (secretsSet.indexOf(secretName) >= 0) { + console.log("skipping..."); + } else { + form.push(templateSecretFormEntry(secret)); + } + } + return form; +} + +export function upgradeFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const variableData: VariableData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + const secrets = {}; + // ideally we would be able to force a template version here, + // maybe rework backend types to force this in the API response + // even if we don't need it in the config files + const templateVersion: number = template.version || 0; + const payload = { + template_version: templateVersion, + variables: variableData, + secrets: secrets, + }; + return payload; +} diff --git a/client/src/components/ObjectStore/Instances/icons.ts b/client/src/components/ConfigTemplates/icons.ts similarity index 100% rename from client/src/components/ObjectStore/Instances/icons.ts rename to client/src/components/ConfigTemplates/icons.ts diff --git a/client/src/components/ConfigTemplates/test_fixtures.ts b/client/src/components/ConfigTemplates/test_fixtures.ts new file mode 100644 index 000000000000..bbf4530a762f --- /dev/null +++ b/client/src/components/ConfigTemplates/test_fixtures.ts @@ -0,0 +1,82 @@ +import type { FileSourceTemplateSummary } from "@/api/fileSources"; +import type { UserConcreteObjectStore } from "@/components/ObjectStore/Instances/types"; +import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templates/types"; + +export const STANDARD_OBJECT_STORE_TEMPLATE: ObjectStoreTemplateSummary = { + type: "s3", + name: "moo", + description: null, + variables: [ + { + name: "oldvar", + type: "string", + help: "old var help", + }, + { + name: "newvar", + type: "string", + help: "new var help", + }, + ], + secrets: [ + { + name: "oldsecret", + help: "old secret help", + }, + { + name: "newsecret", + help: "new secret help", + }, + ], + id: "moo", + version: 2, + badges: [], +}; + +export const STANDARD_FILE_SOURCE_TEMPLATE: FileSourceTemplateSummary = { + type: "s3fs", + name: "moo", + description: null, + variables: [ + { + name: "oldvar", + type: "string", + help: "old var help", + }, + { + name: "newvar", + type: "string", + help: "new var help", + }, + ], + secrets: [ + { + name: "oldsecret", + help: "old secret help", + }, + { + name: "newsecret", + help: "new secret help", + }, + ], + id: "moo", + version: 2, +}; + +export const OBJECT_STORE_INSTANCE: UserConcreteObjectStore = { + type: "s3", + name: "moo", + description: undefined, + template_id: "moo", + template_version: 1, + badges: [], + variables: { + oldvar: "my old value", + droppedvar: "this will be dropped", + }, + secrets: ["oldsecret", "droppedsecret"], + quota: { enabled: false }, + private: false, + id: 4, + uuid: "112f889f-72d7-4619-a8e8-510a8c685aa7", +}; diff --git a/client/src/components/FileSources/FileSourceTypeSpan.vue b/client/src/components/FileSources/FileSourceTypeSpan.vue new file mode 100644 index 000000000000..538b635c4a42 --- /dev/null +++ b/client/src/components/FileSources/FileSourceTypeSpan.vue @@ -0,0 +1,23 @@ + + + + + diff --git a/client/src/components/FileSources/Instances/CreateForm.vue b/client/src/components/FileSources/Instances/CreateForm.vue new file mode 100644 index 000000000000..8079cf36735d --- /dev/null +++ b/client/src/components/FileSources/Instances/CreateForm.vue @@ -0,0 +1,53 @@ + + diff --git a/client/src/components/FileSources/Instances/CreateInstance.vue b/client/src/components/FileSources/Instances/CreateInstance.vue new file mode 100644 index 000000000000..9b93e2d7c269 --- /dev/null +++ b/client/src/components/FileSources/Instances/CreateInstance.vue @@ -0,0 +1,34 @@ + + + diff --git a/client/src/components/FileSources/Instances/EditInstance.vue b/client/src/components/FileSources/Instances/EditInstance.vue new file mode 100644 index 000000000000..311bb7fdc838 --- /dev/null +++ b/client/src/components/FileSources/Instances/EditInstance.vue @@ -0,0 +1,77 @@ + + diff --git a/client/src/components/FileSources/Instances/EditSecrets.vue b/client/src/components/FileSources/Instances/EditSecrets.vue new file mode 100644 index 000000000000..dd14126a1242 --- /dev/null +++ b/client/src/components/FileSources/Instances/EditSecrets.vue @@ -0,0 +1,28 @@ + + diff --git a/client/src/components/FileSources/Instances/InstanceDropdown.vue b/client/src/components/FileSources/Instances/InstanceDropdown.vue new file mode 100644 index 000000000000..f71e3fd23022 --- /dev/null +++ b/client/src/components/FileSources/Instances/InstanceDropdown.vue @@ -0,0 +1,30 @@ + + + diff --git a/client/src/components/FileSources/Instances/ManageIndex.vue b/client/src/components/FileSources/Instances/ManageIndex.vue new file mode 100644 index 000000000000..3ad9601cbaaa --- /dev/null +++ b/client/src/components/FileSources/Instances/ManageIndex.vue @@ -0,0 +1,68 @@ + + + diff --git a/client/src/components/FileSources/Instances/UpgradeForm.vue b/client/src/components/FileSources/Instances/UpgradeForm.vue new file mode 100644 index 000000000000..4ca85e7caf67 --- /dev/null +++ b/client/src/components/FileSources/Instances/UpgradeForm.vue @@ -0,0 +1,63 @@ + + diff --git a/client/src/components/FileSources/Instances/UpgradeInstance.vue b/client/src/components/FileSources/Instances/UpgradeInstance.vue new file mode 100644 index 000000000000..a2aa1ed672f4 --- /dev/null +++ b/client/src/components/FileSources/Instances/UpgradeInstance.vue @@ -0,0 +1,27 @@ + + diff --git a/client/src/components/FileSources/Instances/instance.ts b/client/src/components/FileSources/Instances/instance.ts new file mode 100644 index 000000000000..f45fd6bf76dd --- /dev/null +++ b/client/src/components/FileSources/Instances/instance.ts @@ -0,0 +1,23 @@ +import { computed, type Ref } from "vue"; + +import type { FileSourceTemplateSummary, UserFileSourceModel } from "@/api/fileSources"; +import { useFileSourceInstancesStore } from "@/stores/fileSourceInstancesStore"; +import { useFileSourceTemplatesStore } from "@/stores/fileSourceTemplatesStore"; + +export function useInstanceAndTemplate(instanceIdRef: Ref) { + const fileSourceTemplatesStore = useFileSourceTemplatesStore(); + const fileSourceInstancesStore = useFileSourceInstancesStore(); + fileSourceInstancesStore.fetchInstances(); + fileSourceTemplatesStore.fetchTemplates(); + + const instance = computed( + () => fileSourceInstancesStore.getInstance(instanceIdRef.value) || null + ); + const template = computed(() => + instance.value + ? fileSourceTemplatesStore.getTemplate(instance.value?.template_id, instance.value?.template_version) + : null + ); + + return { instance, template }; +} diff --git a/client/src/components/FileSources/Instances/routing.ts b/client/src/components/FileSources/Instances/routing.ts new file mode 100644 index 000000000000..6d2ed12bae6e --- /dev/null +++ b/client/src/components/FileSources/Instances/routing.ts @@ -0,0 +1,16 @@ +import { useRouter } from "vue-router/composables"; + +export function useInstanceRouting() { + const router = useRouter(); + + async function goToIndex(query: Record<"message", string>) { + router.push({ + path: "/file_source_instances/index", + query: query, + }); + } + + return { + goToIndex, + }; +} diff --git a/client/src/components/FileSources/Instances/services.ts b/client/src/components/FileSources/Instances/services.ts new file mode 100644 index 000000000000..d5d4b8553826 --- /dev/null +++ b/client/src/components/FileSources/Instances/services.ts @@ -0,0 +1,4 @@ +import { fetcher } from "@/api/schema/fetcher"; + +export const create = fetcher.path("/api/file_source_instances").method("post").create(); +export const update = fetcher.path("/api/file_source_instances/{user_file_source_id}").method("put").create(); diff --git a/client/src/components/FileSources/Templates/CreateUserFileSource.vue b/client/src/components/FileSources/Templates/CreateUserFileSource.vue new file mode 100644 index 000000000000..5cc4a9f8e3aa --- /dev/null +++ b/client/src/components/FileSources/Templates/CreateUserFileSource.vue @@ -0,0 +1,30 @@ + + diff --git a/client/src/components/FileSources/Templates/SelectTemplate.vue b/client/src/components/FileSources/Templates/SelectTemplate.vue new file mode 100644 index 000000000000..6e45ab9f8d3d --- /dev/null +++ b/client/src/components/FileSources/Templates/SelectTemplate.vue @@ -0,0 +1,39 @@ + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummary.vue b/client/src/components/FileSources/Templates/TemplateSummary.vue new file mode 100644 index 000000000000..6cef09f37ed1 --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummary.vue @@ -0,0 +1,22 @@ + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue b/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue new file mode 100644 index 000000000000..c056e03d9b72 --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue @@ -0,0 +1,19 @@ + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummarySpan.vue b/client/src/components/FileSources/Templates/TemplateSummarySpan.vue new file mode 100644 index 000000000000..1c4291dc3aad --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummarySpan.vue @@ -0,0 +1,32 @@ + + + diff --git a/client/src/components/FileSources/style.css b/client/src/components/FileSources/style.css new file mode 100644 index 000000000000..c2962e44de2e --- /dev/null +++ b/client/src/components/FileSources/style.css @@ -0,0 +1,4 @@ +.file-source-help-on-hover { + text-decoration-line: underline; + text-decoration-style: dashed; +} diff --git a/client/src/components/ObjectStore/Instances/CreateForm.vue b/client/src/components/ObjectStore/Instances/CreateForm.vue index e82f0f1c484a..cfb982a8697c 100644 --- a/client/src/components/ObjectStore/Instances/CreateForm.vue +++ b/client/src/components/ObjectStore/Instances/CreateForm.vue @@ -1,18 +1,15 @@ diff --git a/client/src/components/ObjectStore/Instances/EditInstance.vue b/client/src/components/ObjectStore/Instances/EditInstance.vue index 16e4dd9c4631..2569be940178 100644 --- a/client/src/components/ObjectStore/Instances/EditInstance.vue +++ b/client/src/components/ObjectStore/Instances/EditInstance.vue @@ -1,19 +1,16 @@ diff --git a/client/src/components/ObjectStore/Instances/EditSecrets.vue b/client/src/components/ObjectStore/Instances/EditSecrets.vue index 42038241d42d..56c1e7cd9641 100644 --- a/client/src/components/ObjectStore/Instances/EditSecrets.vue +++ b/client/src/components/ObjectStore/Instances/EditSecrets.vue @@ -6,8 +6,7 @@ import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templa import { update } from "./services"; import type { UserConcreteObjectStore } from "./types"; -import VaultSecret from "./VaultSecret.vue"; -import FormCard from "@/components/Form/FormCard.vue"; +import EditSecretsForm from "@/components/ConfigTemplates/EditSecretsForm.vue"; interface Props { objectStore: UserConcreteObjectStore; @@ -26,14 +25,5 @@ async function onUpdate(secretName: string, secretValue: string) { } diff --git a/client/src/components/ObjectStore/Instances/InstanceDropdown.vue b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue index aaa2836c8289..475d802d391b 100644 --- a/client/src/components/ObjectStore/Instances/InstanceDropdown.vue +++ b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue @@ -1,19 +1,13 @@ diff --git a/client/src/components/ObjectStore/Instances/ManageIndex.vue b/client/src/components/ObjectStore/Instances/ManageIndex.vue index 9695f492dad2..1f87d85ebc09 100644 --- a/client/src/components/ObjectStore/Instances/ManageIndex.vue +++ b/client/src/components/ObjectStore/Instances/ManageIndex.vue @@ -1,20 +1,20 @@ diff --git a/client/src/components/ObjectStore/Instances/types.ts b/client/src/components/ObjectStore/Instances/types.ts index bd956468c7b8..0176eaec757a 100644 --- a/client/src/components/ObjectStore/Instances/types.ts +++ b/client/src/components/ObjectStore/Instances/types.ts @@ -2,8 +2,3 @@ import type { components } from "@/api/schema/schema"; export type UserConcreteObjectStore = components["schemas"]["UserConcreteObjectStoreModel"]; export type CreateInstancePayload = components["schemas"]["CreateInstancePayload"]; -export type ObjectStoreTemplateVariable = components["schemas"]["ObjectStoreTemplateVariable"]; -export type ObjectStoreTemplateSecret = components["schemas"]["ObjectStoreTemplateSecret"]; -export type VariableValueType = (string | boolean | number) | undefined; -export type VariableData = { [key: string]: VariableValueType }; -export type SecretData = { [key: string]: string }; diff --git a/client/src/components/ObjectStore/Instances/util.ts b/client/src/components/ObjectStore/Instances/util.ts deleted file mode 100644 index 77b99b11d73f..000000000000 --- a/client/src/components/ObjectStore/Instances/util.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { markup } from "@/components/ObjectStore/configurationMarkdown"; - -import type { ObjectStoreTemplateSecret, ObjectStoreTemplateVariable, VariableValueType } from "./types"; - -export function metadataFormEntryName() { - return { - name: "_meta_name", - label: "Name", - type: "text", - optional: false, - help: "Label this new object store a name.", - }; -} - -export function metadataFormEntryDescription() { - return { - name: "_meta_description", - label: "Description", - optional: true, - type: "textarea", - help: "Provide some notes to yourself about this object store - perhaps to remind you how it is configured, where it stores the data, etc..", - }; -} - -export function templateVariableFormEntry(variable: ObjectStoreTemplateVariable, variableValue: VariableValueType) { - return { - name: variable.name, - type: "text", - help: markup(variable.help || "", true), - value: variableValue, - }; -} - -export function templateSecretFormEntry(secret: ObjectStoreTemplateSecret) { - return { - name: secret.name, - type: "password", - help: markup(secret.help || "", true), - value: "", - }; -} diff --git a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue index 7ed555be53dc..f783b34b4f82 100644 --- a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue +++ b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue @@ -1,11 +1,11 @@ diff --git a/client/src/components/ObjectStore/Templates/SelectTemplate.vue b/client/src/components/ObjectStore/Templates/SelectTemplate.vue index 5e99fbb0807c..a8e8362eb83a 100644 --- a/client/src/components/ObjectStore/Templates/SelectTemplate.vue +++ b/client/src/components/ObjectStore/Templates/SelectTemplate.vue @@ -2,6 +2,7 @@ import type { ObjectStoreTemplateSummaries } from "./types"; import TemplateSummaryPopover from "./TemplateSummaryPopover.vue"; +import SelectTemplate from "@/components/ConfigTemplates/SelectTemplate.vue"; interface SelectTemplateProps { templates: ObjectStoreTemplateSummaries; @@ -23,26 +24,12 @@ async function handleSubmit(templateId: string) {