diff --git a/client/src/api/configTemplates.ts b/client/src/api/configTemplates.ts new file mode 100644 index 000000000000..c9679f5876fa --- /dev/null +++ b/client/src/api/configTemplates.ts @@ -0,0 +1,21 @@ +import type { components } from "@/api/schema/schema"; + +export type Instance = + | components["schemas"]["UserFileSourceModel"] + | components["schemas"]["UserConcreteObjectStoreModel"]; + +export type TemplateVariable = components["schemas"]["TemplateVariable"]; +export type TemplateSecret = components["schemas"]["TemplateSecret"]; +export type VariableValueType = (string | boolean | number) | undefined; +export type VariableData = { [key: string]: VariableValueType }; +export type SecretData = { [key: string]: string }; + +export interface TemplateSummary { + description: string | null; + hidden?: boolean; + id: string; + name: string | null; + secrets?: TemplateSecret[] | null; + variables?: TemplateVariable[] | null; + version?: number; +} diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts new file mode 100644 index 000000000000..68935db4573f --- /dev/null +++ b/client/src/api/fileSources.ts @@ -0,0 +1,6 @@ +import { type components } from "@/api/schema"; + +export type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplateSummary"]; +export type FileSourceTemplateSummaries = FileSourceTemplateSummary[]; + +export type UserFileSourceModel = components["schemas"]["UserFileSourceModel"]; diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 3fb5f0c1a609..b798939f5c82 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -318,6 +318,22 @@ export interface paths { /** Download */ get: operations["download_api_drs_download__object_id__get"]; }; + "/api/file_source_instances": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_index"]; + /** Create a user-bound object store. */ + post: operations["file_sources__create_instance"]; + }; + "/api/file_source_instances/{user_file_source_id}": { + /** Get a list of persisted file source instances defined by the requesting user. */ + get: operations["file_sources__instances_get"]; + /** Update or upgrade user file source instance. */ + put: operations["file_sources__instances_update"]; + }; + "/api/file_source_templates": { + /** Get a list of file source templates available to build user defined file sources from */ + get: operations["file_sources__templates_index"]; + }; "/api/folders/{folder_id}/contents": { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. @@ -5234,6 +5250,36 @@ export interface components { */ update_time: string; }; + /** FileSourceTemplateSummaries */ + FileSourceTemplateSummaries: components["schemas"]["FileSourceTemplateSummary"][]; + /** FileSourceTemplateSummary */ + FileSourceTemplateSummary: { + /** Description */ + description: string | null; + /** + * Hidden + * @default false + */ + hidden?: boolean; + /** Id */ + id: string; + /** Name */ + name: string | null; + /** Secrets */ + secrets?: components["schemas"]["TemplateSecret"][] | null; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Variables */ + variables?: components["schemas"]["TemplateVariable"][] | null; + /** + * Version + * @default 0 + */ + version?: number; + }; /** FilesSourcePlugin */ FilesSourcePlugin: { /** @@ -9915,13 +9961,6 @@ export interface components { */ up_to_date: boolean; }; - /** ObjectStoreTemplateSecret */ - ObjectStoreTemplateSecret: { - /** Help */ - help: string | null; - /** Name */ - name: string; - }; /** ObjectStoreTemplateSummaries */ ObjectStoreTemplateSummaries: components["schemas"]["ObjectStoreTemplateSummary"][]; /** ObjectStoreTemplateSummary */ @@ -9940,32 +9979,20 @@ export interface components { /** Name */ name: string | null; /** Secrets */ - secrets?: components["schemas"]["ObjectStoreTemplateSecret"][] | null; + secrets?: components["schemas"]["TemplateSecret"][] | null; /** * Type * @enum {string} */ type: "s3" | "azure_blob" | "disk" | "generic_s3"; /** Variables */ - variables?: components["schemas"]["ObjectStoreTemplateVariable"][] | null; + variables?: components["schemas"]["TemplateVariable"][] | null; /** * Version * @default 0 */ version?: number; }; - /** ObjectStoreTemplateVariable */ - ObjectStoreTemplateVariable: { - /** Help */ - help: string | null; - /** Name */ - name: string; - /** - * Type - * @enum {string} - */ - type: "string" | "boolean" | "integer"; - }; /** OutputReferenceByLabel */ OutputReferenceByLabel: { /** @@ -11842,6 +11869,25 @@ export interface components { * @enum {string} */ TaskState: "PENDING" | "STARTED" | "RETRY" | "FAILURE" | "SUCCESS"; + /** TemplateSecret */ + TemplateSecret: { + /** Help */ + help: string | null; + /** Name */ + name: string; + }; + /** TemplateVariable */ + TemplateVariable: { + /** Help */ + help: string | null; + /** Name */ + name: string; + /** + * Type + * @enum {string} + */ + type: "string" | "path_component" | "boolean" | "integer"; + }; /** ToolDataDetails */ ToolDataDetails: { /** @@ -12572,6 +12618,34 @@ export interface components { */ id: string; }; + /** UserFileSourceModel */ + UserFileSourceModel: { + /** Description */ + description: string | null; + /** Id */ + id: string | number; + /** Name */ + name: string; + /** Secrets */ + secrets: string[]; + /** Template Id */ + template_id: string; + /** Template Version */ + template_version: number; + /** + * Type + * @enum {string} + */ + type: "posix" | "s3fs"; + /** Uri Root */ + uri_root: string; + /** Uuid */ + uuid: string; + /** Variables */ + variables: { + [key: string]: (string | boolean | number) | undefined; + } | null; + }; /** * UserModel * @description User in a transaction context. @@ -14761,6 +14835,142 @@ export interface operations { }; }; }; + file_sources__instances_index: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"][]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__create_instance: { + /** Create a user-bound object store. */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CreateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_get: { + /** Get a list of persisted file source instances defined by the requesting user. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__instances_update: { + /** Update or upgrade user file source instance. */ + parameters: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + /** @description The index for a persisted UserFileSourceStore object. */ + path: { + user_file_source_id: string; + }; + }; + requestBody: { + content: { + "application/json": + | components["schemas"]["UpdateInstanceSecretPayload"] + | components["schemas"]["UpgradeInstancePayload"] + | components["schemas"]["UpdateInstancePayload"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + content: { + "application/json": components["schemas"]["UserFileSourceModel"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + file_sources__templates_index: { + /** Get a list of file source templates available to build user defined file sources from */ + parameters?: { + /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ + header?: { + "run-as"?: string | null; + }; + }; + responses: { + /** @description A list of the configured file source templates. */ + 200: { + content: { + "application/json": components["schemas"]["FileSourceTemplateSummaries"]; + }; + }; + /** @description Validation Error */ + 422: { + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; index_api_folders__folder_id__contents_get: { /** * Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder. diff --git a/client/src/components/ConfigTemplates/CreateInstance.test.ts b/client/src/components/ConfigTemplates/CreateInstance.test.ts new file mode 100644 index 000000000000..fe4e59ad4a59 --- /dev/null +++ b/client/src/components/ConfigTemplates/CreateInstance.test.ts @@ -0,0 +1,32 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import CreateInstance from "./CreateInstance.vue"; + +const localVue = getLocalVue(true); + +describe("CreateInstance", () => { + it("should render a loading message during loading", async () => { + const wrapper = shallowMount(CreateInstance, { + propsData: { + loading: true, + loadingMessage: "component loading...", + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeTruthy(); + }); + + it("should hide a loading message after loading", async () => { + const wrapper = shallowMount(CreateInstance, { + propsData: { + loading: false, + loadingMessage: "component loading...", + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeFalsy(); + }); +}); diff --git a/client/src/components/ConfigTemplates/CreateInstance.vue b/client/src/components/ConfigTemplates/CreateInstance.vue new file mode 100644 index 000000000000..334d341bc590 --- /dev/null +++ b/client/src/components/ConfigTemplates/CreateInstance.vue @@ -0,0 +1,21 @@ + + + + + + + + + + diff --git a/client/src/components/ConfigTemplates/EditSecretsForm.test.ts b/client/src/components/ConfigTemplates/EditSecretsForm.test.ts new file mode 100644 index 000000000000..3a6252da0e29 --- /dev/null +++ b/client/src/components/ConfigTemplates/EditSecretsForm.test.ts @@ -0,0 +1,36 @@ +import { mount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import { STANDARD_FILE_SOURCE_TEMPLATE, STANDARD_OBJECT_STORE_TEMPLATE } from "./test_fixtures"; + +import EditSecretsForm from "./EditSecretsForm.vue"; + +const localVue = getLocalVue(true); + +describe("EditSecretsForm", () => { + it("should render a secrets for for file source templates", async () => { + const wrapper = mount(EditSecretsForm, { + propsData: { + template: STANDARD_FILE_SOURCE_TEMPLATE, + title: "Secrets FORM for file source", + }, + localVue, + }); + const titleText = wrapper.find(".portlet-title-text"); + expect(titleText.exists()).toBeTruthy(); + expect(titleText.text()).toEqual("Secrets FORM for file source"); + }); + + it("should render a secrets for for object store templates", async () => { + const wrapper = mount(EditSecretsForm, { + propsData: { + template: STANDARD_OBJECT_STORE_TEMPLATE, + title: "Secrets FORM for object store", + }, + localVue, + }); + const titleText = wrapper.find(".portlet-title-text"); + expect(titleText.exists()).toBeTruthy(); + expect(titleText.text()).toEqual("Secrets FORM for object store"); + }); +}); diff --git a/client/src/components/ConfigTemplates/EditSecretsForm.vue b/client/src/components/ConfigTemplates/EditSecretsForm.vue new file mode 100644 index 000000000000..e2a904bfa932 --- /dev/null +++ b/client/src/components/ConfigTemplates/EditSecretsForm.vue @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + diff --git a/client/src/components/ConfigTemplates/InstanceDropdown.test.ts b/client/src/components/ConfigTemplates/InstanceDropdown.test.ts new file mode 100644 index 000000000000..e0a54fd53489 --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceDropdown.test.ts @@ -0,0 +1,40 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import InstanceDropdown from "./InstanceDropdown.vue"; + +const localVue = getLocalVue(true); + +describe("InstanceDropdown", () => { + it("should render a drop down without upgrade if upgrade unavailable as an option", async () => { + const wrapper = shallowMount(InstanceDropdown, { + propsData: { + prefix: "file-source", + name: "my cool instance", + routeEdit: "/object_store_instance/edit", + routeUpgrade: "/object_store_instance/upgrade", + isUpgradable: false, + }, + localVue, + }); + const menu = wrapper.find(".dropdown-menu"); + const links = menu.findAll("a"); + expect(links.length).toBe(1); + }); + + it("should render a drop down with upgrade if upgrade available as an option", async () => { + const wrapper = shallowMount(InstanceDropdown, { + propsData: { + prefix: "file-source", + name: "my cool instance", + routeEdit: "/object_store_instance/edit", + routeUpgrade: "/object_store_instance/upgrade", + isUpgradable: true, + }, + localVue, + }); + const menu = wrapper.find(".dropdown-menu"); + const links = menu.findAll("a"); + expect(links.length).toBe(2); + }); +}); diff --git a/client/src/components/ConfigTemplates/InstanceDropdown.vue b/client/src/components/ConfigTemplates/InstanceDropdown.vue new file mode 100644 index 000000000000..e825479bc3e3 --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceDropdown.vue @@ -0,0 +1,50 @@ + + + + + + + {{ name }} + + + + + Upgrade + + + + Edit configuration + + + + diff --git a/client/src/components/ConfigTemplates/InstanceForm.test.ts b/client/src/components/ConfigTemplates/InstanceForm.test.ts new file mode 100644 index 000000000000..546a6670d68f --- /dev/null +++ b/client/src/components/ConfigTemplates/InstanceForm.test.ts @@ -0,0 +1,42 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import InstanceForm from "./InstanceForm.vue"; + +const localVue = getLocalVue(true); + +const inputs: any[] = []; +const SUBMIT_TITLE = "Submit the form!"; + +describe("InstanceForm", () => { + it("should render a loading message and not submit button if inputs is null", async () => { + const wrapper = shallowMount(InstanceForm, { + propsData: { + title: "MY FORM", + loading: true, + inputs: null, + submitTitle: SUBMIT_TITLE, + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeTruthy(); + expect(wrapper.find("#submit").exists()).toBeFalsy(); + }); + + it("should hide a loading message after loading", async () => { + const wrapper = shallowMount(InstanceForm, { + propsData: { + title: "MY FORM", + loading: false, + inputs: inputs, + submitTitle: SUBMIT_TITLE, + }, + localVue, + }); + const loadingSpan = wrapper.findComponent({ name: "LoadingSpan" }).exists(); + expect(loadingSpan).toBeFalsy(); + expect(wrapper.find("#submit").exists()).toBeTruthy(); + expect(wrapper.find("#submit").text()).toEqual(SUBMIT_TITLE); + }); +}); diff --git a/client/src/components/ObjectStore/Instances/InstanceForm.vue b/client/src/components/ConfigTemplates/InstanceForm.vue similarity index 79% rename from client/src/components/ObjectStore/Instances/InstanceForm.vue rename to client/src/components/ConfigTemplates/InstanceForm.vue index 97d4525aa4cb..7e9bba13e4aa 100644 --- a/client/src/components/ObjectStore/Instances/InstanceForm.vue +++ b/client/src/components/ConfigTemplates/InstanceForm.vue @@ -1,4 +1,6 @@ - + @@ -34,9 +37,9 @@ async function handleSubmit() { - + {{ submitTitle }} - + diff --git a/client/src/components/ConfigTemplates/ManageIndexHeader.vue b/client/src/components/ConfigTemplates/ManageIndexHeader.vue new file mode 100644 index 000000000000..880c1a92c3d7 --- /dev/null +++ b/client/src/components/ConfigTemplates/ManageIndexHeader.vue @@ -0,0 +1,35 @@ + + + + + + {{ message || "" }} + + + + + + {{ _l("Create") }} + + + + + diff --git a/client/src/components/ConfigTemplates/SelectTemplate.test.js b/client/src/components/ConfigTemplates/SelectTemplate.test.js new file mode 100644 index 000000000000..ce631ceb54d0 --- /dev/null +++ b/client/src/components/ConfigTemplates/SelectTemplate.test.js @@ -0,0 +1,33 @@ +import { mount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import { STANDARD_FILE_SOURCE_TEMPLATE } from "./test_fixtures"; + +import SelectTemplate from "./SelectTemplate.vue"; + +const localVue = getLocalVue(true); + +const help = "some help text about selection"; + +describe("SelectTemplate", () => { + it("should render a selection row for supplied templates", async () => { + const wrapper = mount(SelectTemplate, { + propsData: { + templates: [STANDARD_FILE_SOURCE_TEMPLATE], + selectText: help, + idPrefix: "file-source", + }, + localVue, + }); + console.log(wrapper.html()); + const helpText = wrapper.find(".file-source-template-select-help"); + expect(helpText.exists()).toBeTruthy(); + expect(helpText.text()).toBeLocalizationOf(help); + const buttons = wrapper.findAll("button"); + expect(buttons.length).toBe(1); + const button = buttons.at(0); + expect(button.attributes().id).toEqual("file-source-template-button-moo"); + expect(button.attributes()["data-template-id"]).toEqual("moo"); + expect(button.text()).toEqual("moo"); + }); +}); diff --git a/client/src/components/ConfigTemplates/SelectTemplate.vue b/client/src/components/ConfigTemplates/SelectTemplate.vue new file mode 100644 index 000000000000..012edb83ebf0 --- /dev/null +++ b/client/src/components/ConfigTemplates/SelectTemplate.vue @@ -0,0 +1,44 @@ + + + + + + + {{ template.name }} + + + + + + {{ selectText }} + + + + diff --git a/client/src/components/ConfigTemplates/TemplateSummaryPopover.test.ts b/client/src/components/ConfigTemplates/TemplateSummaryPopover.test.ts new file mode 100644 index 000000000000..e230874d65e7 --- /dev/null +++ b/client/src/components/ConfigTemplates/TemplateSummaryPopover.test.ts @@ -0,0 +1,22 @@ +import { shallowMount } from "@vue/test-utils"; +import { getLocalVue } from "tests/jest/helpers"; + +import { STANDARD_FILE_SOURCE_TEMPLATE } from "./test_fixtures"; + +import TemplateSummaryPopover from "./TemplateSummaryPopover.vue"; + +const localVue = getLocalVue(true); + +describe("TemplateSummaryPopover", () => { + it("should render a secrets for for file source templates", async () => { + const wrapper = shallowMount(TemplateSummaryPopover, { + propsData: { + template: STANDARD_FILE_SOURCE_TEMPLATE, + target: "popover-target", + }, + localVue, + }); + const popover = wrapper.findComponent({ name: "BPopover" }); + expect(popover.attributes().target).toEqual("popover-target"); + }); +}); diff --git a/client/src/components/ConfigTemplates/TemplateSummaryPopover.vue b/client/src/components/ConfigTemplates/TemplateSummaryPopover.vue new file mode 100644 index 000000000000..77484b6b8c9a --- /dev/null +++ b/client/src/components/ConfigTemplates/TemplateSummaryPopover.vue @@ -0,0 +1,21 @@ + + + + + {{ props.template.name }} + + + diff --git a/client/src/components/ObjectStore/Instances/VaultSecret.vue b/client/src/components/ConfigTemplates/VaultSecret.vue similarity index 97% rename from client/src/components/ObjectStore/Instances/VaultSecret.vue rename to client/src/components/ConfigTemplates/VaultSecret.vue index 41fb485bd67d..a934c68e2aae 100644 --- a/client/src/components/ObjectStore/Instances/VaultSecret.vue +++ b/client/src/components/ConfigTemplates/VaultSecret.vue @@ -61,5 +61,5 @@ async function onOk() { diff --git a/client/src/components/ConfigTemplates/fields.ts b/client/src/components/ConfigTemplates/fields.ts new file mode 100644 index 000000000000..33a00f7e69f4 --- /dev/null +++ b/client/src/components/ConfigTemplates/fields.ts @@ -0,0 +1,25 @@ +import _l from "@/utils/localization"; + +export const NAME_FIELD = { + key: "name", + label: _l("Name"), + sortable: true, +}; + +export const DESCRIPTION_FIELD = { + key: "description", + label: _l("Description"), + sortable: true, +}; + +export const TYPE_FIELD = { + key: "type", + label: _l("Type"), + sortable: true, +}; + +export const TEMPLATE_FIELD = { + key: "template", + label: _l("From Template"), + sortable: true, +}; diff --git a/client/src/components/ConfigTemplates/formUtil.test.ts b/client/src/components/ConfigTemplates/formUtil.test.ts new file mode 100644 index 000000000000..42d498d62640 --- /dev/null +++ b/client/src/components/ConfigTemplates/formUtil.test.ts @@ -0,0 +1,45 @@ +import { createTemplateForm, upgradeForm } from "./formUtil"; +import { OBJECT_STORE_INSTANCE, STANDARD_FILE_SOURCE_TEMPLATE, STANDARD_OBJECT_STORE_TEMPLATE } from "./test_fixtures"; + +describe("formUtils", () => { + describe("createTemplateForm", () => { + it("should create a form from an object store templates", () => { + const form = createTemplateForm(STANDARD_OBJECT_STORE_TEMPLATE, "storage location"); + expect(form.length).toBe(6); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("_meta_name"); + expect(formEl0?.help).toEqual("Label this new storage location with a name."); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("_meta_description"); + }); + + it("should create a form from a file source templates", () => { + const form = createTemplateForm(STANDARD_FILE_SOURCE_TEMPLATE, "file source"); + expect(form.length).toBe(6); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("_meta_name"); + expect(formEl0?.help).toEqual("Label this new file source with a name."); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("_meta_description"); + }); + }); + + describe("upgradeForm", () => { + it("should create a form from an object store templates", () => { + const form = upgradeForm(STANDARD_OBJECT_STORE_TEMPLATE, OBJECT_STORE_INSTANCE); + expect(form.length).toBe(3); + const formEl0 = form[0]; + expect(formEl0?.name).toEqual("oldvar"); + const formEl1 = form[1]; + expect(formEl1?.name).toEqual("newvar"); + }); + + it("should only ask for new secrets during upgrade", () => { + const form = upgradeForm(STANDARD_OBJECT_STORE_TEMPLATE, OBJECT_STORE_INSTANCE); + expect(form.length).toBe(3); + const formEl0 = form[2]; + expect(formEl0?.name).toEqual("newsecret"); + expect(formEl0?.type).toEqual("password"); + }); + }); +}); diff --git a/client/src/components/ConfigTemplates/formUtil.ts b/client/src/components/ConfigTemplates/formUtil.ts new file mode 100644 index 000000000000..47ad9fc89096 --- /dev/null +++ b/client/src/components/ConfigTemplates/formUtil.ts @@ -0,0 +1,158 @@ +import type { + Instance, + SecretData, + TemplateSecret, + TemplateSummary, + TemplateVariable, + VariableData, + VariableValueType, +} from "@/api/configTemplates"; +import { markup } from "@/components/ObjectStore/configurationMarkdown"; + +export function metadataFormEntryName(what: string) { + return { + name: "_meta_name", + label: "Name", + type: "text", + optional: false, + help: `Label this new ${what} with a name.`, + }; +} + +export function metadataFormEntryDescription(what: string) { + return { + name: "_meta_description", + label: "Description", + optional: true, + type: "textarea", + help: `Provide some notes to yourself about this ${what} - perhaps to remind you how it is configured, where it stores the data, etc..`, + }; +} + +export function templateVariableFormEntry(variable: TemplateVariable, variableValue: VariableValueType) { + return { + name: variable.name, + type: "text", + help: markup(variable.help || "", true), + value: variableValue, + }; +} + +export function templateSecretFormEntry(secret: TemplateSecret) { + return { + name: secret.name, + type: "password", + help: markup(secret.help || "", true), + value: "", + }; +} + +export function editTemplateForm(template: TemplateSummary, what: string, instance: Instance) { + const form = []; + const nameInput = metadataFormEntryName(what); + form.push({ value: instance.name ?? "", ...nameInput }); + + const descriptionInput = metadataFormEntryDescription(what); + form.push({ value: instance.description ?? "", ...descriptionInput }); + + const variables = template.variables ?? []; + const variableValues: VariableData = instance.variables || {}; + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, variableValues[variable.name])); + } + return form; +} + +export function editFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const name = formData["_meta_name"]; + const description = formData["_meta_description"]; + const variableData: VariableData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + const payload = { + name: name, + description: description, + variables: variableData, + }; + return payload; +} + +export function createTemplateForm(template: TemplateSummary, what: string) { + const form = []; + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + form.push(metadataFormEntryName(what)); + form.push(metadataFormEntryDescription(what)); + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, undefined)); + } + for (const secret of secrets) { + form.push(templateSecretFormEntry(secret)); + } + return form; +} + +export function createFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + const variableData: VariableData = {}; + const secretData: SecretData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + for (const secret of secrets) { + secretData[secret.name] = formData[secret.name]; + } + const name: string = formData._meta_name; + const description: string = formData._meta_description; + const payload = { + name: name, + description: description, + secrets: secretData, + variables: variableData, + template_id: template.id, + template_version: template.version ?? 0, + }; + return payload; +} + +export function upgradeForm(template: TemplateSummary, instance: Instance): Array { + const form = []; + const variables = template.variables ?? []; + const secrets = template.secrets ?? []; + const variableValues: VariableData = instance.variables || {}; + const secretsSet = instance.secrets || []; + for (const variable of variables) { + form.push(templateVariableFormEntry(variable, variableValues[variable.name])); + } + for (const secret of secrets) { + const secretName = secret.name; + if (secretsSet.indexOf(secretName) >= 0) { + console.log("skipping..."); + } else { + form.push(templateSecretFormEntry(secret)); + } + } + return form; +} + +export function upgradeFormDataToPayload(template: TemplateSummary, formData: any) { + const variables = template.variables ?? []; + const variableData: VariableData = {}; + for (const variable of variables) { + variableData[variable.name] = formData[variable.name]; + } + const secrets = {}; + // ideally we would be able to force a template version here, + // maybe rework backend types to force this in the API response + // even if we don't need it in the config files + const templateVersion: number = template.version || 0; + const payload = { + template_version: templateVersion, + variables: variableData, + secrets: secrets, + }; + return payload; +} diff --git a/client/src/components/ObjectStore/Instances/icons.ts b/client/src/components/ConfigTemplates/icons.ts similarity index 100% rename from client/src/components/ObjectStore/Instances/icons.ts rename to client/src/components/ConfigTemplates/icons.ts diff --git a/client/src/components/ConfigTemplates/test_fixtures.ts b/client/src/components/ConfigTemplates/test_fixtures.ts new file mode 100644 index 000000000000..bbf4530a762f --- /dev/null +++ b/client/src/components/ConfigTemplates/test_fixtures.ts @@ -0,0 +1,82 @@ +import type { FileSourceTemplateSummary } from "@/api/fileSources"; +import type { UserConcreteObjectStore } from "@/components/ObjectStore/Instances/types"; +import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templates/types"; + +export const STANDARD_OBJECT_STORE_TEMPLATE: ObjectStoreTemplateSummary = { + type: "s3", + name: "moo", + description: null, + variables: [ + { + name: "oldvar", + type: "string", + help: "old var help", + }, + { + name: "newvar", + type: "string", + help: "new var help", + }, + ], + secrets: [ + { + name: "oldsecret", + help: "old secret help", + }, + { + name: "newsecret", + help: "new secret help", + }, + ], + id: "moo", + version: 2, + badges: [], +}; + +export const STANDARD_FILE_SOURCE_TEMPLATE: FileSourceTemplateSummary = { + type: "s3fs", + name: "moo", + description: null, + variables: [ + { + name: "oldvar", + type: "string", + help: "old var help", + }, + { + name: "newvar", + type: "string", + help: "new var help", + }, + ], + secrets: [ + { + name: "oldsecret", + help: "old secret help", + }, + { + name: "newsecret", + help: "new secret help", + }, + ], + id: "moo", + version: 2, +}; + +export const OBJECT_STORE_INSTANCE: UserConcreteObjectStore = { + type: "s3", + name: "moo", + description: undefined, + template_id: "moo", + template_version: 1, + badges: [], + variables: { + oldvar: "my old value", + droppedvar: "this will be dropped", + }, + secrets: ["oldsecret", "droppedsecret"], + quota: { enabled: false }, + private: false, + id: 4, + uuid: "112f889f-72d7-4619-a8e8-510a8c685aa7", +}; diff --git a/client/src/components/FileSources/FileSourceTypeSpan.vue b/client/src/components/FileSources/FileSourceTypeSpan.vue new file mode 100644 index 000000000000..538b635c4a42 --- /dev/null +++ b/client/src/components/FileSources/FileSourceTypeSpan.vue @@ -0,0 +1,23 @@ + + + + {{ type }} + + + diff --git a/client/src/components/FileSources/Instances/CreateForm.vue b/client/src/components/FileSources/Instances/CreateForm.vue new file mode 100644 index 000000000000..8079cf36735d --- /dev/null +++ b/client/src/components/FileSources/Instances/CreateForm.vue @@ -0,0 +1,53 @@ + + + + + {{ error }} + + + + diff --git a/client/src/components/FileSources/Instances/CreateInstance.vue b/client/src/components/FileSources/Instances/CreateInstance.vue new file mode 100644 index 000000000000..9b93e2d7c269 --- /dev/null +++ b/client/src/components/FileSources/Instances/CreateInstance.vue @@ -0,0 +1,34 @@ + + + + + + + + diff --git a/client/src/components/FileSources/Instances/EditInstance.vue b/client/src/components/FileSources/Instances/EditInstance.vue new file mode 100644 index 000000000000..311bb7fdc838 --- /dev/null +++ b/client/src/components/FileSources/Instances/EditInstance.vue @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + diff --git a/client/src/components/FileSources/Instances/EditSecrets.vue b/client/src/components/FileSources/Instances/EditSecrets.vue new file mode 100644 index 000000000000..dd14126a1242 --- /dev/null +++ b/client/src/components/FileSources/Instances/EditSecrets.vue @@ -0,0 +1,28 @@ + + + + diff --git a/client/src/components/FileSources/Instances/InstanceDropdown.vue b/client/src/components/FileSources/Instances/InstanceDropdown.vue new file mode 100644 index 000000000000..f71e3fd23022 --- /dev/null +++ b/client/src/components/FileSources/Instances/InstanceDropdown.vue @@ -0,0 +1,30 @@ + + + + + diff --git a/client/src/components/FileSources/Instances/ManageIndex.vue b/client/src/components/FileSources/Instances/ManageIndex.vue new file mode 100644 index 000000000000..3ad9601cbaaa --- /dev/null +++ b/client/src/components/FileSources/Instances/ManageIndex.vue @@ -0,0 +1,68 @@ + + + + + + + + + + + + No file source instances found for your users, click the create button to configure a new one. + + + + + + + + + + + + + + + diff --git a/client/src/components/FileSources/Instances/UpgradeForm.vue b/client/src/components/FileSources/Instances/UpgradeForm.vue new file mode 100644 index 000000000000..4ca85e7caf67 --- /dev/null +++ b/client/src/components/FileSources/Instances/UpgradeForm.vue @@ -0,0 +1,63 @@ + + + + + {{ error }} + + + + diff --git a/client/src/components/FileSources/Instances/UpgradeInstance.vue b/client/src/components/FileSources/Instances/UpgradeInstance.vue new file mode 100644 index 000000000000..a2aa1ed672f4 --- /dev/null +++ b/client/src/components/FileSources/Instances/UpgradeInstance.vue @@ -0,0 +1,27 @@ + + + + + diff --git a/client/src/components/FileSources/Instances/instance.ts b/client/src/components/FileSources/Instances/instance.ts new file mode 100644 index 000000000000..f45fd6bf76dd --- /dev/null +++ b/client/src/components/FileSources/Instances/instance.ts @@ -0,0 +1,23 @@ +import { computed, type Ref } from "vue"; + +import type { FileSourceTemplateSummary, UserFileSourceModel } from "@/api/fileSources"; +import { useFileSourceInstancesStore } from "@/stores/fileSourceInstancesStore"; +import { useFileSourceTemplatesStore } from "@/stores/fileSourceTemplatesStore"; + +export function useInstanceAndTemplate(instanceIdRef: Ref) { + const fileSourceTemplatesStore = useFileSourceTemplatesStore(); + const fileSourceInstancesStore = useFileSourceInstancesStore(); + fileSourceInstancesStore.fetchInstances(); + fileSourceTemplatesStore.fetchTemplates(); + + const instance = computed( + () => fileSourceInstancesStore.getInstance(instanceIdRef.value) || null + ); + const template = computed(() => + instance.value + ? fileSourceTemplatesStore.getTemplate(instance.value?.template_id, instance.value?.template_version) + : null + ); + + return { instance, template }; +} diff --git a/client/src/components/FileSources/Instances/routing.ts b/client/src/components/FileSources/Instances/routing.ts new file mode 100644 index 000000000000..6d2ed12bae6e --- /dev/null +++ b/client/src/components/FileSources/Instances/routing.ts @@ -0,0 +1,16 @@ +import { useRouter } from "vue-router/composables"; + +export function useInstanceRouting() { + const router = useRouter(); + + async function goToIndex(query: Record<"message", string>) { + router.push({ + path: "/file_source_instances/index", + query: query, + }); + } + + return { + goToIndex, + }; +} diff --git a/client/src/components/FileSources/Instances/services.ts b/client/src/components/FileSources/Instances/services.ts new file mode 100644 index 000000000000..d5d4b8553826 --- /dev/null +++ b/client/src/components/FileSources/Instances/services.ts @@ -0,0 +1,4 @@ +import { fetcher } from "@/api/schema/fetcher"; + +export const create = fetcher.path("/api/file_source_instances").method("post").create(); +export const update = fetcher.path("/api/file_source_instances/{user_file_source_id}").method("put").create(); diff --git a/client/src/components/FileSources/Templates/CreateUserFileSource.vue b/client/src/components/FileSources/Templates/CreateUserFileSource.vue new file mode 100644 index 000000000000..5cc4a9f8e3aa --- /dev/null +++ b/client/src/components/FileSources/Templates/CreateUserFileSource.vue @@ -0,0 +1,30 @@ + + + + + + diff --git a/client/src/components/FileSources/Templates/SelectTemplate.vue b/client/src/components/FileSources/Templates/SelectTemplate.vue new file mode 100644 index 000000000000..6e45ab9f8d3d --- /dev/null +++ b/client/src/components/FileSources/Templates/SelectTemplate.vue @@ -0,0 +1,39 @@ + + + + + + + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummary.vue b/client/src/components/FileSources/Templates/TemplateSummary.vue new file mode 100644 index 000000000000..6cef09f37ed1 --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummary.vue @@ -0,0 +1,22 @@ + + + + + This template produces file sources of type . + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue b/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue new file mode 100644 index 000000000000..c056e03d9b72 --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummaryPopover.vue @@ -0,0 +1,19 @@ + + + + + + + diff --git a/client/src/components/FileSources/Templates/TemplateSummarySpan.vue b/client/src/components/FileSources/Templates/TemplateSummarySpan.vue new file mode 100644 index 000000000000..1c4291dc3aad --- /dev/null +++ b/client/src/components/FileSources/Templates/TemplateSummarySpan.vue @@ -0,0 +1,32 @@ + + + + + + + {{ template.name }} + + + + Loading template information for {{ templateId }} {{ templateVersion }} + + diff --git a/client/src/components/FileSources/style.css b/client/src/components/FileSources/style.css new file mode 100644 index 000000000000..c2962e44de2e --- /dev/null +++ b/client/src/components/FileSources/style.css @@ -0,0 +1,4 @@ +.file-source-help-on-hover { + text-decoration-line: underline; + text-decoration-style: dashed; +} diff --git a/client/src/components/ObjectStore/Instances/CreateForm.vue b/client/src/components/ObjectStore/Instances/CreateForm.vue index e82f0f1c484a..cfb982a8697c 100644 --- a/client/src/components/ObjectStore/Instances/CreateForm.vue +++ b/client/src/components/ObjectStore/Instances/CreateForm.vue @@ -1,18 +1,15 @@ - + {{ error }} - - + + diff --git a/client/src/components/ObjectStore/Instances/EditInstance.vue b/client/src/components/ObjectStore/Instances/EditInstance.vue index 16e4dd9c4631..2569be940178 100644 --- a/client/src/components/ObjectStore/Instances/EditInstance.vue +++ b/client/src/components/ObjectStore/Instances/EditInstance.vue @@ -1,19 +1,16 @@ - - - - - + + + + + - - - + + + diff --git a/client/src/components/ObjectStore/Instances/EditSecrets.vue b/client/src/components/ObjectStore/Instances/EditSecrets.vue index 42038241d42d..56c1e7cd9641 100644 --- a/client/src/components/ObjectStore/Instances/EditSecrets.vue +++ b/client/src/components/ObjectStore/Instances/EditSecrets.vue @@ -6,8 +6,7 @@ import type { ObjectStoreTemplateSummary } from "@/components/ObjectStore/Templa import { update } from "./services"; import type { UserConcreteObjectStore } from "./types"; -import VaultSecret from "./VaultSecret.vue"; -import FormCard from "@/components/Form/FormCard.vue"; +import EditSecretsForm from "@/components/ConfigTemplates/EditSecretsForm.vue"; interface Props { objectStore: UserConcreteObjectStore; @@ -26,14 +25,5 @@ async function onUpdate(secretName: string, secretValue: string) { } - - - - - - - - - - + diff --git a/client/src/components/ObjectStore/Instances/InstanceDropdown.vue b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue index aaa2836c8289..475d802d391b 100644 --- a/client/src/components/ObjectStore/Instances/InstanceDropdown.vue +++ b/client/src/components/ObjectStore/Instances/InstanceDropdown.vue @@ -1,19 +1,13 @@ - - - - {{ props.objectStore.name }} - - - - - Upgrade - - - - Edit configuration - - - + diff --git a/client/src/components/ObjectStore/Instances/ManageIndex.vue b/client/src/components/ObjectStore/Instances/ManageIndex.vue index 9695f492dad2..1f87d85ebc09 100644 --- a/client/src/components/ObjectStore/Instances/ManageIndex.vue +++ b/client/src/components/ObjectStore/Instances/ManageIndex.vue @@ -1,20 +1,20 @@ - + {{ error }} - - + + diff --git a/client/src/components/ObjectStore/Instances/types.ts b/client/src/components/ObjectStore/Instances/types.ts index bd956468c7b8..0176eaec757a 100644 --- a/client/src/components/ObjectStore/Instances/types.ts +++ b/client/src/components/ObjectStore/Instances/types.ts @@ -2,8 +2,3 @@ import type { components } from "@/api/schema/schema"; export type UserConcreteObjectStore = components["schemas"]["UserConcreteObjectStoreModel"]; export type CreateInstancePayload = components["schemas"]["CreateInstancePayload"]; -export type ObjectStoreTemplateVariable = components["schemas"]["ObjectStoreTemplateVariable"]; -export type ObjectStoreTemplateSecret = components["schemas"]["ObjectStoreTemplateSecret"]; -export type VariableValueType = (string | boolean | number) | undefined; -export type VariableData = { [key: string]: VariableValueType }; -export type SecretData = { [key: string]: string }; diff --git a/client/src/components/ObjectStore/Instances/util.ts b/client/src/components/ObjectStore/Instances/util.ts deleted file mode 100644 index 77b99b11d73f..000000000000 --- a/client/src/components/ObjectStore/Instances/util.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { markup } from "@/components/ObjectStore/configurationMarkdown"; - -import type { ObjectStoreTemplateSecret, ObjectStoreTemplateVariable, VariableValueType } from "./types"; - -export function metadataFormEntryName() { - return { - name: "_meta_name", - label: "Name", - type: "text", - optional: false, - help: "Label this new object store a name.", - }; -} - -export function metadataFormEntryDescription() { - return { - name: "_meta_description", - label: "Description", - optional: true, - type: "textarea", - help: "Provide some notes to yourself about this object store - perhaps to remind you how it is configured, where it stores the data, etc..", - }; -} - -export function templateVariableFormEntry(variable: ObjectStoreTemplateVariable, variableValue: VariableValueType) { - return { - name: variable.name, - type: "text", - help: markup(variable.help || "", true), - value: variableValue, - }; -} - -export function templateSecretFormEntry(secret: ObjectStoreTemplateSecret) { - return { - name: secret.name, - type: "password", - help: markup(secret.help || "", true), - value: "", - }; -} diff --git a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue index 7ed555be53dc..f783b34b4f82 100644 --- a/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue +++ b/client/src/components/ObjectStore/Templates/CreateUserObjectStore.vue @@ -1,11 +1,11 @@ - - - - - {{ error }} - - - - - + + + diff --git a/client/src/components/ObjectStore/Templates/SelectTemplate.vue b/client/src/components/ObjectStore/Templates/SelectTemplate.vue index 5e99fbb0807c..a8e8362eb83a 100644 --- a/client/src/components/ObjectStore/Templates/SelectTemplate.vue +++ b/client/src/components/ObjectStore/Templates/SelectTemplate.vue @@ -2,6 +2,7 @@ import type { ObjectStoreTemplateSummaries } from "./types"; import TemplateSummaryPopover from "./TemplateSummaryPopover.vue"; +import SelectTemplate from "@/components/ConfigTemplates/SelectTemplate.vue"; interface SelectTemplateProps { templates: ObjectStoreTemplateSummaries; @@ -23,26 +24,12 @@ async function handleSubmit(templateId: string) { - - - - {{ template.name }} - - - - - - {{ selectText }} - - - + + props.template.type); - This template produces object stores of type . + This template produces storage locations of type . diff --git a/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue b/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue index 868dca4ff83b..df9f9b5f05f1 100644 --- a/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue +++ b/client/src/components/ObjectStore/Templates/TemplateSummaryPopover.vue @@ -2,20 +2,18 @@ import type { ObjectStoreTemplateSummary } from "./types"; import TemplateSummary from "./TemplateSummary.vue"; +import TemplateSummaryPopover from "@/components/ConfigTemplates/TemplateSummaryPopover.vue"; interface Props { target: String; template: ObjectStoreTemplateSummary; } -const props = defineProps(); - -const popoverPlacement = "rightbottom"; +defineProps(); - - {{ props.template.name }} - - + + + diff --git a/client/src/components/User/UserPreferences.vue b/client/src/components/User/UserPreferences.vue index 6807bda7f47a..47197e0ad92e 100644 --- a/client/src/components/User/UserPreferences.vue +++ b/client/src/components/User/UserPreferences.vue @@ -93,13 +93,21 @@ :user-id="userId"> + { return { message: route.query["message"] }; }, }, { path: "object_store_instances/:instanceId/edit", - component: EditInstance, + component: EditObjectStoreInstance, props: true, }, { path: "object_store_instances/:instanceId/upgrade", - component: UpgradeInstance, + component: UpgradeObjectStoreInstance, props: true, }, { path: "object_store_templates/:templateId/new", - component: CreateInstance, + component: CreateObjectStoreInstance, + props: true, + }, + { + path: "file_source_instances/create", + component: CreateUserFileSource, + }, + { + path: "file_source_instances/index", + component: ManageFileSourceIndex, + props: (route) => { + return { message: route.query["message"] }; + }, + }, + { + path: "file_source_instances/:instanceId/edit", + component: EditFileSourceInstance, + props: true, + }, + { + path: "file_source_instances/:instanceId/upgrade", + component: UpgradeFileSourceInstance, + props: true, + }, + { + path: "file_source_templates/:templateId/new", + component: CreateFileSourceInstance, props: true, }, { diff --git a/client/src/stores/configTemplatesUtil.ts b/client/src/stores/configTemplatesUtil.ts new file mode 100644 index 000000000000..2baeab3acb58 --- /dev/null +++ b/client/src/stores/configTemplatesUtil.ts @@ -0,0 +1,56 @@ +import type { TemplateSummary } from "@/api/configTemplates"; + +export function findTemplate( + templates: T[], + templateId: string, + templateVersion: number +): T | null { + for (const template of templates) { + if (template.id == templateId && template.version == templateVersion) { + return template; + } + } + return null; +} + +export function getLatestVersionMap(templates: T[]): { [key: string]: number } { + const latestVersions: { [key: string]: number } = {}; + templates.forEach((i: T) => { + const templateId = i.id; + const templateVersion = i.version || 0; + if ((latestVersions[templateId] ?? -1) < templateVersion) { + latestVersions[templateId] = templateVersion; + } + }); + return latestVersions; +} + +export function canUpgrade( + templates: T[], + templateId: string, + templateVersion: number +): boolean { + let can = false; + templates.forEach((i: T) => { + if (i.id == templateId && i.version && i.version > templateVersion) { + can = true; + } + }); + return can; +} + +export function getLatestVersion(templates: T[], id: string): T | null { + let latestVersion = -1; + let latestTemplate = null as T | null; + templates.forEach((i: T) => { + const templateId = i.id; + if (templateId == id) { + const templateVersion = i.version || 0; + if (templateVersion > latestVersion) { + latestTemplate = i; + latestVersion = templateVersion; + } + } + }); + return latestTemplate; +} diff --git a/client/src/stores/fileSourceInstancesStore.ts b/client/src/stores/fileSourceInstancesStore.ts new file mode 100644 index 000000000000..5b2bcf97b821 --- /dev/null +++ b/client/src/stores/fileSourceInstancesStore.ts @@ -0,0 +1,51 @@ +import { defineStore } from "pinia"; + +import { fetcher } from "@/api/schema/fetcher"; +import type { components } from "@/api/schema/schema"; +import { errorMessageAsString } from "@/utils/simple-error"; + +const getFileSourceInstances = fetcher.path("/api/file_source_instances").method("get").create(); + +type UserFileSourceModel = components["schemas"]["UserFileSourceModel"]; + +export const useFileSourceInstancesStore = defineStore("fileSourceInstances", { + state: () => ({ + instances: [] as UserFileSourceModel[], + fetched: false, + error: null as string | null, + }), + getters: { + getInstances: (state) => { + return state.instances; + }, + loading: (state) => { + return !state.fetched; + }, + getInstance: (state) => { + return (id: number | string) => state.instances.find((i) => i.id.toString() == id.toString()); + }, + }, + actions: { + async handleInit(instances: UserFileSourceModel[]) { + this.instances = instances; + this.fetched = true; + this.error = null; + }, + async handleError(err: unknown) { + this.error = errorMessageAsString(err); + }, + async fetchInstances() { + try { + const { data: instances } = await getFileSourceInstances({}); + this.handleInit(instances); + } catch (err) { + this.handleError(err); + } + }, + async ensureTemplates() { + if (!this.fetched || this.error != null) { + await this.fetchInstances(); + } + }, + }, +}); diff --git a/client/src/stores/fileSourceTemplatesStore.ts b/client/src/stores/fileSourceTemplatesStore.ts new file mode 100644 index 000000000000..1efd7e67bfe0 --- /dev/null +++ b/client/src/stores/fileSourceTemplatesStore.ts @@ -0,0 +1,70 @@ +import { defineStore } from "pinia"; + +import { fetcher } from "@/api/schema/fetcher"; +import type { components } from "@/api/schema/schema"; +import { errorMessageAsString } from "@/utils/simple-error"; + +import { canUpgrade, findTemplate, getLatestVersion, getLatestVersionMap } from "./configTemplatesUtil"; + +const getFileSourceTemplates = fetcher.path("/api/file_source_templates").method("get").create(); + +type FileSourceTemplateSummary = components["schemas"]["FileSourceTemplateSummary"]; +type FileSourceTemplateSummaries = FileSourceTemplateSummary[]; + +export const useFileSourceTemplatesStore = defineStore("fileSourceTemplatesStore", { + state: () => ({ + templates: [] as FileSourceTemplateSummaries, + fetched: false, + error: null as string | null, + }), + getters: { + latestTemplates: (state) => { + // only expose latest instance by template_version for each template_id + const latestVersions = getLatestVersionMap(state.templates); + return state.templates.filter((i: FileSourceTemplateSummary) => latestVersions[i.id] == (i.version || 0)); + }, + canUpgrade: (state) => { + return (templateId: string, templateVersion: number) => + canUpgrade(state.templates, templateId, templateVersion); + }, + getTemplates: (state) => { + return state.templates; + }, + getTemplate: (state) => { + return (templateId: string, templateVersion: number) => + findTemplate(state.templates, templateId, templateVersion); + }, + getLatestTemplate: (state) => { + return (templateId: string) => getLatestVersion(state.templates, templateId); + }, + hasTemplates: (state) => { + return state.templates.length > 0; + }, + loading: (state) => { + return !state.fetched; + }, + }, + actions: { + async handleInit(templates: FileSourceTemplateSummaries) { + this.templates = templates; + this.fetched = true; + }, + async handleError(err: unknown) { + this.fetched = true; + this.error = errorMessageAsString(err); + }, + async fetchTemplates() { + try { + const { data: templates } = await getFileSourceTemplates({}); + this.handleInit(templates); + } catch (err) { + this.handleError(err); + } + }, + async ensureTemplates() { + if (!this.fetched || this.error != null) { + await this.fetchTemplates(); + } + }, + }, +}); diff --git a/client/src/stores/objectStoreTemplatesStore.ts b/client/src/stores/objectStoreTemplatesStore.ts index c7c340a50f08..4b9c852d494f 100644 --- a/client/src/stores/objectStoreTemplatesStore.ts +++ b/client/src/stores/objectStoreTemplatesStore.ts @@ -4,58 +4,13 @@ import { fetcher } from "@/api/schema/fetcher"; import type { components } from "@/api/schema/schema"; import { errorMessageAsString } from "@/utils/simple-error"; +import { canUpgrade, findTemplate, getLatestVersion, getLatestVersionMap } from "./configTemplatesUtil"; + const getObjectStoreTemplates = fetcher.path("/api/object_store_templates").method("get").create(); type ObjectStoreTemplateSummary = components["schemas"]["ObjectStoreTemplateSummary"]; type ObjectStoreTemplateSummaries = ObjectStoreTemplateSummary[]; -function findTemplate(templates: ObjectStoreTemplateSummaries, templateId: string, templateVersion: number) { - for (const template of templates) { - if (template.id == templateId && template.version == templateVersion) { - return template; - } - } - return null; -} - -function getLatestVersionMap(templates: ObjectStoreTemplateSummaries): { [key: string]: number } { - const latestVersions: { [key: string]: number } = {}; - templates.forEach((i: ObjectStoreTemplateSummary) => { - const templateId = i.id; - const templateVersion = i.version || 0; - if ((latestVersions[templateId] ?? -1) < templateVersion) { - latestVersions[templateId] = templateVersion; - } - }); - return latestVersions; -} - -function canUpgrade(templates: ObjectStoreTemplateSummaries, templateId: string, templateVersion: number): boolean { - let can = false; - templates.forEach((i: ObjectStoreTemplateSummary) => { - if (i.id == templateId && i.version && i.version > templateVersion) { - can = true; - } - }); - return can; -} - -function getLatestVersion(templates: ObjectStoreTemplateSummaries, id: string): ObjectStoreTemplateSummary | null { - let latestVersion = -1; - let latestTemplate = null as ObjectStoreTemplateSummary | null; - templates.forEach((i: ObjectStoreTemplateSummary) => { - const templateId = i.id; - if (templateId == id) { - const templateVersion = i.version || 0; - if (templateVersion > latestVersion) { - latestTemplate = i; - latestVersion = templateVersion; - } - } - }); - return latestTemplate; -} - export const useObjectStoreTemplatesStore = defineStore("objectStoreTemplatesStore", { state: () => ({ templates: [] as ObjectStoreTemplateSummaries, diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml index c314661963a9..73ea2fe2006f 100644 --- a/client/src/utils/navigation/navigation.yml +++ b/client/src/utils/navigation/navigation.yml @@ -95,6 +95,7 @@ preferences: username_input: "input[id='username']" preferred_storage: '.preferred-storage' manage_object_stores: '.manage-object-stores' + manage_file_sources: '.manage-file-sources' object_store_selection: selectors: diff --git a/doc/source/admin/data.md b/doc/source/admin/data.md new file mode 100644 index 000000000000..a4d621ccbd86 --- /dev/null +++ b/doc/source/admin/data.md @@ -0,0 +1,148 @@ +# Connecting Users and Data + +Galaxy has countless ways for users to connect with things that might be considered their "data" - +file sources (aka "remote files"), object stores (aka "storage locations"), data libraries, the +upload API, visualizations, display applications, custom tools, etc... + +This document is going to discuss two of these (file sources and object stores) that are most +important Galaxy administrators and how to build Galaxy configuration that allow administrators +to let users tie into various pieces of infrastructure. + +## Datasets vs Files + +File sources in Galaxy are a sprawling concept but essentially they provide users access to simple +files (stored hierarchically into folders) that can be navigated and imported into Galaxy. +Importing a "file" into Galaxy generally +creates a copy of that file into a Galaxy "object store". Once these files are stored in Galaxy, +they become "datasets". +A Galaxy dataset is much more than a simple file - Galaxy datasets include various generic metadata +a datatype, datatype specific metadata, and ownership and sharing rules managed by Galaxy. + +Galaxy object stores (called storage locations in the UI) store datasets and global (accessible to +all users) object stores +are configured with the ``galaxy.yml`` property ``object_store_config_file`` (or +``object_store_config`` for a configuration embedded right in ``galaxy.yml``) that defaults to +``object_store_conf.xml`` or ``object_store_conf.yml`` if either is present in Galaxy's configuration +directory. Galaxy file sources provide users access to raw files and global files sources are configured with the ``galaxy.yml`` property ``file_sources_config_file`` (or ``file_sources`` for embedded configurations) that defaults to ``file_sources_conf.yml`` if that file is present in Galaxy's configuration directory. + +Some of Galaxy's most updated and complete administrator documentation can be found in configuration +sample files - this is definitely the case for object stores and file sources. The relevant sample +configuration files include +[file_sources_conf.yml.sample](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/config/sample/file_sources_conf.yml.sample) and +[object_store_conf.sample.yml](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/config/sample/object_store_conf.sample.yml). + +File sources and object stores configured with the above files essentially are available to all users +of your Galaxy instance - hence this document describes them as "global" file sources and object +stores. File source configurations do allow some templating that does allow the a global file source +to be materialized differently for different users. For instance, you as an admin may setup a Dropbox +file source and may explicitly add custom user properties that allow that single Dropbox file source +to read from a user's preferences. Since there is just one Dropbox service and most people only have +a single Dropbox account, this use case can be somewhat adequately addressed by the global file source +and the global user preferences file. For a use case like Amazon S3 buckets though for instance, +a single bucket file source that is parameterized one way is probably more clearly inadequate. For +instance, users would very likely want to attach different buckets for different projects. +Additionally, the Galaxy user interface doesn't tie the user preferences to the particular file +source and so this method introduces a huge education burden on your Galaxy instance. Finally, +the templating available to file sources are not available for object stores - and allowing users +to describe how they would like datasets stored and to pay for their own dataset storage are important +use cases. + +This document is going to describe Galaxy configuration template libraries that allow the +administrator to setup templates for file sources and object stores that your users may instantiate +as they see fit. User's can instantiate multiple instances of any template, the template concept +can apply to both file source and object store plugins, and the user interface is unified from the +template configuration file (you as the admin do not need to explicitly declare user preferences and +your users do not need to navigate seemingly unrelated preferences to get plugins to work). + +## Object Store Templates + +Galaxy's object store templates are configured as a YAML list of template objects. This list +can be placed ``object_store_templates.yml`` in Galaxy configuration directory (or any path +pointed to by the configuration option ``object_store_templates_config_file`` in ``galaxy.yml``). +Alternatively, the configuration can be placed directly into ``galaxy.yml`` using the +``object_store_templates`` configuration option. + +A minimal object store template might look something like: + +```yaml +- id: project_scratch + name: Project Scratch + version: 0 + description: Folder on institutional scratch disk area bound to your user. + variables: + project_name: + type: string + help: The name of your project scratch. + configuration: + type: disk + files_dir: '/scratch/for_galaxy/{{ user.username }}/{{ variables.project_name }}' + badges: + - type: faster + - type: less_secure + - type: not_backed_up +``` + +### YAML Syntax + +![galaxy.objectstore.templates.models](object_store_templates.png) + +## File Source Templates + +Galaxy's file source templates are configured as a YAML list of template objects. This list +can be placed ``file_source_templates.yml`` in Galaxy configuration directory (or any path +pointed to by the configuration option ``file_source_templates_config_file`` in ``galaxy.yml``). +Alternatively, the configuration can be placed directly into ``galaxy.yml`` using the +``file_source_templates`` configuration option. + +![galaxy.files.templates.models](file_source_templates.png) + +## Playing Nicer with Ansible + +Many large instances of Galaxy are configured with Ansible and much of the existing administrator +documentation leverages Ansible. The configuration template files using Jinja templating and so +does Ansible by default. This might result in a lack of clarity of when templates (strings +starting with ``{{`` and ending with ``}}``) are being evaluated. Ansible templates are evaluated +at deploy time and the configuration objects describing plugins are evaluated at Galaxy runtime. + +The easiest way to fix this is probably to store these templates files in your Ansible as plain files +and not templates. If you'd like to use Ansible templating to build up these files you'll very +likely need to tell either Galaxy or Ansible to use something other than ``{{`` and ``}}`` for +templating variables. This can be done by placing a directive at the top of your template that +is consumed by Ansible. For instance, to have ``[%`` and ``%]`` used instead of ``{{`` and ``}}`` +by Ansible at deploy time, the file could start with: + +``` +#jinja2:variable_start_string:'[%' , variable_end_string:'%]' +``` + +In this case, variables wrapped by ``[%`` and ``%]`` are expanded by Ansible and use the Ansible +environment and ``{`` and ``}`` are reserved for Galaxy templating. + +Alternatively, Galaxy can be configured to use a custom template on a per-configuration +object basis by setting the ``template_start`` and/or ``template_end`` variables. + +The following template chunk shows how to override the templating Galaxy does for a +particular object store configuration. Similar templating overrides work for file source +plugin templates. + +```yaml +- id: path_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/@= user.username | ensure_path_component =@//@= variables.project_name | ensure_path_component =@' + template_start: '@=' + template_end: '=@' + variables: + project_name: + type: path_component + help: Project name used in path for this template library. +``` + +- https://github.com/ansible/ansible/pull/75306 +- https://stackoverflow.com/questions/12083319/add-custom-tokens-in-jinja2-e-g-somevar + +## Connecting Configuration Templates to Secrets + +## Versioning Templates diff --git a/doc/source/admin/file_source_templates.png b/doc/source/admin/file_source_templates.png new file mode 100644 index 000000000000..920837cf3521 Binary files /dev/null and b/doc/source/admin/file_source_templates.png differ diff --git a/doc/source/admin/gen_diagrams.py b/doc/source/admin/gen_diagrams.py new file mode 100644 index 000000000000..06c96c6f36da --- /dev/null +++ b/doc/source/admin/gen_diagrams.py @@ -0,0 +1,25 @@ +import os +import sys + +import erdantic as erd + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "lib"))) + +from galaxy.files.templates.models import FileSourceTemplate +from galaxy.objectstore.templates.models import ( + ObjectStoreTemplate, + S3ObjectStoreConfiguration, + S3ObjectStoreTemplateConfiguration, +) + +DOC_SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__))) + +class_to_diagram = { + ObjectStoreTemplate: "object_store_templates", + S3ObjectStoreTemplateConfiguration: "object_store_s3_configuration_template", + S3ObjectStoreConfiguration: "object_store_s3_configuration", + FileSourceTemplate: "file_source_templates", +} + +for clazz, diagram_name in class_to_diagram.items(): + erd.draw(clazz, out=f"{DOC_SOURCE_DIR}/{diagram_name}.png") diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 61ed7db63ba4..f7853a895a0b 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -11,6 +11,7 @@ This documentation is in the midst of being ported and unified based on resource config config_logging production + data security nginx apache diff --git a/doc/source/admin/object_store_s3_configuration.png b/doc/source/admin/object_store_s3_configuration.png new file mode 100644 index 000000000000..cead1584ddbf Binary files /dev/null and b/doc/source/admin/object_store_s3_configuration.png differ diff --git a/doc/source/admin/object_store_s3_configuration_template.png b/doc/source/admin/object_store_s3_configuration_template.png new file mode 100644 index 000000000000..9a104c85df55 Binary files /dev/null and b/doc/source/admin/object_store_s3_configuration_template.png differ diff --git a/doc/source/admin/object_store_templates.png b/doc/source/admin/object_store_templates.png new file mode 100644 index 000000000000..2af89cb0ab48 Binary files /dev/null and b/doc/source/admin/object_store_templates.png differ diff --git a/doc/source/admin/s3_configuration.png b/doc/source/admin/s3_configuration.png new file mode 100644 index 000000000000..cead1584ddbf Binary files /dev/null and b/doc/source/admin/s3_configuration.png differ diff --git a/doc/source/admin/s3_configuration_template.png b/doc/source/admin/s3_configuration_template.png new file mode 100644 index 000000000000..9a104c85df55 Binary files /dev/null and b/doc/source/admin/s3_configuration_template.png differ diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 682b85fc751b..782a6ef868f3 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -35,13 +35,27 @@ ) from galaxy.config_watchers import ConfigWatchers from galaxy.datatypes.registry import Registry -from galaxy.files import ConfiguredFileSources +from galaxy.files import ( + ConfiguredFileSources, + ConfiguredFileSourcesConf, + UserDefinedFileSources, +) +from galaxy.files.plugins import ( + FileSourcePluginLoader, + FileSourcePluginsConfig, +) +from galaxy.files.templates import ConfiguredFileSourceTemplates from galaxy.job_metrics import JobMetrics from galaxy.jobs.manager import JobManager from galaxy.managers.api_keys import ApiKeyManager from galaxy.managers.citations import CitationsManager from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.dbkeys import GenomeBuilds +from galaxy.managers.file_source_instances import ( + FileSourceInstancesManager, + UserDefinedFileSourcesConfig, + UserDefinedFileSourcesImpl, +) from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager @@ -585,9 +599,28 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl ) # ConfiguredFileSources - self.file_sources = self._register_singleton( - ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config) + vault_configured = is_vault_configured(self.vault) + file_sources_config: FileSourcePluginsConfig = FileSourcePluginsConfig.from_app_config(self.config) + self._register_singleton(FileSourcePluginsConfig, file_sources_config) + file_source_plugin_loader = FileSourcePluginLoader() + self._register_singleton(FileSourcePluginLoader, file_source_plugin_loader) + self._register_singleton( + UserDefinedFileSourcesConfig, UserDefinedFileSourcesConfig.from_app_config(self.config) + ) + self._register_abstract_singleton( + UserDefinedFileSources, UserDefinedFileSourcesImpl # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717 + ) + templates = ConfiguredFileSourceTemplates.from_app_config(self.config, vault_configured=vault_configured) + self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates) + configured_file_source_conf: ConfiguredFileSourcesConf = ConfiguredFileSourcesConf.from_app_config(self.config) + file_sources = ConfiguredFileSources( + file_sources_config, + configured_file_source_conf, + load_stock_plugins=True, + plugin_loader=file_source_plugin_loader, ) + self.file_sources = self._register_singleton(ConfiguredFileSources, file_sources) + self._register_singleton(FileSourceInstancesManager) # Load security policy. self.security_agent = self.model.security_agent diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index d50842158020..0930a3ebbe18 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -42,6 +42,11 @@ GalaxyDataTestConfig, ) from galaxy.security import idencoding +from galaxy.security.vault import ( + UserVaultWrapper, + Vault, + VaultFactory, +) from galaxy.short_term_storage import ( ShortTermStorageAllocator, ShortTermStorageConfiguration, @@ -171,6 +176,19 @@ def wait_for_toolbox_reload(self, toolbox): def reindex_tool_search(self) -> None: raise NotImplementedError + def setup_test_vault(self): + config = { + "encryption_keys": [ + "5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg=", + "iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0=", + "IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g=", + ], + } + vault = VaultFactory.from_vault_type(self, "database", config) + # Ignored because of https://github.com/python/mypy/issues/4717 + self[Vault] = vault # type: ignore[type-abstract] + self.vault = vault + class MockLock: def __enter__(self): @@ -352,6 +370,11 @@ def fill_template(self, filename, template_lookup=None, **kwargs): kwargs.update(h=MockTemplateHelpers()) return template.render(**kwargs) + @property + def user_vault(self): + """Provide access to a user's personal vault.""" + return UserVaultWrapper(self.app.vault, self.user) + class MockVisualizationsRegistry: BUILT_IN_VISUALIZATIONS = ["trackster"] diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index cdb8b47151de..5c147c02a9c8 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -566,6 +566,21 @@ mapping: desc: | Configured Object Store templates embedded into Galaxy's config. + file_source_templates_config_file: + type: str + default: file_source_templates.yml + path_resolves_to: config_dir + required: false + desc: | + Configured user file source templates configuration file. + + file_source_templates: + type: seq + sequence: + - type: any + desc: | + Configured user file source templates embedded into Galaxy's config. + user_object_store_index_by: type: str default: 'uuid' diff --git a/lib/galaxy/files/__init__.py b/lib/galaxy/files/__init__.py index 6ddf924f4089..07b9de43d971 100644 --- a/lib/galaxy/files/__init__.py +++ b/lib/galaxy/files/__init__.py @@ -23,6 +23,7 @@ plugin_source_from_dict, plugin_source_from_path, PluginConfigSource, + PluginConfigsT, ) from .plugins import ( FileSourcePluginLoader, @@ -46,25 +47,107 @@ class NoMatchingFileSource(Exception): pass +class UserDefinedFileSources(Protocol): + """Entry-point for Galaxy to inject user-defined object stores. + + Supplied object of this class is used to write out concrete + description of file sources when serializing all file sources + available to a user. + """ + + def validate_uri_root(self, uri: str, user_context: "ProvidesFileSourcesUserContext") -> None: + pass + + def find_best_match(self, url: str) -> Optional[FileSourceScore]: + pass + + def user_file_sources_to_dicts( + self, + for_serialization: bool, + user_context: "FileSourceDictifiable", + browsable_only: Optional[bool] = False, + include_kind: Optional[Set[PluginKind]] = None, + exclude_kind: Optional[Set[PluginKind]] = None, + ) -> List[FilesSourceProperties]: + """Write out user file sources as list of config dictionaries.""" + # config_dicts: List[FilesSourceProperties] = [] + # for file_source in self.user_file_sources(): + # as_dict = file_source.to_dict(for_serialization=for_serialization, user_context=user_context) + # config_dicts.append(as_dict) + # return config_dicts + + +class NullUserDefinedFileSources(UserDefinedFileSources): + + def validate_uri_root(self, uri: str, user_context: "ProvidesFileSourcesUserContext") -> None: + return None + + def find_best_match(self, url: str) -> Optional[FileSourceScore]: + return None + + def user_file_sources_to_dicts( + self, + for_serialization: bool, + user_context: "FileSourceDictifiable", + browsable_only: Optional[bool] = False, + include_kind: Optional[Set[PluginKind]] = None, + exclude_kind: Optional[Set[PluginKind]] = None, + ) -> List[FilesSourceProperties]: + return [] + + +def _ensure_user_defined_file_sources( + user_defined_file_sources: Optional[UserDefinedFileSources] = None, +) -> UserDefinedFileSources: + if user_defined_file_sources is not None: + return user_defined_file_sources + else: + return NullUserDefinedFileSources() + + +class ConfiguredFileSourcesConf: + conf_dict: Optional[PluginConfigsT] + conf_file: Optional[str] + + def __init__(self, conf_dict: Optional[PluginConfigsT] = None, conf_file: Optional[str] = None): + self.conf_dict = conf_dict + self.conf_file = conf_file + + @staticmethod + def from_app_config(config): + config_file = config.file_sources_config_file + config_dict = None + if not config_file or not os.path.exists(config_file): + config_file = None + config_dict = config.file_sources + return ConfiguredFileSourcesConf(config_dict, config_file) + + class ConfiguredFileSources: """Load plugins and resolve Galaxy URIs to FileSource objects.""" _file_sources: List[BaseFilesSource] + _plugin_loader: FileSourcePluginLoader + _user_defined_file_sources: UserDefinedFileSources def __init__( self, file_sources_config: FileSourcePluginsConfig, - conf_file=None, - conf_dict=None, - load_stock_plugins=False, + configured_file_source_conf: Optional[ConfiguredFileSourcesConf] = None, + load_stock_plugins: bool = False, + plugin_loader: Optional[FileSourcePluginLoader] = None, + user_defined_file_sources: Optional[UserDefinedFileSources] = None, ): self._file_sources_config = file_sources_config - self._plugin_loader = FileSourcePluginLoader() + self._plugin_loader = plugin_loader or FileSourcePluginLoader() + self._user_defined_file_sources = _ensure_user_defined_file_sources(user_defined_file_sources) file_sources: List[BaseFilesSource] = [] - if conf_file is not None: - file_sources = self._load_plugins_from_file(conf_file) - elif conf_dict is not None: - plugin_source = plugin_source_from_dict(conf_dict) + if configured_file_source_conf is None: + configured_file_source_conf = ConfiguredFileSourcesConf(conf_dict=[]) + if configured_file_source_conf.conf_file is not None: + file_sources = self._load_plugins_from_file(configured_file_source_conf.conf_file) + elif configured_file_source_conf.conf_dict is not None: + plugin_source = plugin_source_from_dict(configured_file_source_conf.conf_dict) file_sources = self._parse_plugin_source(plugin_source) else: file_sources = [] @@ -154,6 +237,7 @@ def validate_uri_root(self, uri: str, user_context: "FileSourcesUserContext"): raise exceptions.ObjectNotFound( "Your FTP directory does not exist, attempting to upload files to it may cause it to be created." ) + self._user_defined_file_sources.validate_uri_root(uri, user_context) def looks_like_uri(self, path_or_uri): # is this string a URI this object understands how to realize @@ -183,6 +267,16 @@ def plugins_to_dict( continue el = file_source.to_dict(for_serialization=for_serialization, user_context=user_context) rval.append(el) + if user_context: + rval.extend( + self._user_defined_file_sources.user_file_sources_to_dicts( + for_serialization, + user_context, + browsable_only=browsable_only, + include_kind=include_kind, + exclude_kind=exclude_kind, + ) + ) return rval def to_dict(self, for_serialization: bool = False, user_context: "OptionalUserContext" = None) -> Dict[str, Any]: @@ -191,18 +285,6 @@ def to_dict(self, for_serialization: bool = False, user_context: "OptionalUserCo "config": self._file_sources_config.to_dict(), } - @staticmethod - def from_app_config(config): - config_file = config.file_sources_config_file - config_dict = None - if not config_file or not os.path.exists(config_file): - config_file = None - config_dict = config.file_sources - file_sources_config = FileSourcePluginsConfig.from_app_config(config) - return ConfiguredFileSources( - file_sources_config, conf_file=config_file, conf_dict=config_dict, load_stock_plugins=True - ) - @staticmethod def from_dict(as_dict, load_stock_plugins=False): if as_dict is not None: @@ -212,8 +294,9 @@ def from_dict(as_dict, load_stock_plugins=False): else: sources_as_dict = [] file_sources_config = FileSourcePluginsConfig() + configured_file_sources_conf = ConfiguredFileSourcesConf(conf_dict=sources_as_dict) return ConfiguredFileSources( - file_sources_config, conf_dict=sources_as_dict, load_stock_plugins=load_stock_plugins + file_sources_config, configured_file_sources_conf, load_stock_plugins=load_stock_plugins ) @@ -221,7 +304,7 @@ class NullConfiguredFileSources(ConfiguredFileSources): def __init__( self, ): - super().__init__(FileSourcePluginsConfig()) + super().__init__(FileSourcePluginsConfig(), ConfiguredFileSourcesConf(conf_dict=[])) class DictifiableFilesSourceContext(Protocol): @@ -288,7 +371,7 @@ def email(self): return user and user.email @property - def username(self): + def username(self) -> Optional[str]: user = self.trans.user return user and user.username @@ -344,7 +427,7 @@ def email(self): return self._kwd.get("email") @property - def username(self): + def username(self) -> Optional[str]: return self._kwd.get("username") @property diff --git a/lib/galaxy/files/plugins.py b/lib/galaxy/files/plugins.py index 27f5f2de96d2..4f9c182f846a 100644 --- a/lib/galaxy/files/plugins.py +++ b/lib/galaxy/files/plugins.py @@ -1,6 +1,8 @@ from typing import ( + cast, List, Optional, + Type, TYPE_CHECKING, ) @@ -86,6 +88,9 @@ def _file_source_plugins_dict(self): return plugins_dict(galaxy.files.sources, "plugin_type") + def get_plugin_type_class(self, plugin_type: str) -> Type["BaseFilesSource"]: + return cast(Type["BaseFilesSource"], self._plugin_classes[plugin_type]) + def load_plugins( self, plugin_source: PluginConfigSource, file_source_plugin_config: FileSourcePluginsConfig ) -> List["BaseFilesSource"]: diff --git a/lib/galaxy/files/sources/__init__.py b/lib/galaxy/files/sources/__init__.py index ae5a59299d78..fd63e9790c32 100644 --- a/lib/galaxy/files/sources/__init__.py +++ b/lib/galaxy/files/sources/__init__.py @@ -12,6 +12,7 @@ List, Optional, Set, + Type, TYPE_CHECKING, Union, ) @@ -297,18 +298,22 @@ def get_browsable(self) -> bool: """Return true if the filesource implements the SupportsBrowsing interface.""" +def file_source_type_is_browsable(target_type: Type["BaseFilesSource"]) -> bool: + # Check whether the list method has been overridden + return target_type.list != BaseFilesSource.list or target_type._list != BaseFilesSource._list + + class BaseFilesSource(FilesSource): plugin_kind: ClassVar[PluginKind] = PluginKind.rfs # Remote File Source by default, override in subclasses def get_browsable(self) -> bool: - # Check whether the list method has been overridden - return type(self).list != BaseFilesSource.list or type(self)._list != BaseFilesSource._list + return file_source_type_is_browsable(type(self)) def get_prefix(self) -> Optional[str]: return self.id def get_scheme(self) -> str: - return "gxfiles" + return self.scheme or "gxfiles" def get_writable(self) -> bool: return self.writable diff --git a/lib/galaxy/files/templates/__init__.py b/lib/galaxy/files/templates/__init__.py new file mode 100644 index 000000000000..aa8faf85fd30 --- /dev/null +++ b/lib/galaxy/files/templates/__init__.py @@ -0,0 +1,17 @@ +from .manager import ConfiguredFileSourceTemplates +from .models import ( + FileSourceConfiguration, + FileSourceTemplate, + FileSourceTemplateSummaries, + FileSourceTemplateType, + template_to_configuration, +) + +__all__ = ( + "ConfiguredFileSourceTemplates", + "FileSourceConfiguration", + "FileSourceTemplate", + "FileSourceTemplateSummaries", + "FileSourceTemplateType", + "template_to_configuration", +) diff --git a/lib/galaxy/files/templates/manager.py b/lib/galaxy/files/templates/manager.py new file mode 100644 index 000000000000..c757e6482014 --- /dev/null +++ b/lib/galaxy/files/templates/manager.py @@ -0,0 +1,86 @@ +import os +from typing import ( + List, + Optional, + Protocol, +) + +from yaml import safe_load + +from galaxy.util.config_templates import ( + apply_syntactic_sugar, + find_template, + find_template_by, + InstanceDefinition, + RawTemplateConfig, + TemplateReference, + validate_secrets_and_variables, + verify_vault_configured_if_uses_secrets, +) +from .models import ( + FileSourceTemplate, + FileSourceTemplateCatalog, + FileSourceTemplateSummaries, +) + +SECRETS_NEED_VAULT_MESSAGE = "The file source templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration." + + +class AppConfigProtocol(Protocol): + file_source_templates: Optional[List[RawTemplateConfig]] + file_source_templates_config_file: Optional[str] + + +class ConfiguredFileSourceTemplates: + catalog: FileSourceTemplateCatalog + + def __init__(self, catalog: FileSourceTemplateCatalog): + self.catalog = catalog + + @staticmethod + def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "ConfiguredFileSourceTemplates": + raw_config = config.file_source_templates + if raw_config is None: + config_file = config.file_source_templates_config_file + if config_file and os.path.exists(config_file): + with open(config_file) as f: + raw_config = safe_load(f) + if raw_config is None: + raw_config = [] + catalog = raw_config_to_catalog(raw_config) + verify_vault_configured_if_uses_secrets( + catalog, + vault_configured, + SECRETS_NEED_VAULT_MESSAGE, + ) + templates = ConfiguredFileSourceTemplates(catalog) + return templates + + @property + def summaries(self) -> FileSourceTemplateSummaries: + templates = self.catalog.root + summaries = [] + for template in templates: + template_dict = template.model_dump() + configuration = template_dict.pop("configuration") + object_store_type = configuration["type"] + template_dict["type"] = object_store_type + summaries.append(template_dict) + return FileSourceTemplateSummaries.model_validate(summaries) + + def find_template(self, instance_reference: TemplateReference) -> FileSourceTemplate: + """Find the corresponding template and throw ObjectNotFound if not available.""" + return find_template(self.catalog.root, instance_reference, "file source") + + def find_template_by(self, template_id: str, template_version: int) -> FileSourceTemplate: + return find_template_by(self.catalog.root, template_id, template_version, "file source") + + def validate(self, instance: InstanceDefinition): + template = self.find_template(instance) + validate_secrets_and_variables(instance, template) + # TODO: validate no extra variables + + +def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> FileSourceTemplateCatalog: + effective_root = apply_syntactic_sugar(raw_config) + return FileSourceTemplateCatalog.model_validate(effective_root) diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py new file mode 100644 index 000000000000..2f43230afd35 --- /dev/null +++ b/lib/galaxy/files/templates/models.py @@ -0,0 +1,129 @@ +from typing import ( + Any, + Dict, + List, + Literal, + Optional, + Type, + Union, +) + +from pydantic import RootModel + +from galaxy.util.config_templates import ( + expand_raw_config, + MarkdownContent, + StrictModel, + TemplateExpansion, + TemplateSecret, + TemplateVariable, + TemplateVariableValueType, +) + +FileSourceTemplateType = Literal["posix", "s3fs"] + + +class PosixFileSourceTemplateConfiguration(StrictModel): + type: Literal["posix"] + root: Union[str, TemplateExpansion] + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class PosixFileSourceConfiguration(StrictModel): + type: Literal["posix"] + root: str + + +class S3FSFileSourceTemplateConfiguration(StrictModel): + type: Literal["s3fs"] + endpoint_url: Optional[Union[str, TemplateExpansion]] = None + anon: Optional[Union[bool, TemplateExpansion]] = False + secret: Optional[Union[str, TemplateExpansion]] = None + key: Optional[Union[str, TemplateExpansion]] = None + bucket: Optional[Union[str, TemplateExpansion]] = None + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class S3FSFileSourceConfiguration(StrictModel): + type: Literal["s3fs"] + endpoint_url: Optional[str] = None + anon: Optional[bool] = False + secret: Optional[str] = None + key: Optional[str] = None + bucket: Optional[str] = None + + +FileSourceTemplateConfiguration = Union[ + PosixFileSourceTemplateConfiguration, + S3FSFileSourceTemplateConfiguration, +] +FileSourceConfiguration = Union[ + PosixFileSourceConfiguration, + S3FSFileSourceConfiguration, +] + + +class FileSourceTemplateBase(StrictModel): + """Version of FileSourceTemplate we can send to the UI/API. + + The configuration key in the child type may have secretes + and shouldn't be exposed over the API - at least to non-admins. + """ + + id: str + name: Optional[str] + description: Optional[MarkdownContent] + # The UI should just show the most recent version but allow + # admins to define newer versions with new parameterizations + # and keep old versions in template catalog for backward compatibility + # for users with existing stores of that template. + version: int = 0 + # Like with multiple versions, allow admins to deprecate a + # template by hiding but keep it in the catalog for backward + # compatibility for users with existing stores of that template. + hidden: bool = False + variables: Optional[List[TemplateVariable]] = None + secrets: Optional[List[TemplateSecret]] = None + + +class FileSourceTemplateSummary(FileSourceTemplateBase): + type: FileSourceTemplateType + + +class FileSourceTemplate(FileSourceTemplateBase): + configuration: FileSourceTemplateConfiguration + + +FileSourceTemplateCatalog = RootModel[List[FileSourceTemplate]] + + +class FileSourceTemplateSummaries(RootModel): + root: List[FileSourceTemplateSummary] + + +def template_to_configuration( + template: FileSourceTemplate, + variables: Dict[str, TemplateVariableValueType], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> FileSourceConfiguration: + configuration_template = template.configuration + raw_config = expand_raw_config(configuration_template, variables, secrets, user_details) + return to_configuration_object(raw_config) + + +TypesToConfigurationClasses: Dict[FileSourceTemplateType, Type[FileSourceConfiguration]] = { + "posix": PosixFileSourceConfiguration, + "s3fs": S3FSFileSourceConfiguration, +} + + +def to_configuration_object(configuration_dict: Dict[str, Any]) -> FileSourceConfiguration: + if "type" not in configuration_dict: + raise KeyError("Configuration objects require a file source 'type' key, none found.") + object_store_type = configuration_dict["type"] + if object_store_type not in TypesToConfigurationClasses: + raise ValueError(f"Unknown file source type found in raw configuration dictionary ({object_store_type}).") + return TypesToConfigurationClasses[object_store_type](**configuration_dict) diff --git a/lib/galaxy/files/unittest_utils/__init__.py b/lib/galaxy/files/unittest_utils/__init__.py index c8ebbc13a5b3..e04da53aabd9 100644 --- a/lib/galaxy/files/unittest_utils/__init__.py +++ b/lib/galaxy/files/unittest_utils/__init__.py @@ -2,13 +2,16 @@ import tempfile from typing import Tuple -from galaxy.files import ConfiguredFileSources +from galaxy.files import ( + ConfiguredFileSources, + ConfiguredFileSourcesConf, +) from galaxy.files.plugins import FileSourcePluginsConfig class TestConfiguredFileSources(ConfiguredFileSources): def __init__(self, file_sources_config: FileSourcePluginsConfig, conf_dict: dict, test_root: str): - super().__init__(file_sources_config, conf_dict=conf_dict) + super().__init__(file_sources_config, ConfiguredFileSourcesConf(conf_dict=conf_dict)) self.test_root = test_root diff --git a/lib/galaxy/managers/file_source_instances.py b/lib/galaxy/managers/file_source_instances.py new file mode 100644 index 000000000000..e8d55496030e --- /dev/null +++ b/lib/galaxy/managers/file_source_instances.py @@ -0,0 +1,421 @@ +import logging +from typing import ( + Any, + cast, + Dict, + List, + Literal, + Optional, + Set, + Union, +) +from uuid import uuid4 + +from pydantic import BaseModel + +from galaxy.exceptions import ( + ItemOwnershipException, + RequestParameterInvalidException, + RequestParameterMissingException, +) +from galaxy.files import ( + FileSourceDictifiable, + FileSourceScore, + ProvidesFileSourcesUserContext, + UserDefinedFileSources, +) +from galaxy.files.plugins import ( + FileSourcePluginLoader, + FileSourcePluginsConfig, +) +from galaxy.files.sources import ( + BaseFilesSource, + file_source_type_is_browsable, + FilesSourceProperties, + PluginKind, +) +from galaxy.files.templates import ( + ConfiguredFileSourceTemplates, + FileSourceConfiguration, + FileSourceTemplateSummaries, + FileSourceTemplateType, +) +from galaxy.managers.context import ProvidesUserContext +from galaxy.model import ( + OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE, + User, + UserFileSource, +) +from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.security.vault import Vault +from galaxy.util.config_templates import TemplateVariableValueType +from galaxy.util.plugin_config import plugin_source_from_dict +from .object_store_instances import ( + CreateInstancePayload, + ModifyInstancePayload, + recover_secrets, + UpdateInstancePayload, + UpdateInstanceSecretPayload, + UpgradeInstancePayload, +) + +log = logging.getLogger(__name__) + +USER_FILE_SOURCES_SCHEME = "gxuserfiles" + + +class UserFileSourceModel(BaseModel): + id: Union[str, int] + uuid: str + uri_root: str + name: str + description: Optional[str] + type: FileSourceTemplateType + template_id: str + template_version: int + variables: Optional[Dict[str, TemplateVariableValueType]] + secrets: List[str] + + +class UserDefinedFileSourcesConfig(BaseModel): + user_object_store_index_by: Literal["uuid", "id"] + + @staticmethod + def from_app_config(config) -> "UserDefinedFileSourcesConfig": + user_object_store_index_by = config.user_object_store_index_by + assert user_object_store_index_by in ["uuid", "id"] + return UserDefinedFileSourcesConfig(user_object_store_index_by=user_object_store_index_by) + + +class FileSourceInstancesManager: + _catalog: ConfiguredFileSourceTemplates + _sa_session: galaxy_scoped_session + _app_vault: Vault + _app_config: UserDefinedFileSourcesConfig + + def __init__( + self, + catalog: ConfiguredFileSourceTemplates, + sa_session: galaxy_scoped_session, + vault: Vault, + app_config: UserDefinedFileSourcesConfig, + ): + self._catalog = catalog + self._sa_session = sa_session + self._app_vault = vault + self._app_config = app_config + + @property + def summaries(self) -> FileSourceTemplateSummaries: + return self._catalog.summaries + + def index(self, trans: ProvidesUserContext) -> List[UserFileSourceModel]: + stores = self._sa_session.query(UserFileSource).filter(UserFileSource.user_id == trans.user.id).all() + return [self._to_model(trans, s) for s in stores] + + def show(self, trans: ProvidesUserContext, id: Union[str, int]) -> UserFileSourceModel: + user_file_source = self._get(trans, id) + return self._to_model(trans, user_file_source) + + def modify_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: ModifyInstancePayload + ) -> UserFileSourceModel: + if isinstance(payload, UpgradeInstancePayload): + return self._upgrade_instance(trans, id, payload) + elif isinstance(payload, UpdateInstanceSecretPayload): + return self._update_instance_secret(trans, id, payload) + else: + assert isinstance(payload, UpdateInstancePayload) + return self._update_instance(trans, id, payload) + + def _upgrade_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpgradeInstancePayload + ) -> UserFileSourceModel: + persisted_object_store = self._get(trans, id) + catalog = self._catalog + template = catalog.find_template_by(persisted_object_store.template_id, payload.template_version) + persisted_object_store.template_version = template.version + persisted_object_store.template_definition = template.model_dump() + old_variables = persisted_object_store.template_variables or {} + updated_variables = payload.variables + actual_variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = {} + for variable in template.variables or []: + variable_name = variable.name + old_value = old_variables.get(variable_name) + updated_value = updated_variables.get(variable_name, old_value) + if updated_value: + actual_variables[variable_name] = updated_value + + persisted_object_store.template_variables = actual_variables + old_secrets = persisted_object_store.template_secrets or [] + new_secrets = payload.secrets + + recorded_secrets = persisted_object_store.template_secrets or [] + + user_vault = trans.user_vault + upgraded_template_secrets = [] + for secret in template.secrets or []: + secret_name = secret.name + upgraded_template_secrets.append(secret_name) + if secret_name not in new_secrets and secret_name not in old_secrets: + raise RequestParameterMissingException(f"secret {secret_name} not set in supplied request") + if secret_name not in new_secrets: + # keep old value + continue + + secret_value = new_secrets[secret_name] + key = persisted_object_store.vault_key(secret_name, self._app_config) + user_vault.write_secret(key, secret_value) + if secret_name not in recorded_secrets: + recorded_secrets.append(secret_name) + + secrets_to_delete: List[str] = [] + for recorded_secret in recorded_secrets: + if recorded_secret not in upgraded_template_secrets: + key = persisted_object_store.vault_key(recorded_secret, self._app_config) + log.info(f"deleting {key} from user vault") + user_vault.delete_secret(key) + secrets_to_delete.append(recorded_secret) + + for secret_to_delete in secrets_to_delete: + recorded_secrets.remove(secret_to_delete) + + persisted_object_store.template_secrets = recorded_secrets + self._save(persisted_object_store) + rval = self._to_model(trans, persisted_object_store) + return rval + + def _update_instance( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstancePayload + ) -> UserFileSourceModel: + persisted_file_source = self._get(trans, id) + if payload.name is not None: + persisted_file_source.name = payload.name + if payload.description is not None: + persisted_file_source.description = payload.description + if payload.variables is not None: + # maybe just record the valid variables according to template like in upgrade + persisted_file_source.template_variables = payload.variables + self._save(persisted_file_source) + return self._to_model(trans, persisted_file_source) + + def _update_instance_secret( + self, trans: ProvidesUserContext, id: Union[str, int], payload: UpdateInstanceSecretPayload + ) -> UserFileSourceModel: + persisted_file_source = self._get(trans, id) + user_vault = trans.user_vault + key = persisted_file_source.vault_key(payload.secret_name, self._app_config) + user_vault.write_secret(key, payload.secret_value) + return self._to_model(trans, persisted_file_source) + + def create_instance(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> UserFileSourceModel: + catalog = self._catalog + catalog.validate(payload) + template = catalog.find_template(payload) + assert template + user_vault = trans.user_vault + persisted_file_source = UserFileSource() + persisted_file_source.user_id = trans.user.id + assert persisted_file_source.user_id + persisted_file_source.uuid = uuid4().hex + persisted_file_source.template_definition = template.model_dump() + persisted_file_source.template_id = template.id + persisted_file_source.template_version = template.version + persisted_file_source.template_variables = payload.variables + persisted_file_source.name = payload.name + persisted_file_source.description = payload.description + self._save(persisted_file_source) + + # the exception handling below will cleanup object stores that cannot be + # finalized with a successful secret setting but it might be worth considering + # something more robust. Two ideas would be to set a uuid on the persisted_file_source + # and key the secrets on that instead of the of the ID (but this raises the question + # are unused secrets in the vault maybe even worse than broken db objects) or + # set a state on the DB objects and with INITIAL and ACTIVE states. State + # idea might be nice because then we could add INACTIVE state that would prevent + # new data from being added but still allow access. + recorded_secrets = [] + try: + for secret, value in payload.secrets.items(): + key = persisted_file_source.vault_key(secret, self._app_config) + user_vault.write_secret(key, value) + recorded_secrets.append(secret) + except Exception: + self._sa_session.delete(persisted_file_source) + raise + persisted_file_source.template_secrets = recorded_secrets + self._save(persisted_file_source) + return self._to_model(trans, persisted_file_source) + + def _index_filter(self, id: Union[str, int]): + index_by = self._app_config.user_object_store_index_by + index_filter: Any + if index_by == "id": + id_as_int = int(id) + index_filter = UserFileSource.__table__.c.id == id_as_int + else: + id_as_str = str(id) + index_filter = UserFileSource.__table__.c.uuid == id_as_str + return index_filter + + def _get(self, trans: ProvidesUserContext, id: Union[str, int]) -> UserFileSource: + filter = self._index_filter(id) + user_file_source = self._sa_session.query(UserFileSource).filter(filter).one_or_none() + if user_file_source is None: + raise RequestParameterInvalidException(f"Failed to fetch object store for id {id}") + if user_file_source.user != trans.user: + raise ItemOwnershipException() + return user_file_source + + def _save(self, user_file_source: UserFileSource) -> None: + self._sa_session.add(user_file_source) + self._sa_session.flush([user_file_source]) + self._sa_session.commit() + + def _to_model(self, trans, persisted_file_source: UserFileSource) -> UserFileSourceModel: + file_source_type = persisted_file_source.template.configuration.type + # These shouldn't be null but sometimes can be? + secrets = persisted_file_source.template_secrets or [] + index_by = self._app_config.user_object_store_index_by + response_id: Union[str, int] + if index_by == "id": + ufs_id = str(persisted_file_source.id) + response_id = persisted_file_source.id + else: + ufs_id = str(persisted_file_source.uuid) + response_id = ufs_id + uri_root = f"{USER_FILE_SOURCES_SCHEME}://{ufs_id}" + return UserFileSourceModel( + id=response_id, + uuid=str(persisted_file_source.uuid), + uri_root=uri_root, + type=file_source_type, + template_id=persisted_file_source.template_id, + template_version=persisted_file_source.template_version, + variables=persisted_file_source.template_variables, + secrets=secrets, + name=persisted_file_source.name, + description=persisted_file_source.description, + ) + + +class UserDefinedFileSourcesImpl(UserDefinedFileSources): + _sa_session: galaxy_scoped_session + _app_config: UserDefinedFileSourcesConfig + _file_sources_config: FileSourcePluginsConfig + _plugin_loader: FileSourcePluginLoader + _app_vault: Vault + + def __init__( + self, + sa_session: galaxy_scoped_session, + app_config: UserDefinedFileSourcesConfig, + file_sources_config: FileSourcePluginsConfig, + plugin_loader: FileSourcePluginLoader, + vault: Vault, + ): + self._sa_session = sa_session + self._app_config = app_config + self._plugin_loader = plugin_loader + self._file_sources_config = file_sources_config + self._app_vault = vault + + def _user_file_source(self, uri: str) -> Optional[UserFileSource]: + if "://" not in uri: + return None + uri_scheme, uri_root = uri.split("://", 1) + if uri_scheme != USER_FILE_SOURCES_SCHEME: + return None + index_by = self._app_config.user_object_store_index_by + index_filter: Any + if index_by == "id": + index_filter = UserFileSource.__table__.c.id == uri_root + else: + index_filter = UserFileSource.__table__.c.uuid == uri_root + + user_object_store: UserFileSource = self._sa_session.query(UserFileSource).filter(index_filter).one() + return user_object_store + + def _file_source_properties_from_uri(self, uri: str) -> Optional[FilesSourceProperties]: + user_file_source = self._user_file_source(uri) + if not user_file_source: + return None + return self._file_source_properties(user_file_source) + + def _file_source_properties(self, user_file_source: UserFileSource) -> FilesSourceProperties: + secrets = recover_secrets(user_file_source, self._app_vault, self._app_config) + file_source_configuration: FileSourceConfiguration = user_file_source.file_source_configuration(secrets=secrets) + file_source_properties = cast(FilesSourceProperties, file_source_configuration.model_dump()) + file_source_properties["label"] = user_file_source.name + file_source_properties["doc"] = user_file_source.description + file_source_properties["id"] = f"{user_file_source.uuid}" + file_source_properties["scheme"] = USER_FILE_SOURCES_SCHEME + # when should this be false - I think it can depend on the plugin type + # no reason to prevent users from writing to their own file sources? + file_source_properties["writable"] = True + return file_source_properties + + def validate_uri_root(self, uri: str, user_context: "ProvidesFileSourcesUserContext") -> None: + user_object_store = self._user_file_source(uri) + if not user_object_store: + return + if user_object_store.user.username != user_context.username: + raise ItemOwnershipException("Your Galaxy user does not have access to the requested resource.") + + def find_best_match(self, url: str) -> Optional[FileSourceScore]: + files_source_properties = self._file_source_properties_from_uri(url) + if files_source_properties is None: + return None + file_source = self._file_source(files_source_properties) + return FileSourceScore(file_source, len(url)) + + def _file_source(self, files_source_properties: FilesSourceProperties) -> BaseFilesSource: + plugin_source = plugin_source_from_dict([cast(Dict[str, Any], files_source_properties)]) + file_source = self._plugin_loader.load_plugins( + plugin_source, + self._file_sources_config, + )[0] + return file_source + + def _all_user_file_source_properties(self, user_context: FileSourceDictifiable) -> List[FilesSourceProperties]: + username_filter = User.__table__.c.username == user_context.username + user: User = self._sa_session.query(User).filter(username_filter).one() + all_file_source_properties: List[FilesSourceProperties] = [] + for user_file_source in user.file_sources: + files_source_properties = self._file_source_properties(user_file_source) + all_file_source_properties.append(files_source_properties) + return all_file_source_properties + + def user_file_sources_to_dicts( + self, + for_serialization: bool, + user_context: FileSourceDictifiable, + browsable_only: Optional[bool] = False, + include_kind: Optional[Set[PluginKind]] = None, + exclude_kind: Optional[Set[PluginKind]] = None, + ) -> List[FilesSourceProperties]: + """Write out user file sources as list of config dictionaries.""" + as_dicts = [] + for files_source_properties in self._all_user_file_source_properties(user_context): + plugin_kind = PluginKind.rfs + if include_kind and plugin_kind not in include_kind: + continue + if exclude_kind and plugin_kind in exclude_kind: + continue + files_source_type = files_source_properties["type"] + is_browsable = file_source_type_is_browsable(self._plugin_loader.get_plugin_type_class(files_source_type)) + if browsable_only and not is_browsable: + continue + file_source = self._file_source(files_source_properties) + as_dicts.append(file_source.to_dict(for_serialization=for_serialization, user_context=user_context)) + return as_dicts + + +__all__ = ( + "CreateInstancePayload", + "FileSourceInstancesManager", + "ModifyInstancePayload", + "UpdateInstancePayload", + "UpdateInstanceSecretPayload", + "UpgradeInstancePayload", +) diff --git a/lib/galaxy/managers/object_store_instances.py b/lib/galaxy/managers/object_store_instances.py index ae1565e462ab..09b5f7e64326 100644 --- a/lib/galaxy/managers/object_store_instances.py +++ b/lib/galaxy/managers/object_store_instances.py @@ -26,9 +26,11 @@ ) from galaxy.managers.context import ProvidesUserContext from galaxy.model import ( + HasConfigSecrets, OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE, User, UserObjectStore, + UsesTemplatesAppConfig, ) from galaxy.model.scoped_session import galaxy_scoped_session from galaxy.objectstore import ( @@ -44,11 +46,11 @@ ObjectStoreTemplateSummaries, ObjectStoreTemplateType, ) -from galaxy.objectstore.templates.models import ObjectStoreTemplateVariableValueType from galaxy.security.vault import ( UserVaultWrapper, Vault, ) +from galaxy.util.config_templates import TemplateVariableValueType log = logging.getLogger(__name__) @@ -58,14 +60,14 @@ class CreateInstancePayload(BaseModel): description: Optional[str] = None template_id: str template_version: int - variables: Dict[str, ObjectStoreTemplateVariableValueType] + variables: Dict[str, TemplateVariableValueType] secrets: Dict[str, str] class UpdateInstancePayload(BaseModel): name: Optional[str] = None description: Optional[str] = None - variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] = None + variables: Optional[Dict[str, TemplateVariableValueType]] = None class UpdateInstanceSecretPayload(BaseModel): @@ -75,7 +77,7 @@ class UpdateInstanceSecretPayload(BaseModel): class UpgradeInstancePayload(BaseModel): template_version: int - variables: Dict[str, ObjectStoreTemplateVariableValueType] + variables: Dict[str, TemplateVariableValueType] secrets: Dict[str, str] @@ -85,7 +87,7 @@ class UserConcreteObjectStoreModel(ConcreteObjectStoreModel): type: ObjectStoreTemplateType template_id: str template_version: int - variables: Optional[Dict[str, ObjectStoreTemplateVariableValueType]] + variables: Optional[Dict[str, TemplateVariableValueType]] secrets: List[str] @@ -160,7 +162,7 @@ def _upgrade_instance( continue secret_value = new_secrets[secret_name] - key = user_vault_key(persisted_object_store, secret_name, self._app_config) + key = persisted_object_store.vault_key(secret_name, self._app_config) user_vault.write_secret(key, secret_value) if secret_name not in recorded_secrets: recorded_secrets.append(secret_name) @@ -168,7 +170,7 @@ def _upgrade_instance( secrets_to_delete: List[str] = [] for recorded_secret in recorded_secrets: if recorded_secret not in upgraded_template_secrets: - key = user_vault_key(persisted_object_store, recorded_secret, self._app_config) + key = persisted_object_store.vault_key(recorded_secret, self._app_config) log.info(f"deleting {key} from user vault") user_vault.delete_secret(key) secrets_to_delete.append(recorded_secret) @@ -203,7 +205,7 @@ def _update_instance_secret( ) -> UserConcreteObjectStoreModel: persisted_object_store = self._get(trans, id) user_vault = trans.user_vault - key = user_vault_key(persisted_object_store, payload.secret_name, self._app_config) + key = persisted_object_store.vault_key(payload.secret_name, self._app_config) user_vault.write_secret(key, payload.secret_value) return self._to_model(trans, persisted_object_store) @@ -238,7 +240,7 @@ def create_instance( recorded_secrets = [] try: for secret, value in payload.secrets.items(): - key = user_vault_key(persisted_object_store, secret, self._app_config) + key = persisted_object_store.vault_key(secret, self._app_config) user_vault.write_secret(key, value) recorded_secrets.append(secret) except Exception: @@ -321,19 +323,8 @@ def _to_model(self, trans, persisted_object_store: UserObjectStore) -> UserConcr ) -def user_vault_key(user_object_store: UserObjectStore, secret: str, app_config: UserObjectStoresAppConfig) -> str: - if app_config.user_object_store_index_by == "id": - uos_id = str(user_object_store.id) - else: - uos_id = str(user_object_store.uuid) - assert uos_id - user_vault_id_prefix = f"object_store_config/{uos_id}" - key = f"{user_vault_id_prefix}/{secret}" - return key - - def recover_secrets( - user_object_store: UserObjectStore, vault: Vault, app_config: UserObjectStoresAppConfig + user_object_store: HasConfigSecrets, vault: Vault, app_config: UsesTemplatesAppConfig ) -> Dict[str, str]: user: User = user_object_store.user user_vault = UserVaultWrapper(vault, user) @@ -344,9 +335,8 @@ def recover_secrets( # persisted. persisted_secret_names = user_object_store.template_secrets or [] for secret in persisted_secret_names: - vault_key = user_vault_key(user_object_store, secret, app_config) + vault_key = user_object_store.vault_key(secret, app_config) secret_value = user_vault.read_secret(vault_key) - # assert secret_value if secret_value is not None: secrets[secret] = secret_value return secrets diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 707ee1af05e5..dfd76e524d95 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -133,6 +133,11 @@ import galaxy.model.tags import galaxy.security.passwords import galaxy.util +from galaxy.files.templates import ( + FileSourceConfiguration, + FileSourceTemplate, + template_to_configuration as file_source_template_to_configuration, +) from galaxy.model.base import ( ensure_object_added_to_session, transaction, @@ -155,7 +160,7 @@ from galaxy.objectstore.templates import ( ObjectStoreConfiguration, ObjectStoreTemplate, - template_to_configuration, + template_to_configuration as object_store_template_to_configuration, ) from galaxy.schema.invocation import ( InvocationCancellationUserRequest, @@ -777,6 +782,7 @@ class User(Base, Dictifiable, RepresentById): back_populates="user", order_by=lambda: desc(GalaxySession.update_time), cascade_backrefs=False ) object_stores: Mapped[List["UserObjectStore"]] = relationship(back_populates="user") + file_sources: Mapped[List["UserFileSource"]] = relationship(back_populates="user") quotas: Mapped[List["UserQuotaAssociation"]] = relationship(back_populates="user") quota_source_usages: Mapped[List["UserQuotaSourceUsage"]] = relationship(back_populates="user") social_auth: Mapped[List["UserAuthnzToken"]] = relationship(back_populates="user") @@ -10879,8 +10885,29 @@ def __init__(self, name=None, value=None): self.value = value -class UserObjectStore(Base, RepresentById): +class UsesTemplatesAppConfig(Protocol): + user_object_store_index_by: Literal["uuid", "id"] + + +class HasConfigSecrets(RepresentById): + secret_config_type: str + template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] + uuid: Mapped[Union[UUID, str]] + user: Mapped["User"] + + def vault_key(self, secret: str, app_config: UsesTemplatesAppConfig) -> str: + if app_config.user_object_store_index_by == "id": + id_str = str(self.id) + else: + id_str = str(self.uuid) + user_vault_id_prefix = f"{self.secret_config_type}/{id_str}" + key = f"{user_vault_id_prefix}/{secret}" + return key + + +class UserObjectStore(Base, HasConfigSecrets, RepresentById): __tablename__ = "user_object_store" + secret_config_type = "object_store_config" id: Mapped[int] = mapped_column(Integer, primary_key=True) user_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("galaxy_user.id"), index=True) @@ -10907,7 +10934,7 @@ class UserObjectStore(Base, RepresentById): # Track a list of secrets that were defined for this object store at creation template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] = mapped_column(JSONType) - user = relationship("User", back_populates="object_stores") + user: Mapped["User"] = relationship("User", back_populates="object_stores") @property def template(self) -> ObjectStoreTemplate: @@ -10921,7 +10948,7 @@ def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConf "id": user.id, } variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {} - return template_to_configuration( + return object_store_template_to_configuration( self.template, variables=variables, secrets=secrets, @@ -10929,6 +10956,57 @@ def object_store_configuration(self, secrets: Dict[str, Any]) -> ObjectStoreConf ) +class UserFileSource(Base, HasConfigSecrets, RepresentById): + __tablename__ = "user_file_source" + secret_config_type = "file_source_config" + + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[Optional[int]] = mapped_column(ForeignKey("galaxy_user.id"), index=True) + uuid: Mapped[Union[UUID, str]] = mapped_column(UUIDType(), index=True) + create_time: Mapped[datetime] = mapped_column(default=now) + update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, index=True) + # user specified name of the instance they've created + name: Mapped[str] = mapped_column(String(255), index=True) + # user specified description of the instance they've created + description: Mapped[Optional[str]] = mapped_column(Text) + # the template store id + template_id: Mapped[str] = mapped_column(String(255), index=True) + # the template store version (0, 1, ...) + template_version: Mapped[int] = mapped_column(index=True) + # Full template from file_sources_templates.yml catalog. + # For tools we just store references, so here we could easily just use + # the id/version and not record the definition... as the templates change + # over time this choice has some big consequences despite being easy to swap + # implementations. + template_definition: Mapped[Optional[OBJECT_STORE_TEMPLATE_DEFINITION_TYPE]] = mapped_column(JSONType) + # Big JSON blob of the variable name -> value mapping defined for the store's + # variables by the user. + template_variables: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE]] = mapped_column(JSONType) + # Track a list of secrets that were defined for this object store at creation + template_secrets: Mapped[Optional[OBJECT_STORE_TEMPLATE_CONFIGURATION_SECRET_NAMES_TYPE]] = mapped_column(JSONType) + + user: Mapped["User"] = relationship("User", back_populates="file_sources") + + @property + def template(self) -> FileSourceTemplate: + return FileSourceTemplate(**self.template_definition or {}) + + def file_source_configuration(self, secrets: Dict[str, Any]) -> FileSourceConfiguration: + user = self.user + user_details = { + "username": user.username, + "email": user.email, + "id": user.id, + } + variables: OBJECT_STORE_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {} + return file_source_template_to_configuration( + template=self.template, + variables=variables, + secrets=secrets, + user_details=user_details, + ) + + class UserAction(Base, RepresentById): __tablename__ = "user_action" diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py new file mode 100644 index 000000000000..8e0b57760820 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/c14a3c93d66b_add_user_defined_file_sources.py @@ -0,0 +1,57 @@ +"""add user defined file sources + +Revision ID: c14a3c93d66b +Revises: c14a3c93d66a +Create Date: 2023-04-01 17:25:37.553039 + +""" + +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Integer, + String, + Text, +) + +from galaxy.model.custom_types import ( + JSONType, + UUIDType, +) +from galaxy.model.migrations.util import ( + create_table, + drop_table, +) + +# revision identifiers, used by Alembic. +revision = "c14a3c93d66b" +down_revision = "c14a3c93d66a" +branch_labels = None +depends_on = None + + +# database object names used in this revision +table_name = "user_file_source" + + +def upgrade(): + create_table( + table_name, + Column("id", Integer, primary_key=True), + Column("user_id", Integer, ForeignKey("galaxy_user.id"), nullable=False, index=True), + Column("uuid", UUIDType, nullable=False, index=True), + Column("name", String(255), index=True), + Column("description", Text, index=True), + Column("create_time", DateTime), + Column("update_time", DateTime), + Column("template_id", String(255), index=True), + Column("template_version", Integer, index=True), + Column("template_definition", JSONType), + Column("template_variables", JSONType), + Column("template_secrets", JSONType), + ) + + +def downgrade(): + drop_table(table_name) diff --git a/lib/galaxy/objectstore/templates/manager.py b/lib/galaxy/objectstore/templates/manager.py index 901931b956c2..1ae86b60c2ee 100644 --- a/lib/galaxy/objectstore/templates/manager.py +++ b/lib/galaxy/objectstore/templates/manager.py @@ -1,7 +1,5 @@ import os from typing import ( - Any, - Dict, List, Optional, ) @@ -9,35 +7,29 @@ from typing_extensions import Protocol from yaml import safe_load -from galaxy.exceptions import ( - ObjectNotFound, - RequestParameterMissingException, -) from galaxy.objectstore.badges import serialize_badges +from galaxy.util.config_templates import ( + apply_syntactic_sugar, + find_template, + find_template_by, + InstanceDefinition, + RawTemplateConfig, + TemplateReference, + validate_secrets_and_variables, + verify_vault_configured_if_uses_secrets, +) from .models import ( ObjectStoreTemplate, ObjectStoreTemplateCatalog, ObjectStoreTemplateSummaries, ) -RawTemplateConfig = Dict[str, Any] - class AppConfigProtocol(Protocol): object_store_templates: Optional[List[RawTemplateConfig]] object_store_templates_config_file: Optional[str] -class TemplateReference(Protocol): - template_id: str - template_version: int - - -class InstanceDefinition(TemplateReference, Protocol): - variables: Dict[str, Any] - secrets: Dict[str, str] - - SECRETS_NEED_VAULT_MESSAGE = "The object store templates configuration can not be used - a Galaxy vault must be configured for templates that use secrets - please set the vault_config_file configuration option to point at a valid vault configuration." @@ -57,19 +49,15 @@ def from_app_config(config: AppConfigProtocol, vault_configured=None) -> "Config raw_config = safe_load(f) if raw_config is None: raw_config = [] - templates = ConfiguredObjectStoreTemplates(raw_config_to_catalog(raw_config)) - if vault_configured is False and templates.configuration_uses_secrets: - raise Exception(SECRETS_NEED_VAULT_MESSAGE) + catalog = raw_config_to_catalog(raw_config) + verify_vault_configured_if_uses_secrets( + catalog, + vault_configured, + SECRETS_NEED_VAULT_MESSAGE, + ) + templates = ConfiguredObjectStoreTemplates(catalog) return templates - @property - def configuration_uses_secrets(self) -> bool: - templates = self.catalog.root - for template in templates: - if template.secrets and len(template.secrets) > 0: - return True - return False - @property def summaries(self) -> ObjectStoreTemplateSummaries: templates = self.catalog.root @@ -87,55 +75,17 @@ def summaries(self) -> ObjectStoreTemplateSummaries: def find_template(self, instance_reference: TemplateReference) -> ObjectStoreTemplate: """Find the corresponding template and throw ObjectNotFound if not available.""" - template_id = instance_reference.template_id - template_version = instance_reference.template_version - return self.find_template_by(template_id, template_version) + return find_template(self.catalog.root, instance_reference, "object store") def find_template_by(self, template_id: str, template_version: int) -> ObjectStoreTemplate: - templates = self.catalog.root - - for template in templates: - if template.id == template_id and template.version == template_version: - return template - - raise ObjectNotFound( - f"Could not find a object store template with id {template_id} and version {template_version}" - ) + return find_template_by(self.catalog.root, template_id, template_version, "object store") def validate(self, instance: InstanceDefinition): template = self.find_template(instance) - secrets = instance.secrets - for template_secret in template.secrets or []: - name = template_secret.name - if name not in secrets: - raise RequestParameterMissingException(f"Must define secret '{name}'") - variables = instance.variables - for template_variable in template.variables or []: - name = template_variable.name - if name not in variables: - raise RequestParameterMissingException(f"Must define variable '{name}'") + validate_secrets_and_variables(instance, template) # TODO: validate no extra variables def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> ObjectStoreTemplateCatalog: - effective_root = _apply_syntactic_sugar(raw_config) + effective_root = apply_syntactic_sugar(raw_config) return ObjectStoreTemplateCatalog.model_validate(effective_root) - - -def _apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: - templates = [] - for template in raw_templates: - _force_key_to_list(template, "variables") - _force_key_to_list(template, "secrets") - templates.append(template) - return templates - - -def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: - value = template.get(key, None) - if isinstance(value, dict): - value_as_list = [] - for key_name, key_value in value.items(): - key_value["name"] = key_name - value_as_list.append(key_value) - template[key] = value_as_list diff --git a/lib/galaxy/objectstore/templates/models.py b/lib/galaxy/objectstore/templates/models.py index 9f225786194a..92b3abd2b120 100644 --- a/lib/galaxy/objectstore/templates/models.py +++ b/lib/galaxy/objectstore/templates/models.py @@ -7,28 +7,26 @@ Union, ) -from boltons.iterutils import remap -from jinja2.nativetypes import NativeEnvironment -from pydantic import ( - BaseModel, - ConfigDict, - RootModel, -) +from pydantic import RootModel from typing_extensions import Literal from galaxy.objectstore.badges import ( BadgeDict, StoredBadgeDict, ) +from galaxy.util.config_templates import ( + expand_raw_config, + MarkdownContent, + StrictModel, + TemplateExpansion, + TemplateSecret, + TemplateVariable, + TemplateVariableType, + TemplateVariableValueType, +) - -class StrictModel(BaseModel): - model_config = ConfigDict(extra="forbid") - - -ObjectStoreTemplateVariableType = Literal["string", "boolean", "integer"] -ObjectStoreTemplateVariableValueType = Union[str, bool, int] -TemplateExpansion = str +ObjectStoreTemplateVariableType = TemplateVariableType +ObjectStoreTemplateVariableValueType = TemplateVariableValueType ObjectStoreTemplateType = Literal["s3", "azure_blob", "disk", "generic_s3"] @@ -60,6 +58,8 @@ class S3ObjectStoreTemplateConfiguration(StrictModel): auth: S3AuthTemplate bucket: S3BucketTemplate badges: BadgeList = None + template_start: Optional[str] = None + template_end: Optional[str] = None class S3ObjectStoreConfiguration(StrictModel): @@ -92,6 +92,8 @@ class AzureObjectStoreTemplateConfiguration(StrictModel): auth: AzureAuthTemplate container: AzureContainerTemplate badges: BadgeList = None + template_start: Optional[str] = None + template_end: Optional[str] = None class AzureObjectStoreConfiguration(StrictModel): @@ -105,6 +107,8 @@ class DiskObjectStoreTemplateConfiguration(StrictModel): type: Literal["disk"] files_dir: Union[str, TemplateExpansion] badges: BadgeList = None + template_start: Optional[str] = None + template_end: Optional[str] = None class DiskObjectStoreConfiguration(StrictModel): @@ -133,6 +137,8 @@ class GenericS3ObjectStoreTemplateConfiguration(StrictModel): bucket: S3BucketTemplate connection: S3ConnectionTemplate badges: BadgeList = None + template_start: Optional[str] = None + template_end: Optional[str] = None class GenericS3ObjectStoreConfiguration(StrictModel): @@ -155,18 +161,10 @@ class GenericS3ObjectStoreConfiguration(StrictModel): AzureObjectStoreConfiguration, GenericS3ObjectStoreConfiguration, ] -MarkdownContent = str -class ObjectStoreTemplateVariable(StrictModel): - name: str - help: Optional[MarkdownContent] - type: ObjectStoreTemplateVariableType - - -class ObjectStoreTemplateSecret(StrictModel): - name: str - help: Optional[MarkdownContent] +ObjectStoreTemplateVariable = TemplateVariable +ObjectStoreTemplateSecret = TemplateSecret class ObjectStoreTemplateBase(StrictModel): @@ -188,8 +186,8 @@ class ObjectStoreTemplateBase(StrictModel): # template by hiding but keep it in the catalog for backward # compatibility for users with existing stores of that template. hidden: bool = False - variables: Optional[List[ObjectStoreTemplateVariable]] = None - secrets: Optional[List[ObjectStoreTemplateSecret]] = None + variables: Optional[List[TemplateVariable]] = None + secrets: Optional[List[TemplateSecret]] = None class ObjectStoreTemplateSummary(ObjectStoreTemplateBase): @@ -215,20 +213,7 @@ def template_to_configuration( user_details: Dict[str, Any], ) -> ObjectStoreConfiguration: configuration_template = template.configuration - template_variables = { - "variables": variables, - "secrets": secrets, - "user": user_details, - } - - def expand_template(_, key, value): - if isinstance(value, str) and "{{" in value and "}}" in value: - # NativeEnvironment preserves Python types - template = NativeEnvironment().from_string(value) - return key, template.render(**template_variables) - return key, value - - raw_config = remap(configuration_template.model_dump(), visit=expand_template) + raw_config = expand_raw_config(configuration_template, variables, secrets, user_details) return to_configuration_object(raw_config) diff --git a/lib/galaxy/util/config_templates.py b/lib/galaxy/util/config_templates.py new file mode 100644 index 000000000000..48b784c39639 --- /dev/null +++ b/lib/galaxy/util/config_templates.py @@ -0,0 +1,271 @@ +"""Utilities for defining user configuration bits from admin templates. + +This is capturing code shared by file source templates and object store templates. +""" + +import os +from typing import ( + Any, + Dict, + List, + Optional, + Type, + TypeVar, + Union, +) + +from boltons.iterutils import remap +from pydantic import ( + BaseModel, + ConfigDict, +) +from typing_extensions import ( + Literal, + Protocol, +) + +try: + from jinja2.nativetypes import NativeEnvironment +except ImportError: + NativeEnvironment = None # type:ignore[assignment, misc, unused-ignore] + +from galaxy.exceptions import ( + ObjectNotFound, + RequestParameterInvalidException, + RequestParameterMissingException, +) + +TemplateVariableType = Literal["string", "path_component", "boolean", "integer"] +TemplateVariableValueType = Union[str, bool, int] +TemplateExpansion = str +MarkdownContent = str +RawTemplateConfig = Dict[str, Any] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class TemplateVariable(StrictModel): + name: str + help: Optional[MarkdownContent] + type: TemplateVariableType + + +class TemplateSecret(StrictModel): + name: str + help: Optional[MarkdownContent] + + +def _ensure_path_component(input: Any): + input_as_string = str(input) + if not acts_as_simple_path_component(input_as_string): + raise Exception("Path manipulation detected, failing evaluation") + return input + + +# NativeEnvironment preserves Python types +def _environment(template_start: str, template_end: str) -> NativeEnvironment: + env = NativeEnvironment( + variable_start_string=template_start, + variable_end_string=template_end, + ) + env.filters["ensure_path_component"] = _ensure_path_component + return env + + +class TemplateConfiguration(Protocol): + + def model_dump(self) -> Dict[str, Any]: + """Implements a pydantic model dump to build simple JSON dictionary.""" + + @property + def template_start(self) -> Optional[str]: + """Set a custom variable start for Jinja variable substitution. + + https://stackoverflow.com/questions/12083319/add-custom-tokens-in-jinja2-e-g-somevar + """ + + @property + def template_end(self) -> Optional[str]: + """Set a custom variable end for Jinja variable substitution. + + https://stackoverflow.com/questions/12083319/add-custom-tokens-in-jinja2-e-g-somevar + """ + + +def expand_raw_config( + template_configuration: TemplateConfiguration, + variables: Dict[str, TemplateVariableValueType], + secrets: Dict[str, str], + user_details: Dict[str, Any], +) -> Dict[str, Any]: + template_variables = { + "variables": variables, + "secrets": secrets, + "user": user_details, + } + + template_start = template_configuration.template_start or "{{" + template_end = template_configuration.template_end or "}}" + + def expand_template(_, key, value): + if isinstance(value, str) and template_start in value and template_end in value: + template = _environment(template_start, template_end).from_string(value) + return key, template.render(**template_variables) + return key, value + + raw_config = remap(template_configuration.model_dump(), visit=expand_template) + _clean_template_meta_parameters(raw_config) + return raw_config + + +def verify_vault_configured_if_uses_secrets(catalog, vault_configured: bool, exception_message: str) -> None: + if _catalog_uses_secrets(catalog) and not vault_configured: + raise Exception(exception_message) + + +def _catalog_uses_secrets(catalog) -> bool: + templates = catalog.root + for template in templates: + if template.secrets and len(template.secrets) > 0: + return True + return False + + +def _clean_template_meta_parameters(config: RawTemplateConfig) -> RawTemplateConfig: + # slight templating differences between what is allowed in the template definition + # and what is allowed in the actual configuration objects we send to respective modules + # to instantiate plugins. In particular, descriptions of how templating is done should + # eliminated after templates have been expanded. + meta_parameters = ["template_start", "template_end"] + for meta_parameter in meta_parameters: + if meta_parameter in config: + del config[meta_parameter] + return config + + +# cwl-like - convert simple dictionary to list of dictionaries for quickly +# configuring variables and secrets +def apply_syntactic_sugar(raw_templates: List[RawTemplateConfig]) -> List[RawTemplateConfig]: + templates = [] + for template in raw_templates: + _force_key_to_list(template, "variables") + _force_key_to_list(template, "secrets") + templates.append(template) + return templates + + +def _force_key_to_list(template: RawTemplateConfig, key: str) -> None: + value = template.get(key, None) + if isinstance(value, dict): + value_as_list = [] + for key_name, key_value in value.items(): + key_value["name"] = key_name + value_as_list.append(key_value) + template[key] = value_as_list + + +class TemplateReference(Protocol): + template_id: str + template_version: int + + +class InstanceDefinition(TemplateReference, Protocol): + variables: Dict[str, Any] + secrets: Dict[str, str] + + +class Template(Protocol): + @property + def id(self) -> str: + pass + + @property + def version(self) -> int: + pass + + @property + def variables(self) -> Optional[List[TemplateVariable]]: + pass + + @property + def secrets(self) -> Optional[List[TemplateSecret]]: + pass + + +T = TypeVar("T", bound=Template, covariant=True) + + +def find_template(templates: List[T], instance_reference: TemplateReference, what: str) -> T: + template_id = instance_reference.template_id + template_version = instance_reference.template_version + return find_template_by(templates, template_id, template_version, what) + + +def find_template_by(templates: List[T], template_id: str, template_version: int, what: str) -> T: + for template in templates: + if template.id == template_id and template.version == template_version: + return template + + raise ObjectNotFound(f"Could not find a {what} template with id {template_id} and version {template_version}") + + +def validate_secrets_and_variables(instance: InstanceDefinition, template: Template) -> None: + secrets = instance.secrets + for template_secret in template.secrets or []: + name = template_secret.name + if name not in secrets: + raise RequestParameterMissingException(f"Must define secret '{name}'") + secret_value = secrets[name] + if not isinstance(secret_value, str): + raise RequestParameterInvalidException(f"Secret value for secret '{name}' must be of type str") + variables = instance.variables + for template_variable in template.variables or []: + name = template_variable.name + if name not in variables: + raise RequestParameterMissingException(f"Must define variable '{name}'") + variable_value = variables[name] + template_type = template_variable.type + if template_type in ["string", "path_component"]: + if not isinstance(variable_value, str): + raise RequestParameterInvalidException(f"Variable value for variable '{name}' must be of type str") + if template_type == "path_component": + if ".." in variable_value or "/" in variable_value: + raise RequestParameterInvalidException( + f"Variable value for variable '{name}' must be simple path component, invalid characters found" + ) + if not acts_as_simple_path_component(variable_value): + raise RequestParameterInvalidException( + f"Variable value for variable '{name}' must be simple path component, invalid characters found" + ) + if template_type == "integer": + if not _is_of_exact_type(variable_value, int): + raise RequestParameterInvalidException(f"Variable value for variable '{name}' must be of type int") + if template_type == "boolean": + if not _is_of_exact_type(variable_value, bool): + raise RequestParameterInvalidException(f"Variable value for variable '{name}' must be of type bool") + + +def _is_of_exact_type(object: Any, target_type: Type): + # isinstance(False, int) and False == 0 are both True in Python... + # We are creating a DSL here that is intentionally more strict than Python + # so we are using type() instead of isinstance and we have the test coverage + # to ensure this is the desired behavior and remains. Think JSON typing, not + # pythonic typing. Galaxy's internals as a Python project should not be + # exposed here. + return type(object) == target_type # noqa: E721 + + +def acts_as_simple_path_component(value: str): + cwd = os.getcwd() + abs_path = os.path.abspath(f"{cwd}/{value}") + unaffected_by_normpath = os.path.normpath(abs_path) == abs_path + if not unaffected_by_normpath: + return False + should_be_cwd, should_be_value = os.path.split(abs_path) + if should_be_cwd != cwd: + return False + if should_be_value != value: + return False + return True diff --git a/lib/galaxy/webapps/galaxy/api/file_sources.py b/lib/galaxy/webapps/galaxy/api/file_sources.py new file mode 100644 index 000000000000..9e14f9209ea0 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/file_sources.py @@ -0,0 +1,95 @@ +import logging +from typing import List + +from fastapi import ( + Body, + Path, +) + +from galaxy.files.templates import FileSourceTemplateSummaries +from galaxy.managers.context import ProvidesUserContext +from galaxy.managers.file_source_instances import ( + CreateInstancePayload, + FileSourceInstancesManager, + ModifyInstancePayload, + UserFileSourceModel, +) +from . import ( + depends, + DependsOnTrans, + Router, +) + +log = logging.getLogger(__name__) + +router = Router(tags=["file_sources"]) + + +UserFileSourceIdPathParam: str = Path( + ..., title="User File Source ID", description="The index for a persisted UserFileSourceStore object." +) + + +@router.cbv +class FastAPIFileSources: + file_source_instances_manager: FileSourceInstancesManager = depends(FileSourceInstancesManager) + + @router.get( + "/api/file_source_templates", + summary="Get a list of file source templates available to build user defined file sources from", + response_description="A list of the configured file source templates.", + operation_id="file_sources__templates_index", + ) + def index_templates( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> FileSourceTemplateSummaries: + return self.file_source_instances_manager.summaries + + @router.post( + "/api/file_source_instances", + summary="Create a user-bound object store.", + operation_id="file_sources__create_instance", + ) + def create( + self, + trans: ProvidesUserContext = DependsOnTrans, + payload: CreateInstancePayload = Body(...), + ) -> UserFileSourceModel: + return self.file_source_instances_manager.create_instance(trans, payload) + + @router.get( + "/api/file_source_instances", + summary="Get a list of persisted file source instances defined by the requesting user.", + operation_id="file_sources__instances_index", + ) + def instance_index( + self, + trans: ProvidesUserContext = DependsOnTrans, + ) -> List[UserFileSourceModel]: + return self.file_source_instances_manager.index(trans) + + @router.get( + "/api/file_source_instances/{user_file_source_id}", + summary="Get a list of persisted file source instances defined by the requesting user.", + operation_id="file_sources__instances_get", + ) + def instances_show( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_file_source_id: str = UserFileSourceIdPathParam, + ) -> UserFileSourceModel: + return self.file_source_instances_manager.show(trans, user_file_source_id) + + @router.put( + "/api/file_source_instances/{user_file_source_id}", + summary="Update or upgrade user file source instance.", + operation_id="file_sources__instances_update", + ) + def update_instance( + self, + trans: ProvidesUserContext = DependsOnTrans, + user_file_source_id: str = UserFileSourceIdPathParam, + payload: ModifyInstancePayload = Body(...), + ) -> UserFileSourceModel: + return self.file_source_instances_manager.modify_instance(trans, user_file_source_id, payload) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 0fdcef4b59cf..da31b21fac5e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -227,6 +227,11 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route("/object_store_instances/{user_object_store_id}/edit") webapp.add_client_route("/object_store_instances/{user_object_store_id}/upgrade") webapp.add_client_route("/object_store_templates/{template_id}/new") + webapp.add_client_route("/file_source_instances/create") + webapp.add_client_route("/file_source_instances/index") + webapp.add_client_route("/file_source_instances/{user_file_source_id}/edit") + webapp.add_client_route("/file_source_instances/{user_file_source_id}/upgrade") + webapp.add_client_route("/file_source_templates/{template_id}/new") webapp.add_client_route("/welcome/new") webapp.add_client_route("/visualizations") webapp.add_client_route("/visualizations/edit") diff --git a/packages/files/setup.cfg b/packages/files/setup.cfg index f294659aa77f..dc0599c9b7c3 100644 --- a/packages/files/setup.cfg +++ b/packages/files/setup.cfg @@ -32,7 +32,7 @@ version = 23.2.dev0 [options] include_package_data = True install_requires = - galaxy-util + galaxy-util[config_template] fs isal typing-extensions diff --git a/packages/objectstore/setup.cfg b/packages/objectstore/setup.cfg index 0b47a88eaa10..6ba5fe551e67 100644 --- a/packages/objectstore/setup.cfg +++ b/packages/objectstore/setup.cfg @@ -33,10 +33,9 @@ version = 23.2.dev0 [options] include_package_data = True install_requires = - galaxy-util + galaxy-util[config_template] pydantic>=2,!=2.6.0,!=2.6.1 PyYAML - Jinja2 packages = find: python_requires = >=3.7 diff --git a/packages/test.sh b/packages/test.sh index 0d1d1655d941..be98e038285e 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -49,7 +49,7 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow. # Install extras (if needed) if [ "$package_dir" = "util" ]; then - pip install '.[template,jstree]' + pip install '.[template,jstree,config_template]' elif [ "$package_dir" = "tool_util" ]; then pip install '.[cwl,mulled,edam,extended-assertions]' else diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index 70add5d8311c..d0a614868fa5 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -53,6 +53,9 @@ jstree = template = Cheetah3 future>=1.0.0 +config_template = + Jinja2 + pydantic [options.packages.find] exclude = diff --git a/test/unit/app/managers/test_user_file_sources.py b/test/unit/app/managers/test_user_file_sources.py new file mode 100644 index 000000000000..2f831418a108 --- /dev/null +++ b/test/unit/app/managers/test_user_file_sources.py @@ -0,0 +1,108 @@ +from typing import ( + List, + Optional, +) + +from galaxy.files.templates import ConfiguredFileSourceTemplates +from galaxy.managers.file_source_instances import ( + CreateInstancePayload, + FileSourceInstancesManager, + USER_FILE_SOURCES_SCHEME, + UserDefinedFileSourcesConfig, + UserDefinedFileSourcesImpl, + UserFileSourceModel, +) +from galaxy.util.config_templates import RawTemplateConfig +from .base import BaseTestCase + +SIMPLE_FILE_SOURCE_NAME = "myfilesource" +SIMPLE_FILE_SOURCE_DESCRIPTION = "a description of my file source" + + +class Config: + file_source_templates: Optional[List[RawTemplateConfig]] = None + file_source_templates_config_file: Optional[str] = None + + def __init__(self, templates: List[RawTemplateConfig]): + self.file_source_templates = templates + + +def home_directory_template(tmp_path): + return { + "id": "home_directory", + "name": "Home Directory", + "description": "Your Home Directory on this System", + "configuration": { + "type": "posix", + "root": str(tmp_path / "{{ user.username }}"), + }, + } + + +class TestFileSourcesTestCase(BaseTestCase): + manager: FileSourceInstancesManager + file_sources: UserDefinedFileSourcesImpl + + def test_create_posix(self, tmp_path): + self._init_managers(tmp_path) + user_file_source = self._create_user_file_source() + uri_root = user_file_source.uri_root + match = self.file_sources.find_best_match(uri_root) + assert match + assert match.score == len(uri_root) + file_source = match.file_source + assert file_source + assert file_source.label == SIMPLE_FILE_SOURCE_NAME + assert file_source.doc == SIMPLE_FILE_SOURCE_DESCRIPTION + assert file_source.get_scheme() == USER_FILE_SOURCES_SCHEME + assert file_source.get_uri_root() == uri_root + + def test_io(self, tmp_path): + self._init_managers(tmp_path) + user_file_source = self._create_user_file_source() + uri_root = user_file_source.uri_root + match = self.file_sources.find_best_match(uri_root) + assert match + file_source = match.file_source + + temp_file = tmp_path / "tmp_file" + temp_file.write_text("Moo Cow", "utf-8") + file_source.write_from("/moo", str(temp_file)) + target = tmp_path / "round_trip" + file_source.realize_to("/moo", target) + assert target.read_text("utf-8") == "Moo Cow" + + def test_show(self, tmp_path): + self._init_managers(tmp_path) + user_file_source = self._create_user_file_source() + user_file_source_showed = self.manager.show(self.trans, user_file_source.uuid) + assert user_file_source_showed + assert user_file_source_showed.uuid == user_file_source.uuid + assert user_file_source_showed.id == user_file_source.id + + def _create_user_file_source(self) -> UserFileSourceModel: + create_payload = CreateInstancePayload( + name=SIMPLE_FILE_SOURCE_NAME, + description=SIMPLE_FILE_SOURCE_DESCRIPTION, + template_id="home_directory", + template_version=0, + variables={}, + secrets={}, + ) + user_file_source = self.manager.create_instance(self.trans, create_payload) + return user_file_source + + def _init_managers(self, tmp_path): + file_sources_config = UserDefinedFileSourcesConfig(user_object_store_index_by="uuid") + self.app[UserDefinedFileSourcesConfig] = file_sources_config + self.app.setup_test_vault() + templates_config = Config([home_directory_template(tmp_path)]) + templates = ConfiguredFileSourceTemplates.from_app_config( + templates_config, + vault_configured=True, + ) + self.app[ConfiguredFileSourceTemplates] = templates + file_sources = self.app[UserDefinedFileSourcesImpl] + manager = self.app[FileSourceInstancesManager] + self.file_sources = file_sources + self.manager = manager diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 9b23a7005f48..f5d88dd57402 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -5,6 +5,7 @@ from galaxy.files import ( ConfiguredFileSources, + ConfiguredFileSourcesConf, DictFileSourcesUserContext, OptionalUserContext, ) @@ -157,7 +158,7 @@ def write_from( def configured_file_sources(conf_file): file_sources_config = FileSourcePluginsConfig() - return ConfiguredFileSources(file_sources_config, conf_file=conf_file) + return ConfiguredFileSources(file_sources_config, ConfiguredFileSourcesConf(conf_file=conf_file)) def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False): diff --git a/test/unit/files/test_posix.py b/test/unit/files/test_posix.py index f64322db082b..a5749500c597 100644 --- a/test/unit/files/test_posix.py +++ b/test/unit/files/test_posix.py @@ -12,7 +12,10 @@ ItemAccessibilityException, RequestParameterInvalidException, ) -from galaxy.files import ConfiguredFileSources +from galaxy.files import ( + ConfiguredFileSources, + ConfiguredFileSourcesConf, +) from galaxy.files.plugins import FileSourcePluginsConfig from galaxy.files.unittest_utils import ( setup_root, @@ -159,7 +162,7 @@ def test_user_ftp_explicit_config(): "type": "gxftp", } tmp, root = setup_root() - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[plugin]) + file_sources = ConfiguredFileSources(file_sources_config, ConfiguredFileSourcesConf(conf_dict=[plugin])) user_context = user_context_fixture(user_ftp_dir=root) write_file_fixtures(tmp, root) @@ -181,7 +184,9 @@ def test_user_ftp_implicit_config(): ftp_upload_dir=root, ftp_upload_purge=False, ) - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[], load_stock_plugins=True) + file_sources = ConfiguredFileSources( + file_sources_config, ConfiguredFileSourcesConf(conf_dict=[]), load_stock_plugins=True + ) user_context = user_context_fixture(user_ftp_dir=root) write_file_fixtures(tmp, root) assert os.path.exists(os.path.join(root, "a")) @@ -199,7 +204,9 @@ def test_user_ftp_respects_upload_purge_off(): ftp_upload_dir=root, ftp_upload_purge=True, ) - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[], load_stock_plugins=True) + file_sources = ConfiguredFileSources( + file_sources_config, ConfiguredFileSourcesConf(conf_dict=[]), load_stock_plugins=True + ) user_context = user_context_fixture(user_ftp_dir=root) write_file_fixtures(tmp, root) assert_realizes_as(file_sources, "gxftp://a", "a\n", user_context=user_context) @@ -211,7 +218,9 @@ def test_user_ftp_respects_upload_purge_on_by_default(): file_sources_config = FileSourcePluginsConfig( ftp_upload_dir=root, ) - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[], load_stock_plugins=True) + file_sources = ConfiguredFileSources( + file_sources_config, ConfiguredFileSourcesConf(conf_dict=[]), load_stock_plugins=True + ) user_context = user_context_fixture(user_ftp_dir=root) write_file_fixtures(tmp, root) assert_realizes_as(file_sources, "gxftp://a", "a\n", user_context=user_context) @@ -226,7 +235,7 @@ def test_import_dir_explicit_config(): plugin = { "type": "gximport", } - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[plugin]) + file_sources = ConfiguredFileSources(file_sources_config, ConfiguredFileSourcesConf(conf_dict=[plugin])) write_file_fixtures(tmp, root) assert_realizes_as(file_sources, "gximport://a", "a\n") @@ -237,7 +246,9 @@ def test_import_dir_implicit_config(): file_sources_config = FileSourcePluginsConfig( library_import_dir=root, ) - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[], load_stock_plugins=True) + file_sources = ConfiguredFileSources( + file_sources_config, ConfiguredFileSourcesConf(conf_dict=[]), load_stock_plugins=True + ) write_file_fixtures(tmp, root) assert_realizes_as(file_sources, "gximport://a", "a\n") @@ -248,7 +259,9 @@ def test_user_import_dir_implicit_config(): file_sources_config = FileSourcePluginsConfig( user_library_import_dir=root, ) - file_sources = ConfiguredFileSources(file_sources_config, conf_dict=[], load_stock_plugins=True) + file_sources = ConfiguredFileSources( + file_sources_config, ConfiguredFileSourcesConf(conf_dict=[]), load_stock_plugins=True + ) write_file_fixtures(tmp, os.path.join(root, EMAIL)) diff --git a/test/unit/files/test_template_manager.py b/test/unit/files/test_template_manager.py new file mode 100644 index 000000000000..5c89405b0c8b --- /dev/null +++ b/test/unit/files/test_template_manager.py @@ -0,0 +1,58 @@ +from galaxy.files.templates import ConfiguredFileSourceTemplates +from .test_template_models import ( + LIBRARY_AWS, + LIBRARY_HOME_DIRECTORY, +) + + +class MockConfig: + def __init__(self, config_path): + self.file_source_templates = None + self.file_source_templates_config_file = config_path + + +def test_manager(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_HOME_DIRECTORY, "utf-8") + config = MockConfig(config_path) + templates = ConfiguredFileSourceTemplates.from_app_config(config) + summaries = templates.summaries + assert summaries + assert len(summaries.root) == 1 + + +def test_manager_throws_exception_if_vault_is_required_but_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AWS, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc, "catalog creation should result in an exception" + assert "vault must be configured" in str(exc) + + +def test_manager_with_secrets_is_fine_if_vault_is_required_and_configured(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_AWS, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=True) + except Exception as e: + exc = e + assert exc is None + + +def test_manager_does_not_throw_exception_if_vault_is_not_required(tmpdir): + config_path = tmpdir / "conf.yml" + config_path.write_text(LIBRARY_HOME_DIRECTORY, "utf-8") + config = MockConfig(config_path) + exc = None + try: + ConfiguredFileSourceTemplates.from_app_config(config, vault_configured=False) + except Exception as e: + exc = e + assert exc is None diff --git a/test/unit/files/test_template_models.py b/test/unit/files/test_template_models.py new file mode 100644 index 000000000000..eccfaf073e42 --- /dev/null +++ b/test/unit/files/test_template_models.py @@ -0,0 +1,91 @@ +from yaml import safe_load + +from galaxy.files.templates.manager import raw_config_to_catalog +from galaxy.files.templates.models import ( + FileSourceTemplate, + FileSourceTemplateCatalog, + PosixFileSourceConfiguration, + S3FSFileSourceConfiguration, + template_to_configuration, +) + +LIBRARY_AWS = """ +- id: aws_bucket + name: Amazon Bucket + description: An Amazon S3 Bucket + variables: + bucket_name: + type: string + help: Name of bucket to use when connecting to AWS resources. + secrets: + access_key: + help: AWS access key to use when connecting to AWS resources. + secret_key: + help: AWS secret key to use when connecting to AWS resources. + configuration: + type: s3fs + key: '{{ secrets.access_key}}' + secret: '{{ secrets.secret_key}}' + bucket: '{{ variables.bucket_name}}' +""" + + +def test_aws_s3_config(): + template_library = _parse_template_library(LIBRARY_AWS) + s3_template = _assert_has_one_template(template_library) + assert s3_template.description == "An Amazon S3 Bucket" + configuration_obj = template_to_configuration( + s3_template, + {"bucket_name": "sec3"}, + {"access_key": "sec1", "secret_key": "sec2"}, + user_details={}, + ) + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, S3FSFileSourceConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "s3fs" + assert configuration["key"] == "sec1" + assert configuration["secret"] == "sec2" + assert configuration["bucket"] == "sec3" + + +LIBRARY_HOME_DIRECTORY = """ +- id: home_directory + name: Home Directory + description: Your Home Directory on this System + configuration: + type: posix + root: "/home/{{ user.username}}/" +""" + + +def test_a_posix_template(): + template_library = _parse_template_library(LIBRARY_HOME_DIRECTORY) + posix_template = _assert_has_one_template(template_library) + assert posix_template.description == "Your Home Directory on this System" + configuration_obj = template_to_configuration( + posix_template, + {}, + {}, + user_details={"username": "foobar"}, + ) + + # expanded configuration should validate with template expansions... + assert isinstance(configuration_obj, PosixFileSourceConfiguration) + configuration = configuration_obj.model_dump() + + assert configuration["type"] == "posix" + assert configuration["root"] == "/home/foobar/" + + +def _assert_has_one_template(catalog: FileSourceTemplateCatalog) -> FileSourceTemplate: + assert len(catalog.root) == 1 + template = catalog.root[0] + return template + + +def _parse_template_library(contents: str) -> FileSourceTemplateCatalog: + raw_contents = safe_load(contents) + return raw_config_to_catalog(raw_contents) diff --git a/test/unit/objectstore/test_template_models.py b/test/unit/objectstore/test_template_models.py index 505f9b3e8d99..f87a29d58f3c 100644 --- a/test/unit/objectstore/test_template_models.py +++ b/test/unit/objectstore/test_template_models.py @@ -1,3 +1,5 @@ +import os + from yaml import safe_load from galaxy.objectstore.templates.manager import raw_config_to_catalog @@ -166,6 +168,74 @@ def test_parsing_multiple_posix(): assert secure_configuration.files_dir == "/data/secure/jane" +LIBRARY_WITH_PATH_PARAMETER = """ +- id: path_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/{{ user.username | ensure_path_component }}//{{ variables.project_name | ensure_path_component }}' + variables: + project_name: + type: string # dont do this in practice - use path_component for more eager validation + help: Project name used in path for this template library. +""" + + +def test_parsing_with_path_security(): + template_library = _parse_template_library(LIBRARY_WITH_PATH_PARAMETER) + assert len(template_library.root) == 1 + path_template = template_library.root[0] + + assert path_template.version == 0 + + user_details = {"username": "jane"} + variables = {"project_name": "moo"} + + general_configuration = template_to_configuration(path_template, variables, {}, user_details=user_details) + assert isinstance(general_configuration, DiskObjectStoreConfiguration) + assert os.path.abspath(general_configuration.files_dir) == "/data/general/jane/moo" + + variables = {"project_name": "../moo"} + exc = None + try: + template_to_configuration(path_template, variables, {}, user_details=user_details) + except Exception as e: + exc = e + assert exc is not None + + +LIBRARY_WITH_CUSTOM_TEMPLATE_START_END = """ +- id: path_disk + name: General Disk + description: General Disk Bound to You + configuration: + type: disk + files_dir: '/data/general/@= user.username | ensure_path_component =@//@= variables.project_name | ensure_path_component =@' + template_start: '@=' + template_end: '=@' + variables: + project_name: + type: string # dont do this in practice - use path_component for more eager validation + help: Project name used in path for this template library. +""" + + +def test_custom_template_start_and_ends(): + template_library = _parse_template_library(LIBRARY_WITH_CUSTOM_TEMPLATE_START_END) + assert len(template_library.root) == 1 + path_template = template_library.root[0] + + assert path_template.version == 0 + + user_details = {"username": "jane"} + variables = {"project_name": "moo"} + + general_configuration = template_to_configuration(path_template, variables, {}, user_details=user_details) + assert isinstance(general_configuration, DiskObjectStoreConfiguration) + assert os.path.abspath(general_configuration.files_dir) == "/data/general/jane/moo" + + LIBRARY_AZURE_CONTAINER = """ - id: amazon_bucket name: Azure Container diff --git a/test/unit/util/test_config_template_validation.py b/test/unit/util/test_config_template_validation.py new file mode 100644 index 000000000000..74da67525c78 --- /dev/null +++ b/test/unit/util/test_config_template_validation.py @@ -0,0 +1,131 @@ +from typing import ( + Any, + Dict, + List, + Optional, +) + +from galaxy.exceptions import ( + RequestParameterInvalidException, + RequestParameterMissingException, +) +from galaxy.util.config_templates import ( + StrictModel, + TemplateSecret, + TemplateVariable, + validate_secrets_and_variables, +) + +TEST_TEMPLATE_ID = "test_id" +TEST_TEMPLATE_VERSION = 0 + + +class TestTemplate(StrictModel): + id: str + version: int + variables: Optional[List[TemplateVariable]] + secrets: Optional[List[TemplateSecret]] + + +def _template_with_variable(variable: TemplateVariable) -> TestTemplate: + return TestTemplate( + id=TEST_TEMPLATE_ID, + version=TEST_TEMPLATE_VERSION, + variables=[variable], + secrets=None, + ) + + +class TestInstanceDefinition(StrictModel): + template_id: str + template_version: int + variables: Dict[str, Any] + secrets: Dict[str, str] + + +def _test_instance_with_variables(variables: Dict[str, Any]) -> TestInstanceDefinition: + return TestInstanceDefinition( + template_id=TEST_TEMPLATE_ID, + template_version=TEST_TEMPLATE_VERSION, + variables=variables, + secrets={}, + ) + + +def test_variable_typing_string(): + template = _template_with_variable(TemplateVariable(name="test_var", help=None, type="string")) + instance = _test_instance_with_variables({"test_var": "moocow"}) + validate_secrets_and_variables(instance, template) + + instance = _test_instance_with_variables({"test_var": 5}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": False}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + +def test_variable_typing_boolean(): + template = _template_with_variable(TemplateVariable(name="test_var", help=None, type="boolean")) + instance = _test_instance_with_variables({"test_var": False}) + validate_secrets_and_variables(instance, template) + + instance = _test_instance_with_variables({"test_var": 0}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": "false"}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + +def test_variable_typing_int(): + template = _template_with_variable(TemplateVariable(name="test_var", help=None, type="integer")) + instance = _test_instance_with_variables({"test_var": 6}) + validate_secrets_and_variables(instance, template) + + instance = _test_instance_with_variables({"test_var": False}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": "six"}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + +def test_variable_typing_path_component(): + template = _template_with_variable(TemplateVariable(name="test_var", help=None, type="path_component")) + instance = _test_instance_with_variables({"test_var": "simple_directory"}) + validate_secrets_and_variables(instance, template) + + instance = _test_instance_with_variables({"test_var": 0}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": False}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": "../simple_directory"}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + instance = _test_instance_with_variables({"test_var": "/simple_directory"}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterInvalidException) + + +def test_variable_missing(): + template = _template_with_variable(TemplateVariable(name="test_var", help=None, type="path_component")) + instance = _test_instance_with_variables({}) + e = assert_validation_throws(instance, template) + assert isinstance(e, RequestParameterMissingException) + + +def assert_validation_throws(instance: TestInstanceDefinition, template: TestTemplate) -> Exception: + try: + validate_secrets_and_variables(instance, template) + except Exception as e: + return e + raise AssertionError("Expected validation error did not occur.")
+ {{ selectText }} +
- {{ selectText }} -