Skip to content

Commit

Permalink
WIP: file source templates
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmchilton
jmchilton committed Apr 22, 2024
1 parent 4091d5b commit 4c9773c
Show file tree
Hide file tree
Showing 25 changed files with 1,651 additions and 210 deletions.
250 changes: 229 additions & 21 deletions client/src/api/schema/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,22 @@ export interface paths {
/** Download */
get: operations["download_api_drs_download__object_id__get"];
};
"/api/file_source_instances": {
/** Get a list of persisted file source instances defined by the requesting user. */
get: operations["file_sources__instances_index"];
/** Create a user-bound object store. */
post: operations["file_sources__create_instance"];
};
"/api/file_source_instances/{user_file_source_id}": {
/** Get a list of persisted file source instances defined by the requesting user. */
get: operations["file_sources__instances_get"];
/** Update or upgrade user file source instance. */
put: operations["file_sources__instances_update"];
};
"/api/file_source_templates": {
/** Get a list of file source templates available to build user defined file sources from */
get: operations["file_sources__templates_index"];
};
"/api/folders/{folder_id}/contents": {
/**
* Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder.
Expand Down Expand Up @@ -5276,6 +5292,36 @@ export interface components {
*/
update_time: string;
};
/** FileSourceTemplateSummaries */
FileSourceTemplateSummaries: components["schemas"]["FileSourceTemplateSummary"][];
/** FileSourceTemplateSummary */
FileSourceTemplateSummary: {
/** Description */
description: string | null;
/**
* Hidden
* @default false
*/
hidden?: boolean;
/** Id */
id: string;
/** Name */
name: string | null;
/** Secrets */
secrets?: components["schemas"]["TemplateSecret"][] | null;
/**
* Type
* @enum {string}
*/
type: "posix" | "s3fs";
/** Variables */
variables?: components["schemas"]["TemplateVariable"][] | null;
/**
* Version
* @default 0
*/
version?: number;
};
/** FilesSourcePlugin */
FilesSourcePlugin: {
/**
Expand Down Expand Up @@ -9955,13 +10001,6 @@ export interface components {
*/
up_to_date: boolean;
};
/** ObjectStoreTemplateSecret */
ObjectStoreTemplateSecret: {
/** Help */
help: string | null;
/** Name */
name: string;
};
/** ObjectStoreTemplateSummaries */
ObjectStoreTemplateSummaries: components["schemas"]["ObjectStoreTemplateSummary"][];
/** ObjectStoreTemplateSummary */
Expand All @@ -9980,32 +10019,20 @@ export interface components {
/** Name */
name: string | null;
/** Secrets */
secrets?: components["schemas"]["ObjectStoreTemplateSecret"][] | null;
secrets?: components["schemas"]["TemplateSecret"][] | null;
/**
* Type
* @enum {string}
*/
type: "s3" | "azure_blob" | "disk" | "generic_s3";
/** Variables */
variables?: components["schemas"]["ObjectStoreTemplateVariable"][] | null;
variables?: components["schemas"]["TemplateVariable"][] | null;
/**
* Version
* @default 0
*/
version?: number;
};
/** ObjectStoreTemplateVariable */
ObjectStoreTemplateVariable: {
/** Help */
help: string | null;
/** Name */
name: string;
/**
* Type
* @enum {string}
*/
type: "string" | "boolean" | "integer";
};
/** OutputReferenceByLabel */
OutputReferenceByLabel: {
/**
Expand Down Expand Up @@ -11882,6 +11909,25 @@ export interface components {
* @enum {string}
*/
TaskState: "PENDING" | "STARTED" | "RETRY" | "FAILURE" | "SUCCESS";
/** TemplateSecret */
TemplateSecret: {
/** Help */
help: string | null;
/** Name */
name: string;
};
/** TemplateVariable */
TemplateVariable: {
/** Help */
help: string | null;
/** Name */
name: string;
/**
* Type
* @enum {string}
*/
type: "string" | "boolean" | "integer";
};
/** ToolDataDetails */
ToolDataDetails: {
/**
Expand Down Expand Up @@ -12610,6 +12656,32 @@ export interface components {
*/
id: string;
};
/** UserFileSourceModel */
UserFileSourceModel: {
/** Description */
description: string | null;
/** Id */
id: number;
/** Name */
name: string;
/** Secrets */
secrets: string[];
/** Template Id */
template_id: string;
/** Template Version */
template_version: number;
/**
* Type
* @enum {string}
*/
type: "posix" | "s3fs";
/** Uuid */
uuid: string;
/** Variables */
variables: {
[key: string]: (string | boolean | number) | undefined;
} | null;
};
/**
* UserModel
* @description User in a transaction context.
Expand Down Expand Up @@ -14799,6 +14871,142 @@ export interface operations {
};
};
};
file_sources__instances_index: {
/** Get a list of persisted file source instances defined by the requesting user. */
parameters?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json": components["schemas"]["UserFileSourceModel"][];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
file_sources__create_instance: {
/** Create a user-bound object store. */
parameters?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["CreateInstancePayload"];
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json": components["schemas"]["UserFileSourceModel"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
file_sources__instances_get: {
/** Get a list of persisted file source instances defined by the requesting user. */
parameters: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
/** @description The index for a persisted UserFileSourceStore object. */
path: {
user_file_source_id: string;
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json": components["schemas"]["UserFileSourceModel"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
file_sources__instances_update: {
/** Update or upgrade user file source instance. */
parameters: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
/** @description The index for a persisted UserFileSourceStore object. */
path: {
user_file_source_id: string;
};
};
requestBody: {
content: {
"application/json":
| components["schemas"]["UpdateInstanceSecretPayload"]
| components["schemas"]["UpgradeInstancePayload"]
| components["schemas"]["UpdateInstancePayload"];
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json": components["schemas"]["UserFileSourceModel"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
file_sources__templates_index: {
/** Get a list of file source templates available to build user defined file sources from */
parameters?: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
};
responses: {
/** @description A list of the configured file source templates. */
200: {
content: {
"application/json": components["schemas"]["FileSourceTemplateSummaries"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
index_api_folders__folder_id__contents_get: {
/**
* Returns a list of a folder's contents (files and sub-folders) with additional metadata about the folder.
Expand Down
39 changes: 36 additions & 3 deletions lib/galaxy/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,27 @@
)
from galaxy.config_watchers import ConfigWatchers
from galaxy.datatypes.registry import Registry
from galaxy.files import ConfiguredFileSources
from galaxy.files import (
ConfiguredFileSources,
ConfiguredFileSourcesConf,
UserDefinedFileSources,
)
from galaxy.files.plugins import (
FileSourcePluginLoader,
FileSourcePluginsConfig,
)
from galaxy.files.templates import ConfiguredFileSourceTemplates
from galaxy.job_metrics import JobMetrics
from galaxy.jobs.manager import JobManager
from galaxy.managers.api_keys import ApiKeyManager
from galaxy.managers.citations import CitationsManager
from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.dbkeys import GenomeBuilds
from galaxy.managers.file_source_instances import (
FileSourceInstancesManager,
UserDefinedFileSourcesConfig,
UserDefinedFileSourcesImpl,
)
from galaxy.managers.folders import FolderManager
from galaxy.managers.hdas import HDAManager
from galaxy.managers.histories import HistoryManager
Expand Down Expand Up @@ -585,9 +599,28 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl
)

# ConfiguredFileSources
self.file_sources = self._register_singleton(
ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)
vault_configured = is_vault_configured(self.vault)
file_sources_config: FileSourcePluginsConfig = FileSourcePluginsConfig.from_app_config(self.config)
self._register_singleton(FileSourcePluginsConfig, file_sources_config)
file_source_plugin_loader = FileSourcePluginLoader()
self._register_singleton(FileSourcePluginLoader, file_source_plugin_loader)
self._register_singleton(
UserDefinedFileSourcesConfig, UserDefinedFileSourcesConfig.from_app_config(self.config)
)
self._register_abstract_singleton(
UserDefinedFileSources, UserDefinedFileSourcesImpl # type: ignore[type-abstract] # https://github.com/python/mypy/issues/4717
)
templates = ConfiguredFileSourceTemplates.from_app_config(self.config, vault_configured=vault_configured)
self.file_source_templates = self._register_singleton(ConfiguredFileSourceTemplates, templates)
configured_file_source_conf: ConfiguredFileSourcesConf = ConfiguredFileSourcesConf.from_app_config(self.config)
file_sources = ConfiguredFileSources(
file_sources_config,
configured_file_source_conf,
load_stock_plugins=True,
plugin_loader=file_source_plugin_loader,
)
self.file_sources = self._register_singleton(ConfiguredFileSources, file_sources)
self._register_singleton(FileSourceInstancesManager)

# Load security policy.
self.security_agent = self.model.security_agent
Expand Down
15 changes: 15 additions & 0 deletions lib/galaxy/config/schemas/config_schema.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,21 @@ mapping:
desc: |
Configured Object Store templates embedded into Galaxy's config.
file_source_templates_config_file:
type: str
default: file_source_templates.yml
path_resolves_to: config_dir
required: false
desc: |
Configured user file source templates configuration file.
file_source_templates:
type: seq
sequence:
- type: any
desc: |
Configured user file source templates embedded into Galaxy's config.
user_object_store_index_by:
type: str
default: 'uuid'
Expand Down
Loading

0 comments on commit 4c9773c

Please sign in to comment.