Allow oauth2 file sources with Dropbox initial implementation.

client/src/components/ConfigTemplates/ConfigurationTestSummary.vue

@@ -15,6 +15,9 @@ defineProps<Props>();
 <template>
     <BListGroup v-if="testResults">
         <ConfigurationTestItem :status="testResults?.template_definition" />
+        <ConfigurationTestItem
+            v-if="testResults?.oauth2_access_token_generation != null"
+            :status="testResults?.oauth2_access_token_generation" />
         <ConfigurationTestItem :status="testResults?.template_settings" />
         <ConfigurationTestItem :status="testResults?.connection" />
     </BListGroup>

client/src/components/ConfigTemplates/useConfigurationTesting.ts

@@ -43,6 +43,7 @@ export function useConfigurationTesting() {
 export function useConfigurationTemplateCreation<T extends TemplateSummary, R>(
     what: string,
     template: Ref<T>,
+    uuid: Ref<string | undefined>,
     test: (payload: CreateInstancePayload) => Promise<{ data: PluginStatus }>,
     create: (payload: CreateInstancePayload) => Promise<{ data: R }>,
     onCreate: (result: R) => unknown
@@ -52,6 +53,9 @@ export function useConfigurationTemplateCreation<T extends TemplateSummary, R>(
 
     async function onSubmit(formData: any) {
         const payload = createFormDataToPayload(template.value, formData);
+        if (uuid.value) {
+            payload.uuid = uuid.value;
+        }
         let pluginStatus;
         try {
             testRunning.value = true;

client/src/components/FileSources/Instances/CreateForm.vue

@@ -8,6 +8,7 @@ import { create, test } from "./services";
 
 interface CreateFormProps {
     template: FileSourceTemplateSummary;
+    uuid?: string;
 }
 const props = defineProps<CreateFormProps>();
 const title = "Create a new file source for your data";
@@ -20,6 +21,7 @@ const { ActionSummary, error, inputs, InstanceForm, onSubmit, submitTitle, loadi
     useConfigurationTemplateCreation(
         "file source",
         toRef(props, "template"),
+        toRef(props, "uuid"),
         test,
         create,
         (fileSource: UserFileSourceModel) => emit("created", fileSource)

client/src/components/FileSources/Instances/CreateInstance.vue

@@ -1,34 +1,63 @@
 <script setup lang="ts">
-import { computed } from "vue";
+import { computed, watch } from "vue";
 
 import type { UserFileSourceModel } from "@/api/fileSources";
 import { useFileSourceTemplatesStore } from "@/stores/fileSourceTemplatesStore";
 
 import { useInstanceRouting } from "./routing";
+import { getOAuth2Info } from "./services";
 
 import CreateForm from "@/components/FileSources/Instances/CreateForm.vue";
 import LoadingSpan from "@/components/LoadingSpan.vue";
 
 interface Props {
     templateId: string;
+    uuid?: string;
 }
+
+const OAUTH2_TYPES = ["dropbox", "googledrive"];
+
 const fileSourceTemplatesStore = useFileSourceTemplatesStore();
 fileSourceTemplatesStore.fetchTemplates();
 
 const { goToIndex } = useInstanceRouting();
 
 const props = defineProps<Props>();
 const template = computed(() => fileSourceTemplatesStore.getLatestTemplate(props.templateId));
+const requiresOAuth2AuthorizeRedirect = computed(() => {
+    const templateValue = template.value;
+    return props.uuid == undefined && templateValue && OAUTH2_TYPES.indexOf(templateValue.type) >= 0;
+});
 
 async function onCreated(objectStore: UserFileSourceModel) {
     const message = `Created file source ${objectStore.name}`;
     goToIndex({ message });
 }
+
+watch(
+    requiresOAuth2AuthorizeRedirect,
+    async (requiresAuth) => {
+        const templateValue = template.value;
+        if (templateValue && requiresAuth) {
+            const { data } = await getOAuth2Info({
+                template_id: templateValue.id,
+                template_version: templateValue.version || 0,
+            });
+            window.location.href = data.authorize_url;
+        } else {
+            console.log("skipping this...");
+        }
+    },
+    { immediate: true }
+);
 </script>
 
 <template>
     <div>
         <LoadingSpan v-if="!template" message="Loading file source templates" />
-        <CreateForm v-else :template="template" @created="onCreated"></CreateForm>
+        <LoadingSpan
+            v-else-if="requiresOAuth2AuthorizeRedirect"
+            message="Fetching redirect information, you will need to authorize Galaxy to have access to this resource remotely" />
+        <CreateForm v-else :uuid="uuid" :template="template" @created="onCreated"></CreateForm>
     </div>
 </template>

client/src/components/FileSources/Instances/services.ts

@@ -9,6 +9,10 @@ export const testInstance = fetcher
     .create();
 export const update = fetcher.path("/api/file_source_instances/{user_file_source_id}").method("put").create();
 export const testUpdate = fetcher.path("/api/file_source_instances/{user_file_source_id}/test").method("post").create();
+export const getOAuth2Info = fetcher
+    .path("/api/file_source_templates/{template_id}/{template_version}/oauth2")
+    .method("get")
+    .create();
 
 export async function hide(instance: UserFileSourceModel) {
     const payload = { hidden: true };

client/src/components/ObjectStore/Instances/CreateForm.vue

@@ -9,6 +9,7 @@ import { create, test } from "./services";
 
 interface CreateFormProps {
     template: ObjectStoreTemplateSummary;
+    uuid?: string;
 }
 const props = defineProps<CreateFormProps>();
 const title = "Create a new storage location for your data";
@@ -21,6 +22,7 @@ const { ActionSummary, error, inputs, InstanceForm, onSubmit, submitTitle, loadi
     useConfigurationTemplateCreation(
         "storage location",
         toRef(props, "template"),
+        toRef(props, "uuid"),
         test,
         create,
         (instance: UserConcreteObjectStore) => emit("created", instance)

client/src/entry/analysis/router.js

@@ -414,7 +414,10 @@ export function getRouter(Galaxy) {
                     {
                         path: "file_source_templates/:templateId/new",
                         component: CreateFileSourceInstance,
-                        props: true,
+                        props: (route) => ({
+                            templateId: route.params.templateId,
+                            uuid: route.query.uuid,
+                        }),
                     },
                     {
                         path: "pages/create",

doc/source/admin/data.md

@@ -350,6 +350,19 @@ configuration).
 
 ![](file_source_azure_configuration.png)
 
+#### ``dropbox``
+
+The syntax for the ``configuration`` section of ``dropbox`` templates looks like this.
+
+![](file_source_dropbox_configuration_template.png)
+
+At runtime, after the ``configuration`` template is expanded, the resulting dictionary
+passed to Galaxy's file source plugin infrastructure looks like this and should match a subset
+of what you'd be able to add directly to ``file_sources_conf.yml`` (Galaxy's global file source
+configuration).
+
+![](file_source_dropbox_configuration.png)
+
 ### YAML Syntax
 
 ![galaxy.files.templates.models](file_source_templates.png)
@@ -398,6 +411,74 @@ and you are comfortable with it storing your user's secrets.
 :language: yaml
 ```
 
+### Production OAuth 2.0 File Source Templates
+
+Unlike the examples in the previous section. These examples will require a bit of
+configuration on the part of the admin. This is to obtain client credentials from the
+external service and register an OAuth 2.0 redirection callback with the remote service.
+
+#### Dropbox
+
+Once you have OAuth 2.0 client credentials from Dropbox (called ``oauth2_client_id`` and
+``oauth2_client_secret`` here), the following configuration can be used configure your Galaxy
+instance to enable Dropbox.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_dropbox.yml
+:language: yaml
+```
+
+To use this template - you'll need to make your credentials available to Galaxy's
+web and job handler processes using the environment variables ``GALAXY_DROPBOX_APP_CLIENT_ID``
+and ``GALAXY_DROPBOX_APP_CLIENT_SECRET``. Your jobs themselves do not require
+these secrets to be set and will not be given the secrets.
+
+If you'd like to configure these secrets explicit - you can configure them explicitly
+in the configuration. If your configuration file is managed by Ansible, these secrets
+could potentially be populated from your Ansible vault.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_explicit.yml
+:language: yaml
+```
+
+To obtain the OAuth 2.0 credentials from Dropbox, you'll need to navigate to your
+[Dropbox Apps](https://www.dropbox.com/developers/apps) and create a new app 
+for your Galaxy instance with the "Create app" button.
+
+![Screenshot of Dropbox](dropbox_create_button.png)
+
+The only option available is "Scoped access" and this works fine for typical
+Galaxy use cases. You will however want to click "Full Dropbox" to request
+full access to your user's account. You will also need to give your "App" a name
+here, this should likely be something related to your Galaxy instances name.
+
+![Screenshot of Dropbox](dropbox_create_app_options_1.png)
+
+After your app is created, you'll be presented with a management screen for it.
+The first thing you'll want to do is navigate to the permissions tab and enable
+permissions to read and write to files and directories so the file source plugin
+works properly:
+
+![Screenshot of Dropbox](dropbox_scopes.png)
+
+Next, navigate back to the "Settings" tab. You'll need to register a callback
+for your Galaxy instance (it will need HTTPS enabled). This should be the URL
+to your Galaxy instance with ``oauth2_callback`` appended to it.
+
+![Screenshot of Dropbox](dropbox_callback.png)
+
+Finally you'll be able to find the ``oauth2_client_id`` and ``oauth2_client_secret``
+to configured your Galaxy with on this settings page.
+
+![Screenshot of Dropbox](dropbox_client_creds.png)
+
+Until you have 50 users, your App will be considered a "development" application.
+The upshot of this is that your user's will get a scary message during authorization
+but there seems to be no way around this. 50 users would definitely be considered
+a production Galaxy instance but Dropbox operates on a different scale.
+
+For more information on what Dropbox considers a "development" app versus a "production"
+app - checkout the [Dropbox documentation](https://www.dropbox.com/developers/reference/developer-guide#production-approval).
+
 ## Playing Nicer with Ansible
 
 Many large instances of Galaxy are configured with Ansible and much of the existing administrator
@@ -604,3 +685,70 @@ variable is not defined at runtime.
 ```{literalinclude} ../../../lib/galaxy/files/templates/examples/admin_secrets_with_defaults.yml
 :language: yaml
 ```
+
+## OAuth 2.0 Enabled Configurations
+
+[OAuth 2.0](https://oauth.net/2/) has become an industry standard for allowing users
+of various services (e.g. Dropbox or Google Drive) to authorize other services (e.g. Galaxy)
+fine grained access to the services. There is a bit of a dance the services need to do
+but the result can be a fairly nice end-user experience. The framework for configuring
+user defined data access templates can support OAuth 2.0.
+
+Galaxy keeps track of which plugin ``type``s (currently only file source types) require
+OAuth2 to work properly and will take care of authorization redirection, saving refresh tokens,
+etc.. implicitly. One such ``type`` is ``dropbox``. Here is the production Dropbox
+template distributed with Galaxy.
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_dropbox.yml
+:language: yaml
+```
+
+OAuth2 enabled plugin types include template definitions that include ``oauth2_client_id``
+and ``oauth2_client_secret`` in the configuration (as shown in the following specification
+and in the above examples).
+
+![](file_source_dropbox_configuration_template.png)
+
+The above example defines these secrets using environment variables but they can stored in
+Galaxy's Vault explicitly by the admin or written right to the configuration files as shown
+in the next two examples:
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_in_vault.yml
+:language: yaml
+```
+
+```{literalinclude} ../../../lib/galaxy/files/templates/examples/dropbox_client_secrets_explicit.yml
+:language: yaml
+```
+
+Looking at the configuration objects that get generated at runtime
+from these templates though - ``oauth2_client_id`` and ``oauth2_client_secret`` no longer
+appear and instead have been replaced with a ``oauth2_access_token`` parameter.
+Galaxy will take care of stripping out the client (e.g. Galaxy server) information and
+replacing it with short-term access tokens generated for the user's resources.
+
+![](file_source_dropbox_configuration.png)
+
+Normally, a UUID is created for each user configured instance object and this is used
+to store the template's explicitly listed secrets in Galaxy's Vault. For OAuth 2.0
+plugin types - before user's are even prompted for configuration metadata they are redirected
+to the remote service and prompted to authorize Galaxy to act on their behalf when using
+the remote service. If they authorize this, the remote service will send an [`authorization
+code`](https://oauth.net/2/grant-types/authorization-code/) to ``https://<galaxy_url>/oauth2_callback`` along with state information
+to recover which instance is being configured. At this point, Galaxy will fetch a [`refresh token`](https://oauth.net/2/refresh-tokens/) from the remote resource using the
+supplied authorization code. The refresh token is stored in the Vault in key associated with
+the UUID of the object that will be created when the user finishes the creation process.
+Specifically it is stored at 
+
+```
+/galaxy/user/<user_id>/file_source_config/<file_source_instance_uuid>/_oauth2_refresh_token
+```
+
+Here is the prefix at the end of ``_`` is indicating that Galaxy is managing this instead of
+it being listed explicitly in a ``secrets`` section of the template configuration like the
+explicit Vault secrets discussed in this document.
+
+Galaxy knows how to fetch an [`access token`](https://oauth.net/2/access-tokens/) from this
+refresh token that is actually used to interact with the remote resource. This is the property
+``oauth2_access_token`` that is injected into the configuration object shown above and passed
+along to the actual object store or file source plugin implementation.

doc/source/admin/gen_diagrams.py

@@ -8,6 +8,8 @@
 from galaxy.files.templates.models import (
     AzureFileSourceConfiguration,
     AzureFileSourceTemplateConfiguration,
+    DropboxFileSourceConfiguration,
+    DropboxFileSourceTemplateConfiguration,
     FileSourceTemplate,
     FtpFileSourceConfiguration,
     FtpFileSourceTemplateConfiguration,
@@ -47,6 +49,8 @@
     FileSourceTemplate: "file_source_templates",
     AzureFileSourceTemplateConfiguration: "file_source_azure_configuration_template",
     AzureFileSourceConfiguration: "file_source_azure_configuration",
+    DropboxFileSourceTemplateConfiguration: "file_source_dropbox_configuration_template",
+    DropboxFileSourceConfiguration: "file_source_dropbox_configuration",
     PosixFileSourceTemplateConfiguration: "file_source_posix_configuration_template",
     PosixFileSourceConfiguration: "file_source_posix_configuration",
     S3FSFileSourceTemplateConfiguration: "file_source_s3fs_configuration_template",

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -329,7 +329,7 @@ def __init__(self, app=None, user=None, history=None, **kwargs):
         self.security = self.app.security
         self.history = history
 
-        self.request: Any = Bunch(headers={}, is_body_readable=False, host="request.host")
+        self.request: Any = Bunch(headers={}, is_body_readable=False, host="request.host", url_path="mock/url/path")
         self.response: Any = Bunch(headers={}, set_content_type=lambda i: None)
 
     @property

lib/galaxy/files/sources/dropbox.py

@@ -30,6 +30,8 @@ def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None)
         # accessToken has been renamed to access_token in fs.dropboxfs 1.0
         if "accessToken" in props:
             props["access_token"] = props.pop("accessToken")
+        if "oauth2_access_token" in props:
+            props["access_token"] = props.pop("oauth2_access_token")
 
         try:
             handle = DropboxFS(**{**props, **extra_props})

lib/galaxy/files/sources/googledrive.py

@@ -17,6 +17,9 @@ class GoogleDriveFilesSource(PyFilesystem2FilesSource):
 
     def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None):
         props = self._serialization_props(user_context)
+        access_token = props.pop("oauth2_access_token")
+        if access_token:
+            props["token"] = access_token
         credentials = Credentials(**props)
         handle = GoogleDriveFS(credentials)
         return handle

lib/galaxy/files/templates/__init__.py

@@ -4,6 +4,8 @@
     FileSourceTemplate,
     FileSourceTemplateSummaries,
     FileSourceTemplateType,
+    get_oauth2_config,
+    get_oauth2_config_or_none,
     template_to_configuration,
 )
 
@@ -13,5 +15,7 @@
     "FileSourceTemplate",
     "FileSourceTemplateSummaries",
     "FileSourceTemplateType",
+    "get_oauth2_config",
+    "get_oauth2_config_or_none",
     "template_to_configuration",
 )

lib/galaxy/files/templates/examples/dropbox_client_secrets_explicit.yml

@@ -0,0 +1,7 @@
+- id: dropbox
+  name: Dropbox
+  description: Connect to your Dropbox account to download and upload files.
+  configuration:
+    type: dropbox
+    oauth2_client_id: abcdefgh
+    oauth2_client_secret: ijklmnopqr