From 1a81f7590f10204cdfc9954ee0be6f8556910e32 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:59:03 +0100 Subject: [PATCH 1/4] Update docs for webdav file source config Include docs and settings to avoid loading entire files into memory. --- lib/galaxy/config/sample/file_sources_conf.yml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index f057b6e6b8a2..04156f44404d 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -12,6 +12,9 @@ root: ${user.preferences['owncloud|root']} login: ${user.preferences['owncloud|username']} password: ${user.preferences['owncloud|password']} + # By default, the plugin will use temp files to avoid loading entire files into memory. + # You can change the directory here or omit to use the default temp directory. + temp_path: /your/temp/path - type: posix root: '/data/5/galaxy_import/galaxy_user_data/covid-19/data/sequences/' From ddf992cbfde9e6313f3e24b1abf61ab253992d86 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:41:21 +0100 Subject: [PATCH 2/4] Add comment on writable for webdav file source --- lib/galaxy/config/sample/file_sources_conf.yml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 04156f44404d..da81ba996959 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -15,6 +15,9 @@ # By default, the plugin will use temp files to avoid loading entire files into memory. # You can change the directory here or omit to use the default temp directory. temp_path: /your/temp/path + # Set writable to true if you have write access to this source + # and want to allow exporting files to it, by default is read only. + writable: false - type: posix root: '/data/5/galaxy_import/galaxy_user_data/covid-19/data/sequences/' From 5153394f15cc4e106f19a4fb4e885d10129242c1 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:43:09 +0100 Subject: [PATCH 3/4] Set webdav plugin to use temp files by default it avoids the issue with loading the entire files into memory. --- lib/galaxy/files/sources/webdav.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index 86ed55483e70..60450dbb6c98 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -3,6 +3,7 @@ except ImportError: WebDAVFS = None +import tempfile from typing import ( Optional, Union, @@ -22,6 +23,11 @@ class WebDavFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None): props = self._serialization_props(user_context) + use_temp_files = props.pop("use_temp_files", None) + if use_temp_files is None: + # Default to True to avoid memory issues with large files. + props["use_temp_files"] = True + props["temp_path"] = props.get("temp_path", tempfile.gettempdir()) extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {} handle = WebDAVFS(**{**props, **extra_props}) return handle From 7d27634879fe123822e5e4cb405e2af0ddd6cbc7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:25:46 +0100 Subject: [PATCH 4/4] Give webdav plugin a clean temp directory Co-authored-by: mvdbeek --- lib/galaxy/files/sources/webdav.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index 60450dbb6c98..2e11a51d98b2 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -27,7 +27,7 @@ def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None) if use_temp_files is None: # Default to True to avoid memory issues with large files. props["use_temp_files"] = True - props["temp_path"] = props.get("temp_path", tempfile.gettempdir()) + props["temp_path"] = props.get("temp_path", tempfile.TemporaryDirectory(prefix="webdav_")) extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {} handle = WebDAVFS(**{**props, **extra_props}) return handle