From 1a81f7590f10204cdfc9954ee0be6f8556910e32 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:59:03 +0100 Subject: [PATCH 1/6] Update docs for webdav file source config Include docs and settings to avoid loading entire files into memory. --- lib/galaxy/config/sample/file_sources_conf.yml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index f057b6e6b8a2..04156f44404d 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -12,6 +12,9 @@ root: ${user.preferences['owncloud|root']} login: ${user.preferences['owncloud|username']} password: ${user.preferences['owncloud|password']} + # By default, the plugin will use temp files to avoid loading entire files into memory. + # You can change the directory here or omit to use the default temp directory. + temp_path: /your/temp/path - type: posix root: '/data/5/galaxy_import/galaxy_user_data/covid-19/data/sequences/' From ddf992cbfde9e6313f3e24b1abf61ab253992d86 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:41:21 +0100 Subject: [PATCH 2/6] Add comment on writable for webdav file source --- lib/galaxy/config/sample/file_sources_conf.yml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 04156f44404d..da81ba996959 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -15,6 +15,9 @@ # By default, the plugin will use temp files to avoid loading entire files into memory. # You can change the directory here or omit to use the default temp directory. temp_path: /your/temp/path + # Set writable to true if you have write access to this source + # and want to allow exporting files to it, by default is read only. + writable: false - type: posix root: '/data/5/galaxy_import/galaxy_user_data/covid-19/data/sequences/' From 5153394f15cc4e106f19a4fb4e885d10129242c1 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:43:09 +0100 Subject: [PATCH 3/6] Set webdav plugin to use temp files by default it avoids the issue with loading the entire files into memory. --- lib/galaxy/files/sources/webdav.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index 86ed55483e70..60450dbb6c98 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -3,6 +3,7 @@ except ImportError: WebDAVFS = None +import tempfile from typing import ( Optional, Union, @@ -22,6 +23,11 @@ class WebDavFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None): props = self._serialization_props(user_context) + use_temp_files = props.pop("use_temp_files", None) + if use_temp_files is None: + # Default to True to avoid memory issues with large files. + props["use_temp_files"] = True + props["temp_path"] = props.get("temp_path", tempfile.gettempdir()) extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {} handle = WebDAVFS(**{**props, **extra_props}) return handle From 7d27634879fe123822e5e4cb405e2af0ddd6cbc7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:25:46 +0100 Subject: [PATCH 4/6] Give webdav plugin a clean temp directory Co-authored-by: mvdbeek --- lib/galaxy/files/sources/webdav.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index 60450dbb6c98..2e11a51d98b2 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -27,7 +27,7 @@ def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None) if use_temp_files is None: # Default to True to avoid memory issues with large files. props["use_temp_files"] = True - props["temp_path"] = props.get("temp_path", tempfile.gettempdir()) + props["temp_path"] = props.get("temp_path", tempfile.TemporaryDirectory(prefix="webdav_")) extra_props: Union[FilesSourceProperties, dict] = opts.extra_props or {} if opts else {} handle = WebDAVFS(**{**props, **extra_props}) return handle From 10ace7bd1018ea7f0e79388605b892fcab65a369 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 6 Feb 2024 17:33:18 +0000 Subject: [PATCH 5/6] Update python-multipart to 0.0.7 xref: https://github.com/tiangolo/fastapi/security/advisories/GHSA-qf9m-vfgh-m389 --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 90333266c041..ee814dcc29ee 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -142,7 +142,7 @@ pysam==0.21.0 ; python_version >= "3.7" and python_version < "3.12" python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "3.12" python-jose==3.3.0 ; python_version >= "3.7" and python_version < "3.12" python-magic==0.4.27 ; python_version >= "3.7" and python_version < "3.12" -python-multipart==0.0.5 ; python_version >= "3.7" and python_version < "3.12" +python-multipart==0.0.7 ; python_version >= "3.7" and python_version < "3.12" python3-openid==3.2.0 ; python_version >= "3.7" and python_version < "3.12" pytz==2022.7.1 ; python_version >= "3.7" and python_version < "3.12" pyyaml==6.0 ; python_version >= "3.7" and python_version < "3.12" From de8d4562b655b5db3c218f4b98a243b62a3d65a7 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 7 Feb 2024 05:44:11 +0000 Subject: [PATCH 6/6] Add missing Werkzeug test requirement Used in `test/unit/util/test_get_url.py` since commit 69ebbb761aa610c1eab2f92a385c34b47259ac32 . Pin Werkzeug in `lib/galaxy/dependencies/dev-requirements.txt` to latest version compatible with pytest-httpserver 1.0.6 . Fix unit test collection error on Python 3.11: ``` File "/home/runner/work/galaxy/galaxy/galaxy root/.tox/unit-coverage/lib/python3.11/site-packages/pytest_httpserver/httpserver.py", line 26, in from werkzeug.http import parse_authorization_header ImportError: cannot import name 'parse_authorization_header' from 'werkzeug.http' (/home/runner/work/galaxy/galaxy/galaxy root/.tox/unit-coverage/lib/python3.11/site-packages/werkzeug/http.py) ``` --- lib/galaxy/dependencies/dev-requirements.txt | 1 + packages/util/test-requirements.txt | 1 + pyproject.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index b2767bc36878..6393839ea395 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -142,6 +142,7 @@ urllib3[socks]==1.26.14 ; python_version >= "3.7" and python_version < "3.12" vine==5.0.0 ; python_version >= "3.7" and python_version < "3.12" watchdog==2.2.1 ; python_version >= "3.7" and python_version < "3.12" wcwidth==0.2.6 ; python_version >= "3.7" and python_version < "3.12" +werkzeug==2.2.3 ; python_version >= "3.7" and python_version < "3.12" wrapt==1.14.1 ; python_version >= "3.7" and python_version < "3.12" wsproto==1.2.0 ; python_version >= "3.7" and python_version < "3.12" yarl==1.8.2 ; python_version >= "3.7" and python_version < "3.12" diff --git a/packages/util/test-requirements.txt b/packages/util/test-requirements.txt index 779ccfa23c7e..2b7d321b4f1a 100644 --- a/packages/util/test-requirements.txt +++ b/packages/util/test-requirements.txt @@ -1,3 +1,4 @@ pytest pytest-httpserver responses +Werkzeug diff --git a/pyproject.toml b/pyproject.toml index c1c825f1aa84..64673290764d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,7 @@ testfixtures = "*" tuspy = "*" twill = "*" watchdog = "*" +Werkzeug = "*" [tool.ruff] select = ["E", "F", "B", "UP"]