From 5849b06273df27a820b4755f2e9c79635b92a546 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 1 Oct 2024 12:12:15 -0400 Subject: [PATCH] Fixes and expansion of webdav unit tests. --- lib/galaxy/files/sources/_pyfilesystem2.py | 13 ++++-- lib/galaxy/files/sources/webdav.py | 1 + test/unit/files/_util.py | 6 ++- test/unit/files/test_webdav.py | 43 +++++++++++++++---- test/unit/files/webdav_file_sources_conf.yml | 6 +++ ...dav_file_sources_without_use_temp_conf.yml | 7 +++ 6 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 test/unit/files/webdav_file_sources_conf.yml create mode 100644 test/unit/files/webdav_file_sources_without_use_temp_conf.yml diff --git a/lib/galaxy/files/sources/_pyfilesystem2.py b/lib/galaxy/files/sources/_pyfilesystem2.py index afcff2884961..5b3add6e6119 100644 --- a/lib/galaxy/files/sources/_pyfilesystem2.py +++ b/lib/galaxy/files/sources/_pyfilesystem2.py @@ -38,6 +38,7 @@ class PyFilesystem2FilesSource(BaseFilesSource): required_package: ClassVar[str] supports_pagination = True supports_search = True + allow_key_error_on_empty_directories = False # work around a bug in webdav def __init__(self, **kwd: Unpack[FilesSourceProperties]): if self.required_module is None: @@ -65,10 +66,14 @@ def _list( with self._open_fs(user_context=user_context, opts=opts) as h: if recursive: recursive_result: List[AnyRemoteEntry] = [] - for p, dirs, files in h.walk(path, namespaces=["details"]): - to_dict = functools.partial(self._resource_info_to_dict, p) - recursive_result.extend(map(to_dict, dirs)) - recursive_result.extend(map(to_dict, files)) + try: + for p, dirs, files in h.walk(path, namespaces=["details"]): + to_dict = functools.partial(self._resource_info_to_dict, p) + recursive_result.extend(map(to_dict, dirs)) + recursive_result.extend(map(to_dict, files)) + except KeyError: + if not self.allow_key_error_on_empty_directories: + raise return recursive_result, len(recursive_result) else: page = self._to_page(limit, offset) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index a8d8e4c89b0f..99171fb8daee 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -28,6 +28,7 @@ class WebDavFilesSource(PyFilesystem2FilesSource): plugin_type = "webdav" required_module = WebDAVFS required_package = "fs.webdavfs" + allow_key_error_on_empty_directories = True def _open_fs(self, user_context=None, opts: Optional[FilesSourceOptions] = None): props = cast(WebDavFilesSourceProperties, self._serialization_props(user_context)) diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py index 16ad933affa4..2bf1274fcee5 100644 --- a/test/unit/files/_util.py +++ b/test/unit/files/_util.py @@ -2,6 +2,7 @@ import os import tempfile +from typing import Optional from galaxy.files import ( ConfiguredFileSources, @@ -156,8 +157,9 @@ def write_from( file_source_path.file_source.write_from(file_source_path.path, f.name, user_context=user_context) -def configured_file_sources(conf_file): - file_sources_config = FileSourcePluginsConfig() +def configured_file_sources(conf_file, file_sources_config: Optional[FileSourcePluginsConfig] = None): + file_sources_config = file_sources_config or FileSourcePluginsConfig() + assert file_sources_config if isinstance(conf_file, str): conf = ConfiguredFileSourcesConf(conf_file=conf_file) else: diff --git a/test/unit/files/test_webdav.py b/test/unit/files/test_webdav.py index 071281e1bc88..9294881ec247 100644 --- a/test/unit/files/test_webdav.py +++ b/test/unit/files/test_webdav.py @@ -1,9 +1,10 @@ # docker run -v `pwd`/test/integration/webdav/data:/media -e WEBDAV_USERNAME=alice -e WEBDAV_PASSWORD=secret1234 -p 7083:7083 jmchilton/webdavdev -# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/webdav.py +# GALAXY_TEST_WEBDAV=1 pytest test/unit/files/test_webdav.py import os import pytest +from galaxy.files.plugins import FileSourcePluginsConfig from ._util import ( configured_file_sources, find, @@ -17,6 +18,7 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_conf.yml") +FILE_SOURCES_CONF_NO_USE_TEMP_FILES = os.path.join(SCRIPT_DIRECTORY, "webdav_file_sources_without_use_temp_conf.yml") USER_FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "webdav_user_file_sources_conf.yml") skip_if_no_webdav = pytest.mark.skipif(not os.environ.get("GALAXY_TEST_WEBDAV"), reason="GALAXY_TEST_WEBDAV not set") @@ -62,17 +64,40 @@ def test_sniff_to_tmp(): @skip_if_no_webdav def test_serialization(): - # serialize the configured file sources and rematerialize them, - # ensure they still function. This is needed for uploading files. - file_sources = serialize_and_recover(configured_file_sources(FILE_SOURCES_CONF)) + configs = [FILE_SOURCES_CONF_NO_USE_TEMP_FILES, FILE_SOURCES_CONF] + for config in configs: + # serialize the configured file sources and rematerialize them, + # ensure they still function. This is needed for uploading files. + file_sources = serialize_and_recover(configured_file_sources(config)) - res = list_root(file_sources, "gxfiles://test1", recursive=True) - assert find_file_a(res) + res = list_root(file_sources, "gxfiles://test1", recursive=True) + assert find_file_a(res) - res = list_root(file_sources, "gxfiles://test1", recursive=False) - assert find_file_a(res) + res = list_root(file_sources, "gxfiles://test1", recursive=False) + assert find_file_a(res) - _download_and_check_file(file_sources) + _download_and_check_file(file_sources) + + +@skip_if_no_webdav +def test_config_options(): + file_sources = configured_file_sources(FILE_SOURCES_CONF) + fs = file_sources._file_sources[0] + user_context = user_context_fixture() + assert fs._open_fs(user_context).use_temp_files + + file_sources = configured_file_sources(FILE_SOURCES_CONF_NO_USE_TEMP_FILES) + fs = file_sources._file_sources[0] + user_context = user_context_fixture() + assert not fs._open_fs(user_context).use_temp_files + + disable_default_use_temp = FileSourcePluginsConfig( + webdav_use_temp_files=False, + ) + file_sources = configured_file_sources(FILE_SOURCES_CONF, disable_default_use_temp) + fs = file_sources._file_sources[0] + user_context = user_context_fixture() + assert not fs._open_fs(user_context).use_temp_files @skip_if_no_webdav diff --git a/test/unit/files/webdav_file_sources_conf.yml b/test/unit/files/webdav_file_sources_conf.yml new file mode 100644 index 000000000000..6116edd6de1f --- /dev/null +++ b/test/unit/files/webdav_file_sources_conf.yml @@ -0,0 +1,6 @@ +- type: webdav + id: test1 + doc: Test WebDAV server. + url: http://127.0.0.1:7083 + login: alice + password: secret1234 diff --git a/test/unit/files/webdav_file_sources_without_use_temp_conf.yml b/test/unit/files/webdav_file_sources_without_use_temp_conf.yml new file mode 100644 index 000000000000..916f8068f459 --- /dev/null +++ b/test/unit/files/webdav_file_sources_without_use_temp_conf.yml @@ -0,0 +1,7 @@ +- type: webdav + id: test1 + doc: Test WebDAV server. + url: http://127.0.0.1:7083 + login: alice + password: secret1234 + use_temp_files: false