Merge pull request #18135 from jmchilton/s3fs

Update s3fs dependency.
galaxyproject · May 14, 2024 · 225bcf2 · 225bcf2
2 parents 9fa7b04 + 076fff8
commit 225bcf2
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 21 deletions.
diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt
@@ -70,7 +70,7 @@ fastapi-slim==0.111.0 ; python_version >= "3.8" and python_version < "3.13"
 filelock==3.14.0 ; python_version >= "3.8" and python_version < "3.13"
 frozenlist==1.4.1 ; python_version >= "3.8" and python_version < "3.13"
 fs==2.4.16 ; python_version >= "3.8" and python_version < "3.13"
-fsspec==2023.12.2 ; python_version >= "3.8" and python_version < "3.13"
+fsspec==2024.3.1 ; python_version >= "3.8" and python_version < "3.13"
 future==1.0.0 ; python_version >= "3.8" and python_version < "3.13"
 galaxy-sequence-utils==1.1.5 ; python_version >= "3.8" and python_version < "3.13"
 galaxy2cwl==0.1.4 ; python_version >= "3.8" and python_version < "3.13"
@@ -171,7 +171,7 @@ routes==2.5.1 ; python_version >= "3.8" and python_version < "3.13"
 rpds-py==0.18.0 ; python_version >= "3.8" and python_version < "3.13"
 ruamel-yaml-clib==0.2.8 ; platform_python_implementation == "CPython" and python_version < "3.13" and python_version >= "3.8"
 ruamel-yaml==0.18.6 ; python_version >= "3.8" and python_version < "3.13"
-s3fs==2023.12.2 ; python_version >= "3.8" and python_version < "3.13"
+s3fs==2024.3.1 ; python_version >= "3.8" and python_version < "3.13"
 schema-salad==8.5.20240410123758 ; python_version >= "3.8" and python_version < "3.13"
 setuptools-scm==5.0.2 ; python_version >= "3.8" and python_version < "3.13"
 setuptools==69.5.1 ; python_version >= "3.8" and python_version < "3.13"

diff --git a/pyproject.toml b/pyproject.toml
@@ -124,7 +124,7 @@ uvloop = "*"
 WebOb = "*"
 Whoosh = "*"
 zipstream-new = "*"
-s3fs = "^2023.1.0"
+s3fs = "*"
 
 [tool.poetry.group.dev.dependencies]
 ase = ">=3.18.1"

diff --git a/test/unit/files/_util.py b/test/unit/files/_util.py
@@ -164,6 +164,25 @@ def configured_file_sources(conf_file):
     return conf
 
 
+def assert_can_write_and_read_to_conf(conf: dict):
+    test_filename = "moo.txt"
+    test_contents = "Hello World from Files Testing!"
+
+    file_source_id = conf["id"]
+    file_sources = configured_file_sources([conf])
+    test_uri = f"gxfiles://{file_source_id}/{test_filename}"
+    write_from(
+        file_sources,
+        test_uri,
+        test_contents,
+    )
+    assert_realizes_contains(
+        file_sources,
+        test_uri,
+        test_contents,
+    )
+
+
 def assert_simple_file_realize(conf_file, recursive=False, filename="a", contents="a\n", contains=False):
     user_context = user_context_fixture()
     file_sources = configured_file_sources(conf_file)

diff --git a/test/unit/files/test_azure.py b/test/unit/files/test_azure.py
@@ -3,11 +3,7 @@
 import pytest
 
 from galaxy.util.unittest_utils import skip_unless_environ
-from ._util import (
-    assert_realizes_contains,
-    configured_file_sources,
-    write_from,
-)
+from ._util import assert_can_write_and_read_to_conf
 
 pytest.importorskip("fs.azblob")
 
@@ -26,16 +22,4 @@ def test_azure():
         "namespace_type": os.environ.get("GALAXY_TEST_AZURE_NAMESPACE_TYPE", "flat"),
         "writable": True,
     }
-    file_sources = configured_file_sources([conf])
-    test_uri = "gxfiles://azure_test/moo"
-    test_contents = "Hello World from Files Testing!"
-    write_from(
-        file_sources,
-        test_uri,
-        test_contents,
-    )
-    assert_realizes_contains(
-        file_sources,
-        test_uri,
-        test_contents,
-    )
+    assert_can_write_and_read_to_conf(conf)
diff --git a/test/unit/files/test_s3.py b/test/unit/files/test_s3.py
@@ -2,7 +2,9 @@
 
 import pytest
 
+from galaxy.util.unittest_utils import skip_unless_environ
 from ._util import (
+    assert_can_write_and_read_to_conf,
     assert_realizes_contains,
     assert_simple_file_realize,
     configured_file_sources,
@@ -49,3 +51,19 @@ def test_file_source_specific():
     assert file_source_pair.file_source.id == "test1"
 
     assert_realizes_contains(file_sources, file_url, "DATA USE POLICIES", user_context=user_context)
+
+
+@skip_unless_environ("GALAXY_TEST_AWS_ACCESS_KEY")
+@skip_unless_environ("GALAXY_TEST_AWS_SECRET_KEY")
+@skip_unless_environ("GALAXY_TEST_AWS_BUCKET")
+def test_read_write_against_aws():
+    conf = {
+        "type": "s3fs",
+        "id": "playtest",
+        "doc": "Test against Play development server.",
+        "secret": os.environ["GALAXY_TEST_AWS_SECRET_KEY"],
+        "key": os.environ["GALAXY_TEST_AWS_ACCESS_KEY"],
+        "bucket": os.environ["GALAXY_TEST_AWS_BUCKET"],
+        "writable": True,
+    }
+    assert_can_write_and_read_to_conf(conf)