Update hexkit to 3.7 and use its fed s3 fixture

.pre-commit-config.yaml

@@ -45,7 +45,7 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

.pyproject_generation/pyproject_custom.toml

@@ -5,7 +5,7 @@ description = "State Management Service - Provides a REST API for basic infrastr
 dependencies = [
     "typer >= 0.12",
     "ghga-service-commons[api] >= 3.1",
-    "hexkit[mongodb,s3,akafka] >= 3.5",
+    "hexkit[mongodb,s3,akafka] >= 3.7",
     "hvac>=2",
 ]
 

pyproject.toml

@@ -26,7 +26,7 @@ description = "State Management Service - Provides a REST API for basic infrastr
 dependencies = [
     "typer >= 0.12",
     "ghga-service-commons[api] >= 3.1",
-    "hexkit[mongodb,s3,akafka] >= 3.5",
+    "hexkit[mongodb,s3,akafka] >= 3.7",
     "hvac>=2",
 ]
 

tests/conftest.py

@@ -14,11 +14,7 @@
 # limitations under the License.
 """Import necessary test fixtures."""
 
-from collections.abc import AsyncGenerator, Generator
-
 import pytest
-import pytest_asyncio
-from hexkit.custom_types import PytestScope
 from hexkit.providers.akafka.testutils import (  # noqa: F401
     kafka_container_fixture,
     kafka_fixture,
@@ -27,135 +23,18 @@
     mongodb_container_fixture,
     mongodb_fixture,
 )
-from hexkit.providers.s3 import S3ObjectStorage
-from hexkit.providers.s3.testutils import (
-    S3ContainerFixture,
-    S3Fixture,
-    temp_file_object,
+from hexkit.providers.s3.testutils import (  # noqa: F401
+    federated_s3_fixture,
+    s3_multi_container_fixture,
 )
 
-from sms.config import Config
-from tests.fixtures.config import get_config
-
-
-# Move the federation fixtures to hexkit if deemed useful
-class FederatedS3Fixture:
-    """Fixture containing multiple S3 fixtures to simulate federated storage."""
-
-    def __init__(self, storages: dict[str, S3Fixture]):
-        self.nodes = storages
-
-    def get_patched_config(self, *, config: Config):
-        """Update a Config instance so the object storage credentials point to the
-        respective S3Fixtures.
-        """
-        config_vals = config.model_dump()
-        for alias in config.object_storages:
-            node_config = self.nodes[alias].config.model_dump()
-            config_vals["object_storages"][alias]["credentials"] = node_config
-        return Config(**config_vals)
-
-    async def populate_dummy_items(self, alias: str, buckets: dict[str, list[str]]):
-        """Convenience function to populate a specific S3Fixture.
-
-        The `buckets` arg is a dictionary with bucket names as keys and lists of objects
-        as values. The buckets can be empty, and the objects are created with a size of
-        1 byte. This function might be modified or removed in the hexkit version.
-        """
-        if alias not in self.nodes:
-            # This would indicate some kind of mismatch between config and fixture
-            raise RuntimeError(f"Alias '{alias}' not found in the federated S3 fixture")
-        storage = self.nodes[alias]
-        # Populate the buckets so even empty buckets are established
-        await storage.populate_buckets([bucket for bucket in buckets])
-        for bucket, objects in buckets.items():
-            for object in objects:
-                with temp_file_object(bucket, object, 1) as file:
-                    await storage.populate_file_objects([file])
-
-
-class MultiS3ContainerFixture:
-    """Fixture for managing multiple running S3 test containers in order to mimic
-    multiple object storages.
-
-    Without this fixture, separate S3Fixture instances would access the same
-    underlying storage resources.
-    """
-
-    def __init__(self, s3_containers: dict[str, S3ContainerFixture]):
-        self.s3_containers = s3_containers
-
-    def __enter__(self):
-        """Enter the context manager and start the S3 containers."""
-        for container in self.s3_containers.values():
-            container.__enter__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        """Exit the context manager and clean up the S3 containers."""
-        for container in self.s3_containers.values():
-            container.__exit__(exc_type, exc_val, exc_tb)
-
-
-def _multi_s3_container_fixture() -> Generator[MultiS3ContainerFixture, None, None]:
-    """Fixture function for getting multiple running S3 test containers."""
-    config = get_config()
-    s3_containers = {
-        name: S3ContainerFixture(name=name) for name in config.object_storages
-    }
-    with MultiS3ContainerFixture(s3_containers) as multi_s3_container:
-        yield multi_s3_container
+from tests.fixtures.config import DEFAULT_TEST_CONFIG
 
 
-def get_multi_s3_container_fixture():
-    """Get a fixture containing multiple LocalStack test containers.
+@pytest.fixture(scope="session")
+def storage_aliases():
+    """Supplies the aliases to the federated S3 fixture.
 
-    By default, the session scope is used for LocalStack test containers.
+    This tells it how many S3 storages to spin up and what names to associate with them.
     """
-    return pytest.fixture(
-        _multi_s3_container_fixture, scope="session", name="multi_s3_container"
-    )
-
-
-multi_s3_container_fixture = get_multi_s3_container_fixture()
-
-
-def _persistent_federated_s3_fixture(
-    multi_s3_container: MultiS3ContainerFixture,
-) -> Generator[FederatedS3Fixture, None, None]:
-    """Fixture function that gets a persistent FederatedS3Fixture.
-
-    The state of each S3 storage in the fixture is not cleaned up.
-    """
-    storages = {}
-    for name, container in multi_s3_container.s3_containers.items():
-        config = container.s3_config
-        storage = S3ObjectStorage(config=config)
-        storages[name] = S3Fixture(config=config, storage=storage)
-    yield FederatedS3Fixture(storages)
-
-
-async def _clean_federated_s3_fixture(
-    multi_s3_container: MultiS3ContainerFixture,
-) -> AsyncGenerator[FederatedS3Fixture, None]:
-    """Fixture function that gets a clean FederatedS3Fixture instance.
-
-    The state of each S3 storage is cleaned up before yielding the fixture.
-    """
-    for federated_s3_fixture in _persistent_federated_s3_fixture(multi_s3_container):
-        for s3_fixture in federated_s3_fixture.nodes.values():
-            await s3_fixture.delete_buckets()
-        yield federated_s3_fixture
-
-
-def get_federated_s3_fixture(
-    scope: PytestScope = "function", name: str = "federated_s3"
-):
-    """Get a federated S3 storage fixture with desired scope.
-
-    The state of the S3 storage is not cleaned up by the fixture.
-    """
-    return pytest_asyncio.fixture(_clean_federated_s3_fixture, scope=scope, name=name)
-
-
-federated_s3_fixture = get_federated_s3_fixture()
+    return [alias for alias in DEFAULT_TEST_CONFIG.object_storages]

tests/integration/test_objects.py

@@ -15,15 +15,25 @@
 """Integration tests for the /objects endpoints."""
 
 import pytest
+from hexkit.providers.s3 import S3Config
+from hexkit.providers.s3.testutils import FederatedS3Fixture
 
-from tests.conftest import FederatedS3Fixture
-from tests.fixtures.config import DEFAULT_TEST_CONFIG
+from tests.fixtures.config import DEFAULT_TEST_CONFIG, Config
 from tests.fixtures.utils import VALID_BEARER_TOKEN, get_rest_client
 
 pytestmark = pytest.mark.asyncio()
 DEFAULT_ALIAS = "primary"
 
 
+def patch_config_for_alias(
+    alias: str, s3_config: S3Config, original_config: Config = DEFAULT_TEST_CONFIG
+) -> Config:
+    """Update the full config instance with the given s3 config for the given alias."""
+    dumped = original_config.model_dump()
+    dumped["object_storages"][alias]["credentials"] = s3_config
+    return Config(**dumped)
+
+
 def bucket_not_found_json(bucket_id: str) -> dict[str, str]:
     """Return the JSON response for a bucket not found error."""
     return {"detail": f"Bucket with ID '{bucket_id}' does not exist."}
@@ -39,29 +49,6 @@ def invalid_object_json(object_id: str) -> dict[str, str]:
     return {"detail": f"Object ID '{object_id}' is invalid."}
 
 
-async def test_federated_fixture(federated_s3: FederatedS3Fixture):
-    """Test that the federated S3 fixture actually uses separate S3 instances."""
-    buckets = {
-        "bucket1": ["object1", "object2"],
-        "empty": [],
-    }
-
-    await federated_s3.populate_dummy_items("primary", buckets)
-
-    assert await federated_s3.nodes["primary"].storage.does_object_exist(
-        bucket_id="bucket1", object_id="object1"
-    )
-    assert await federated_s3.nodes["primary"].storage.does_bucket_exist(
-        bucket_id="empty"
-    )
-    assert not await federated_s3.nodes["secondary"].storage.does_object_exist(
-        bucket_id="bucket1", object_id="object1"
-    )
-    assert not await federated_s3.nodes["secondary"].storage.does_bucket_exist(
-        bucket_id="empty"
-    )
-
-
 @pytest.mark.parametrize(
     "bucket_id, object_id",
     [
@@ -93,7 +80,8 @@ async def test_does_object_exist(
     buckets: dict[str, list[str]],
 ):
     """Test the /objects/{bucket_id}/{object_id} endpoint."""
-    config = federated_s3.get_patched_config(config=DEFAULT_TEST_CONFIG)
+    s3_config = federated_s3.get_configs_by_alias()[DEFAULT_ALIAS]
+    config = patch_config_for_alias(alias=DEFAULT_ALIAS, s3_config=s3_config)
     await federated_s3.populate_dummy_items(DEFAULT_ALIAS, buckets)
 
     async with get_rest_client(config=config) as client:
@@ -119,7 +107,8 @@ async def test_does_object_exist(
 
 async def test_list_objects(federated_s3: FederatedS3Fixture):
     """Test the GET /objects/{bucket_id} endpoint."""
-    config = federated_s3.get_patched_config(config=DEFAULT_TEST_CONFIG)
+    s3_config = federated_s3.get_configs_by_alias()[DEFAULT_ALIAS]
+    config = patch_config_for_alias(alias=DEFAULT_ALIAS, s3_config=s3_config)
 
     buckets = {
         "bucket1": ["object1", "object2"],
@@ -153,7 +142,8 @@ async def test_list_objects(federated_s3: FederatedS3Fixture):
 
 async def test_delete_objects(federated_s3: FederatedS3Fixture):
     """Test the DELETE /objects/{bucket_id} endpoint."""
-    config = federated_s3.get_patched_config(config=DEFAULT_TEST_CONFIG)
+    s3_config = federated_s3.get_configs_by_alias()[DEFAULT_ALIAS]
+    config = patch_config_for_alias(alias=DEFAULT_ALIAS, s3_config=s3_config)
 
     buckets = {
         "bucket1": ["object1", "object2"],