refactor: hf aggregation repo integrated once

tests/evaluation/conftest.py

@@ -5,6 +5,7 @@
 from uuid import uuid4
 
 from dotenv import load_dotenv
+from fsspec.implementations.memory import MemoryFileSystem  # type: ignore
 from pydantic import BaseModel
 from pytest import fixture
 
@@ -31,8 +32,6 @@
     Runner,
     RunOverview,
 )
-from fsspec.implementations.memory import MemoryFileSystem  # type: ignore
-
 from tests.conftest import DummyStringInput, DummyStringOutput
 
 FAIL_IN_EVAL_INPUT = "fail in eval"
@@ -253,6 +252,11 @@ def temp_file_system() -> Iterable[MemoryFileSystem]:
         mfs.store.clear()
 
 
+@fixture(scope="session")
+def hugging_face_test_repository_id():
+    return f"Aleph-Alpha/test-{str(uuid4())}"
+
+
 @fixture(scope="session")
 def hugging_face_token() -> str:
     load_dotenv()

tests/evaluation/test_aggregation_repository.py

@@ -1,7 +1,9 @@
 from typing import Iterable
+from unittest.mock import patch
 from uuid import uuid4
 
 from _pytest.fixtures import FixtureRequest
+from fsspec.implementations.memory import MemoryFileSystem  # type: ignore
 from pytest import fixture, mark
 
 from intelligence_layer.core import utc_now
@@ -10,14 +12,37 @@
     AggregationRepository,
     EvaluationOverview,
 )
+from intelligence_layer.evaluation.aggregation.hugging_face_aggregation_repository import (
+    HuggingFaceAggregationRepository,
+)
 from tests.evaluation.conftest import DummyAggregatedEvaluation
 
 test_repository_fixtures = [
     "file_aggregation_repository",
     "in_memory_aggregation_repository",
+    "mocked_hugging_face_aggregation_repository",
 ]
 
 
+@fixture
+def mocked_hugging_face_aggregation_repository(
+    temp_file_system: MemoryFileSystem,
+) -> Iterable[HuggingFaceAggregationRepository]:
+    # this repository should already exist and does not have to be deleted after the tests
+    class_to_patch = "intelligence_layer.evaluation.aggregation.hugging_face_aggregation_repository.HuggingFaceAggregationRepository"
+    with patch(f"{class_to_patch}.create_repository", autospec=True), patch(
+        f"{class_to_patch}.delete_repository",
+        autospec=True,
+    ):
+        repo = HuggingFaceAggregationRepository(
+            repository_id="doesn't-matter",
+            token="non-existing-token",
+            private=True,
+        )
+        repo._file_system = temp_file_system
+        yield repo
+
+
 @fixture
 def aggregation_overviews(
     evaluation_overview: EvaluationOverview,
@@ -70,7 +95,6 @@ def test_aggregation_repository_stores_and_returns_an_aggregation_overview(
 def test_aggregation_overview_returns_none_for_not_existing_id(
     repository_fixture: str,
     request: FixtureRequest,
-    aggregation_overview: AggregationOverview[DummyAggregatedEvaluation],
 ) -> None:
     aggregation_repository: AggregationRepository = request.getfixturevalue(
         repository_fixture
@@ -90,9 +114,7 @@ def test_aggregation_overview_returns_none_for_not_existing_id(
 def test_aggregation_overviews_returns_all_aggregation_overviews(
     repository_fixture: str,
     request: FixtureRequest,
-    evaluation_overview: EvaluationOverview,
     aggregation_overviews: Iterable[AggregationOverview[DummyAggregatedEvaluation]],
-    dummy_aggregated_evaluation: DummyAggregatedEvaluation,
 ) -> None:
     aggregation_repository: AggregationRepository = request.getfixturevalue(
         repository_fixture
@@ -116,9 +138,7 @@ def test_aggregation_overviews_returns_all_aggregation_overviews(
 def test_aggregation_overview_ids_returns_sorted_ids(
     repository_fixture: str,
     request: FixtureRequest,
-    evaluation_overview: EvaluationOverview,
     aggregation_overviews: Iterable[AggregationOverview[DummyAggregatedEvaluation]],
-    dummy_aggregated_evaluation: DummyAggregatedEvaluation,
 ) -> None:
     aggregation_repository: AggregationRepository = request.getfixturevalue(
         repository_fixture

tests/evaluation/test_dataset_repository.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 from typing import Any, Iterable
 from unittest.mock import patch
-from uuid import uuid4
 
 import pytest
 from fsspec.implementations.memory import MemoryFileSystem  # type: ignore
@@ -23,16 +22,9 @@ def file_dataset_repository(tmp_path: Path) -> FileDatasetRepository:
     return FileDatasetRepository(tmp_path)
 
 
-@fixture(scope="session")
-def hugging_face_dataset_repository_id() -> str:
-    return f"Aleph-Alpha/test-datasets-{str(uuid4())}"
-
-
-
-
 @fixture
 def mocked_hugging_face_dataset_repository(
-    hugging_face_dataset_repository_id: str, temp_file_system: MemoryFileSystem
+    temp_file_system: MemoryFileSystem,
 ) -> Iterable[HuggingFaceDatasetRepository]:
     # this repository should already exist and does not have to be deleted after the tests
     class_to_patch = "intelligence_layer.evaluation.dataset.hugging_face_dataset_repository.HuggingFaceDatasetRepository"
@@ -41,7 +33,7 @@ def mocked_hugging_face_dataset_repository(
         autospec=True,
     ):
         repo = HuggingFaceDatasetRepository(
-            repository_id=hugging_face_dataset_repository_id,
+            repository_id="doesn't-matter",
             token="non-existing-token",
             private=True,
         )
@@ -344,4 +336,3 @@ def test_example_raises_error_for_not_existing_dataset_id(
             DummyStringInput,
             DummyStringOutput,
         )
-

tests/evaluation/test_hugging_face_aggregation_repository.py

@@ -17,24 +17,21 @@ def dummy_aggregated_evaluation() -> DummyAggregatedEvaluation:
     return DummyAggregatedEvaluation(score=0.5)
 
 
-@fixture(scope="session")
-def hugging_face_aggregation_repository_id() -> str:
-    return "Aleph-Alpha/test-aggregation-datasets"
-
-
+# these fixtures should only be used once and are here for readable tests
+# because creating/deleting HuggingFace repositories can be rate-limited
 @fixture(scope="session")
 def hugging_face_aggregation_repository(
-    hugging_face_token: str, hugging_face_aggregation_repository_id: str
+    hugging_face_token: str, hugging_face_test_repository_id: str
 ) -> Iterable[HuggingFaceAggregationRepository]:
     try:
         yield HuggingFaceAggregationRepository(
-            hugging_face_aggregation_repository_id,
+            hugging_face_test_repository_id,
             token=hugging_face_token,
             private=True,
         )
     finally:
         huggingface_hub.delete_repo(
-            repo_id=hugging_face_aggregation_repository_id,
+            repo_id=hugging_face_test_repository_id,
             token=hugging_face_token,
             repo_type="dataset",
             missing_ok=True,

tests/evaluation/test_hugging_face_dataset_repository.py

@@ -13,19 +13,15 @@ class DummyAggregatedEvaluation(BaseModel):
     score: float
 
 
-@fixture(scope="session")
-def hugging_face_dataset_repository_id() -> str:
-    return f"Aleph-Alpha/test-datasets-{str(uuid4())}"
-
-
 # these fixtures should only be used once and are here for readable tests
+# because creating/deleting HuggingFace repositories can be rate-limited
 @fixture(scope="session")
 def hugging_face_dataset_repository(
-    hugging_face_dataset_repository_id: str, hugging_face_token: str
+    hugging_face_test_repository_id: str, hugging_face_token: str
 ) -> Iterable[HuggingFaceDatasetRepository]:
     # this repository should already exist and does not have to be deleted after the tests
     repo = HuggingFaceDatasetRepository(
-        repository_id=hugging_face_dataset_repository_id,
+        repository_id=hugging_face_test_repository_id,
         token=hugging_face_token,
         private=True,
     )
@@ -93,7 +89,6 @@ def test_hugging_face_repository_supports_all_operations_for_created_dataset(
         assert (
             hugging_face_repository.example(dataset.id, example.id, str, str) == example
         )
-
 
     # one gets None for a not existing example ID
     assert (

tests/evaluation/test_hugging_face_repository.py

@@ -1,7 +1,10 @@
 from uuid import uuid4
+
 import huggingface_hub  # type: ignore
 
-from intelligence_layer.evaluation.infrastructure.hugging_face_repository import HuggingFaceRepository
+from intelligence_layer.evaluation.infrastructure.hugging_face_repository import (
+    HuggingFaceRepository,
+)
 
 
 def test_hugging_face_repository_can_create_and_delete_a_repository(