diff --git a/charts/platform-storage/templates/service-monitor.yaml b/charts/platform-storage/templates/service-monitor.yaml index e81033f..5b29f7a 100644 --- a/charts/platform-storage/templates/service-monitor.yaml +++ b/charts/platform-storage/templates/service-monitor.yaml @@ -5,6 +5,7 @@ metadata: name: {{ include "platformStorage.metrics.fullname" . }} labels: {{ include "platformStorage.labels.standard" . | nindent 4 }} service: platform-storage-metrics + platform.apolo.us/scrape-metrics: "true" spec: selector: matchLabels: diff --git a/platform_storage_api/s3_storage.py b/platform_storage_api/s3_storage.py index 838458f..f5e551c 100644 --- a/platform_storage_api/s3_storage.py +++ b/platform_storage_api/s3_storage.py @@ -93,8 +93,8 @@ async def put_storage_usage(self, storage_usage: StorageUsage) -> None: except ClientError as err: if err.response["ResponseMetadata"]["HTTPStatusCode"] != 404: raise - await self._s3_client.create_bucket(Bucket=self._bucket_name) - await put_object() + await self._s3_client.create_bucket(Bucket=self._bucket_name) + await put_object() async def get_storage_usage(self) -> StorageUsage: try: diff --git a/platform_storage_api/storage_usage.py b/platform_storage_api/storage_usage.py index 78fc59b..fcdbcb2 100644 --- a/platform_storage_api/storage_usage.py +++ b/platform_storage_api/storage_usage.py @@ -141,6 +141,6 @@ def collect(self) -> Iterable[Metric]: ) for project in storage_usage.projects: metric_family.add_metric( - [project.org_name or "no_org", project.project_name], project.used + [project.org_name or "", project.project_name], project.used ) yield metric_family diff --git a/platform_storage_api/worker.py b/platform_storage_api/worker.py index 3474acc..f03be92 100644 --- a/platform_storage_api/worker.py +++ b/platform_storage_api/worker.py @@ -33,7 +33,9 @@ class App: @new_trace async def upload_storage_usage(self) -> None: + LOGGER.info("Starting storage usage collection") await self.storage_usage_service.upload_storage_usage() + LOGGER.info("Finished storage usage collection") def create_path_resolver(config: Config, fs: FileSystem) -> StoragePathResolver: diff --git a/tests/integration/test_metrics.py b/tests/integration/test_metrics.py index acdcb68..06eba16 100644 --- a/tests/integration/test_metrics.py +++ b/tests/integration/test_metrics.py @@ -100,6 +100,6 @@ async def test_metrics( """\ # HELP storage_used_bytes The amount of used storage space in bytes # TYPE storage_used_bytes gauge - storage_used_bytes{org_name="no_org",project_name="test-project"}""" + storage_used_bytes{org_name="",project_name="test-project"}""" ) ) diff --git a/tests/integration/test_storage_usage.py b/tests/integration/test_storage_usage.py new file mode 100644 index 0000000..979b9a9 --- /dev/null +++ b/tests/integration/test_storage_usage.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from unittest import mock + +import pytest +from aioresponses import aioresponses + +from platform_storage_api.s3_storage import StorageMetricsAsyncS3Storage +from platform_storage_api.storage_usage import StorageUsage, StorageUsageService + + +@pytest.fixture() +def aiohttp_mock() -> Iterator[aioresponses]: + with aioresponses(passthrough=["http://0.0.0.0", "http://127.0.0.1"]) as mocked: + yield mocked + + +class TestMetrics: + async def test_upload_storage_usage( + self, + aiohttp_mock: aioresponses, + storage_usage_service: StorageUsageService, + storage_metrics_s3_storage: StorageMetricsAsyncS3Storage, + cluster_name: str, + local_tmp_dir_path: Path, + ) -> None: + aiohttp_mock.get( + f"http://platform-admin/apis/admin/v1/clusters/{cluster_name}/orgs", + payload=[], + ) + + (local_tmp_dir_path / "test-project").mkdir() + + await storage_usage_service.upload_storage_usage() + + storage_usage = await storage_metrics_s3_storage.get_storage_usage() + + assert storage_usage == StorageUsage( + projects=[ + StorageUsage.Project(project_name="test-project", used=mock.ANY), + ] + ) + + async def test_upload_storage_usage__multiple_times( + self, + aiohttp_mock: aioresponses, + storage_usage_service: StorageUsageService, + storage_metrics_s3_storage: StorageMetricsAsyncS3Storage, + cluster_name: str, + local_tmp_dir_path: Path, + ) -> None: + aiohttp_mock.get( + f"http://platform-admin/apis/admin/v1/clusters/{cluster_name}/orgs", + payload=[], + ) + + (local_tmp_dir_path / "test-project").mkdir() + + await storage_usage_service.upload_storage_usage() + + await storage_metrics_s3_storage.get_storage_usage() + storage_usage = await storage_metrics_s3_storage.get_storage_usage() + + assert storage_usage == StorageUsage( + projects=[ + StorageUsage.Project(project_name="test-project", used=mock.ANY), + ] + )