From 8c60923674c5e5c451e9775fe667954ea36b895e Mon Sep 17 00:00:00 2001 From: Raphael Cohen Date: Fri, 25 Oct 2024 18:25:56 +0200 Subject: [PATCH 1/4] feat: Specify docker image when publishing a module --- CHANGELOG.md | 4 ++++ sekoia_automation/scripts/sync_library.py | 1 + tests/scripts/test_sync_library.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 338deab..b09733b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Specify docker image when publishing a module + ### Fixed - Fix tests for async version of connector. diff --git a/sekoia_automation/scripts/sync_library.py b/sekoia_automation/scripts/sync_library.py index a03e5b9..d06ea31 100755 --- a/sekoia_automation/scripts/sync_library.py +++ b/sekoia_automation/scripts/sync_library.py @@ -363,6 +363,7 @@ def load_module(self, module_path: Path): module_info = json.load(fd) docker_name = self._get_module_docker_name(module_info) + module_info["docker"] = f"{docker_name}:{module_info['version']}" if self.registry_check and not self.check_image_on_registry( docker_name, module_info["version"] ): diff --git a/tests/scripts/test_sync_library.py b/tests/scripts/test_sync_library.py index 7eadc9c..3ffa774 100644 --- a/tests/scripts/test_sync_library.py +++ b/tests/scripts/test_sync_library.py @@ -68,6 +68,7 @@ def test_no_module_success(tmp_module, module, action, trigger, connector, **kwa assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" assert history[1].method == "PATCH" assert history[1].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" + assert "docker" in history[1].json() assert history[1].headers["Authorization"] == f"Bearer {API_KEY}" assert history[2].method == "GET" assert history[2].url == f"{SYMPOHNY_URL}/triggers/{trigger['uuid']}" @@ -105,6 +106,7 @@ def test_no_module_404(tmp_module, module, action, trigger, connector, **kwargs) assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" assert history[1].method == "POST" assert history[1].url == f"{SYMPOHNY_URL}/modules" + assert "docker" in history[1].json() assert history[1].headers["Authorization"] == f"Bearer {API_KEY}" assert history[2].method == "GET" assert history[2].url == f"{SYMPOHNY_URL}/triggers/{trigger['uuid']}" @@ -168,6 +170,7 @@ def test_with_module(tmp_module, module, action, trigger, connector, **kwargs): assert history[1].method == "PATCH" assert history[1].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[1].headers["Authorization"] == f"Bearer {API_KEY}" + assert "docker" in history[1].json() assert history[2].method == "GET" assert history[2].url == f"{SYMPOHNY_URL}/triggers/{trigger['uuid']}" assert history[2].headers["Authorization"] == f"Bearer {API_KEY}" From 2650972ff9dfcb56643df8b143ae63cfc2c6183a Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Wed, 2 Oct 2024 11:45:53 +0200 Subject: [PATCH 2/4] refactor(AsyncConnector): outsource the computation of the batchapi url and add tests on it --- sekoia_automation/aio/connector.py | 13 ++++++++----- tests/aio/test_connector.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/sekoia_automation/aio/connector.py b/sekoia_automation/aio/connector.py index 2129dd8..ca9220e 100644 --- a/sekoia_automation/aio/connector.py +++ b/sekoia_automation/aio/connector.py @@ -129,6 +129,13 @@ async def _async_send_chunk( return events_ids + @property + def _batchapi_url(self): + if intake_server := self.configuration.intake_server: + return urljoin(intake_server, "batch") + else: + return urljoin(self.intake_url, "batch") + async def push_data_to_intakes( self, events: list[str] ) -> list[str]: # pragma: no cover @@ -142,10 +149,6 @@ async def push_data_to_intakes( list[str]: """ self._last_events_time = datetime.utcnow() - if intake_server := self.configuration.intake_server: - batch_api = urljoin(intake_server, "batch") - else: - batch_api = urljoin(self.intake_url, "batch") result_ids = [] @@ -153,7 +156,7 @@ async def push_data_to_intakes( async with self.session() as session: forwarders = [ - self._async_send_chunk(session, batch_api, chunk_index, chunk) + self._async_send_chunk(session, self._batchapi_url, chunk_index, chunk) for chunk_index, chunk in enumerate(chunks) ] async for ids in limit_concurrency(forwarders, self.max_concurrency_tasks): diff --git a/tests/aio/test_connector.py b/tests/aio/test_connector.py index dcbe67e..6b409b9 100644 --- a/tests/aio/test_connector.py +++ b/tests/aio/test_connector.py @@ -215,3 +215,23 @@ async def test_async_connector_raise_error( except Exception as e: assert isinstance(e, RuntimeError) assert str(e) == expected_error + + +@pytest.mark.parametrize( + 'base_url,expected_batchapi_url', + [ + ('http://intake.fake.url/', 'http://intake.fake.url/batch'), + ('http://fake.url/intake/', 'http://fake.url/intake/batch'), + ] +) +def test_async_connector_batchapi_url(storage, mocked_trigger_logs, base_url: str, expected_batchapi_url: str): + with patch("sentry_sdk.set_tag"): + async_connector = DummyAsyncConnector(data_path=storage) + + async_connector.trigger_activation = "2022-03-14T11:16:14.236930Z" + async_connector.configuration = { + "intake_key": "", + "intake_server": base_url, + } + + assert async_connector._batchapi_url == expected_batchapi_url From 291a55ee54bd7621f3426fa6356b1300b099a097 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Wed, 2 Oct 2024 11:47:56 +0200 Subject: [PATCH 3/4] fix(AsyncConnector): replace urllib.parse.urljoin by posixpath.join --- sekoia_automation/aio/connector.py | 2 +- tests/aio/test_connector.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sekoia_automation/aio/connector.py b/sekoia_automation/aio/connector.py index ca9220e..19d9091 100644 --- a/sekoia_automation/aio/connector.py +++ b/sekoia_automation/aio/connector.py @@ -6,7 +6,7 @@ from contextlib import asynccontextmanager from datetime import datetime from pathlib import Path -from urllib.parse import urljoin +from posixpath import join as urljoin from aiohttp import ClientSession from aiolimiter import AsyncLimiter diff --git a/tests/aio/test_connector.py b/tests/aio/test_connector.py index 6b409b9..28fc70b 100644 --- a/tests/aio/test_connector.py +++ b/tests/aio/test_connector.py @@ -1,7 +1,7 @@ """Test async connector.""" from unittest.mock import Mock, patch -from urllib.parse import urljoin +from posixpath import join as urljoin import pytest from aiolimiter import AsyncLimiter @@ -222,6 +222,7 @@ async def test_async_connector_raise_error( [ ('http://intake.fake.url/', 'http://intake.fake.url/batch'), ('http://fake.url/intake/', 'http://fake.url/intake/batch'), + ('http://fake.url/intake', 'http://fake.url/intake/batch'), ] ) def test_async_connector_batchapi_url(storage, mocked_trigger_logs, base_url: str, expected_batchapi_url: str): From 2d8dc1bc2e93049c11320fadf35f60b796dbe0f1 Mon Sep 17 00:00:00 2001 From: Sebastien Quioc Date: Wed, 2 Oct 2024 11:49:14 +0200 Subject: [PATCH 4/4] chore(SDK): add entry in the changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b09733b..b18bd50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Replace ulrllib.parse.urljoin by posixpath.join in AsyncConnector - Fix tests for async version of connector. ## 1.16.0 - 2024-10-16