From a1c87626bf310a5f5dc30cd5a707822d66fab07f Mon Sep 17 00:00:00 2001 From: scallec Date: Tue, 26 Mar 2024 16:45:24 +0100 Subject: [PATCH] fix(connector) Use intake URL from file fix new version remove extra space run ruff changelog and version run black Add test update black fix tests again --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- sekoia_automation/aio/connector.py | 7 ++++--- sekoia_automation/connector/__init__.py | 9 +++++---- sekoia_automation/module.py | 5 +++++ tests/aio/test_connector.py | 12 +++++++----- tests/connectors/test_connector.py | 5 ++++- tests/test_trigger.py | 7 +++++++ 8 files changed, 39 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb7448..c10cbc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.12.2] - 2024-03-26 + +### Fixed + +- Use file for batch url computation + ## [1.12.1] - 2024-03-22 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 332381e..a473d93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "sekoia-automation-sdk" -version = "1.12.1" +version = "1.12.2" description = "SDK to create Sekoia.io playbook modules" license = "MIT" readme = "README.md" diff --git a/sekoia_automation/aio/connector.py b/sekoia_automation/aio/connector.py index c7dba8d..8be9cb7 100644 --- a/sekoia_automation/aio/connector.py +++ b/sekoia_automation/aio/connector.py @@ -1,6 +1,5 @@ """Contains connector with async version.""" -import os from abc import ABC from asyncio import AbstractEventLoop, get_event_loop from collections.abc import AsyncGenerator @@ -154,8 +153,10 @@ async def push_data_to_intakes( list[str]: """ self._last_events_time = datetime.utcnow() - intake_host = os.getenv("INTAKE_URL", self.configuration.intake_server) - batch_api = urljoin(intake_host, "batch") + if intake_server := self.configuration.intake_server: + batch_api = urljoin(intake_server, "batch") + else: + batch_api = urljoin(self.intake_url, "batch") result_ids = [] diff --git a/sekoia_automation/connector/__init__.py b/sekoia_automation/connector/__init__.py index 69f4267..9a6a882 100644 --- a/sekoia_automation/connector/__init__.py +++ b/sekoia_automation/connector/__init__.py @@ -1,4 +1,3 @@ -import os import uuid from abc import ABC from collections.abc import Generator, Sequence @@ -36,7 +35,7 @@ class DefaultConnectorConfiguration(BaseModel): - intake_server: str = "https://intake.sekoia.io" + intake_server: str | None = None intake_key: str @@ -171,8 +170,10 @@ def push_events_to_intakes( # Reset the consecutive error count self._error_count = 0 self._last_events_time = datetime.utcnow() - intake_host = os.getenv("INTAKE_URL", self.configuration.intake_server) - batch_api = urljoin(intake_host, "batch") + if intake_server := self.configuration.intake_server: + batch_api = urljoin(intake_server, "batch") + else: + batch_api = urljoin(self.intake_url, "batch") # Dict to collect event_ids for the API collect_ids: dict[int, list] = {} diff --git a/sekoia_automation/module.py b/sekoia_automation/module.py index 3c3eeda..6ff4f6b 100644 --- a/sekoia_automation/module.py +++ b/sekoia_automation/module.py @@ -322,6 +322,7 @@ class ModuleItem(ABC): CALLBACK_URL_FILE_NAME = "url_callback" SECRETS_URL_FILE_NAME = "url_secrets" LOGS_URL_FILE_NAME = "url_logs" + INTAKE_URL_FILE_NAME = "intake_url" name: str | None = None description: str | None = None @@ -419,6 +420,10 @@ def secrets_url(self) -> str: except FileNotFoundError: return self.callback_url.replace("/callback", "/secrets") + @cached_property + def intake_url(self) -> str: + return self.module.load_config(self.INTAKE_URL_FILE_NAME) + @property def _headers(self) -> dict[str, str]: return {"Authorization": f"Bearer {self.token}"} diff --git a/tests/aio/test_connector.py b/tests/aio/test_connector.py index 14c9e2d..e03e476 100644 --- a/tests/aio/test_connector.py +++ b/tests/aio/test_connector.py @@ -1,6 +1,7 @@ """Test async connector.""" -from unittest.mock import Mock, patch +import json +from unittest.mock import Mock, mock_open, patch from urllib.parse import urljoin import pytest @@ -91,7 +92,8 @@ async def test_async_connector_client_session(async_connector: DummyAsyncConnect DummyAsyncConnector.set_rate_limiter(None) other_instance.set_client_session(None) - +@patch("builtins.open", new_callable=mock_open, + read_data=json.dumps({"intake_url": "https://intake.sekoia.io"})) @pytest.mark.asyncio async def test_async_connector_push_single_event( async_connector: DummyAsyncConnector, faker: Faker @@ -116,7 +118,7 @@ async def test_async_connector_push_single_event( single_event_id = faker.uuid4() - request_url = urljoin(async_connector.configuration.intake_server, "batch") + request_url = urljoin("https://intake.sekoia.io", "batch") with aioresponses() as mocked_responses: mocked_responses.post( @@ -155,7 +157,7 @@ async def test_async_connector_push_multiple_events( single_event_id = faker.uuid4() - request_url = urljoin(async_connector.configuration.intake_server, "batch") + request_url = urljoin("https://intake.sekoia.io", "batch") with ( aioresponses() as mocked_responses, @@ -200,7 +202,7 @@ async def test_async_connector_raise_error( stop=stop_after_attempt(1), ) - request_url = urljoin(async_connector.configuration.intake_server, "batch") + request_url = urljoin("https://intake.sekoia.io", "batch") with ( aioresponses() as mocked_responses, diff --git a/tests/connectors/test_connector.py b/tests/connectors/test_connector.py index 5ed868e..14f91aa 100644 --- a/tests/connectors/test_connector.py +++ b/tests/connectors/test_connector.py @@ -1,5 +1,6 @@ +import json import os -from unittest.mock import Mock, PropertyMock, patch +from unittest.mock import Mock, PropertyMock, mock_open, patch import pytest from tenacity import Retrying, stop_after_attempt, wait_none @@ -104,6 +105,8 @@ def test_chunk_events_discard_too_long_message(test_connector): assert test_connector.log.called +@patch("builtins.open", new_callable=mock_open, + read_data=json.dumps({"intake_url": "https://intake.sekoia.io/batch"})) def test_push_event_to_intake_with_2_events(test_connector, mocked_trigger_logs): url = "https://intake.sekoia.io/batch" mocked_trigger_logs.post(url, json={"event_ids": ["001", "002"]}) diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 00c1319..9c47576 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -61,6 +61,13 @@ def test_secrets_url(): mock.assert_called_with(trigger.SECRETS_URL_FILE_NAME) +def test_intake_url(): + trigger = DummyTrigger() + with patch.object(Module, "load_config", return_value="intake") as mock: + assert trigger.intake_url == "intake" + mock.assert_called_with(trigger.INTAKE_URL_FILE_NAME) + + def test_logs_url(): trigger = DummyTrigger() with patch.object(Module, "load_config", return_value="logs") as mock: