Merge pull request #119 from SEKOIA-IO/feat/IncreaseAllowedEventSize

Increase the allowed size for events

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Increase the allowed size for events to 250 kio (instead of 64 kio)
+
 ## [1.12.2] - 2024-03-26
 
 ### Fixed

sekoia_automation/connector/__init__.py

@@ -287,7 +287,7 @@ def _chunk_events(self, events: Sequence) -> Generator[list[Any], None, None]:
         if nb_discarded_events > 0:
             self.log(
                 message=f"{nb_discarded_events} too long events "
-                "were discarded (length > 64kb)"
+                "were discarded (length > 250kb)"
             )
 
     def forward_events(self, events) -> None:

sekoia_automation/constants.py

@@ -7,4 +7,4 @@
 # and the escapement of the characters
 CHUNK_BYTES_FACTOR = 0.9
 CHUNK_BYTES_MAX_SIZE = int(INTAKE_PAYLOAD_BYTES_MAX_SIZE * CHUNK_BYTES_FACTOR)
-EVENT_BYTES_MAX_SIZE = 64 * 1024
+EVENT_BYTES_MAX_SIZE = 250000

tests/connectors/test_connector.py

@@ -88,14 +88,18 @@ def test_chunk_events(test_connector):
 
 
 def test_chunk_events_exceed_size(test_connector):
-    events_a = ["a" * EVENT_BYTES_MAX_SIZE] * int(
-        CHUNK_BYTES_MAX_SIZE / EVENT_BYTES_MAX_SIZE
-    )
-    events_b = ["b"]
-    events = events_a + events_b
+    # list of events that fill one chunk (must pass)
+    event_bytes = 64 * 1024
+    events_a = ["a" * event_bytes] * int(CHUNK_BYTES_MAX_SIZE / event_bytes)
+    # An event that exceed the expected max size for events (mustn't pass)
+    events_b = ["b" * (EVENT_BYTES_MAX_SIZE + 1)]
+
+    # An event that doesn't exceed any limit (must pass)
+    events_c = ["c"]
+    events = events_a + events_b + events_c
     chunks = list(test_connector._chunk_events(events=events))
     assert len(chunks) == 2
-    assert chunks == [events_a, events_b]
+    assert chunks == [events_a, events_c]
 
 
 def test_chunk_events_discard_too_long_message(test_connector):