Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cybereason - add checkpoint #1197

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions Cybereason/cybereason_modules/connector_pull_events.py
Original file line number Diff line number Diff line change
@@ -2,13 +2,15 @@
import time
from collections import defaultdict
from collections.abc import Generator
from datetime import timedelta
from functools import cached_property
from posixpath import join as urljoin
from threading import Event
from typing import Any
from posixpath import join as urljoin

import orjson
import requests
from sekoia_automation.checkpoint import CheckpointTimestamp, TimeUnit
from sekoia_automation.connector import Connector, DefaultConnectorConfiguration

from cybereason_modules import CybereasonModule
@@ -45,7 +47,13 @@ class CybereasonEventConnector(Connector):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.from_date: int = (int(time.time()) - 60) * 1000 # milliseconds
self.checkpoint = CheckpointTimestamp(
path=self._data_path,
time_unit=TimeUnit.MILLISECOND,
start_at=timedelta(minutes=1),
ignore_older_than=timedelta(hours=1),
)
self.from_date: int = self.checkpoint.offset
self._stop_event = Event() # Event to notify we must stop the thread

# Register signal to terminate thread
@@ -265,15 +273,9 @@ def fetch_last_events(self) -> Generator[dict[str, Any], None, None]:
Fetch the last malops from the Cybereason API
"""
from_date = self.from_date
now = int(time.time()) * 1000 # milliseconds

# We don't retrieve messages older than one hour
one_hour_ago = now - 3600000
if from_date < one_hour_ago:
from_date = one_hour_ago

# compute the ending time to retrieve malops (Currently, now)
to_date = now
to_date = int(time.time()) * 1000 # milliseconds

# fetch malops for the timerange
next_malops = self.fetch_malops(from_date, to_date)
@@ -316,6 +318,7 @@ def fetch_last_events(self) -> Generator[dict[str, Any], None, None]:
# save the most recent date and compute the lag
if most_recent_date_seen > self.from_date:
self.from_date = most_recent_date_seen
self.checkpoint.offset = self.from_date
current_lag = int(time.time() - (most_recent_date_seen / 1000))
EVENTS_LAG.labels(intake_key=self.configuration.intake_key).set(current_lag)

4 changes: 2 additions & 2 deletions Cybereason/cybereason_modules/connector_pull_events_new.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
from typing import Any
from posixpath import join as urljoin
from typing import Any

import requests

from cybereason_modules.connector_pull_events import CybereasonEventConnector
from cybereason_modules.constants import MALOP_GET_ALL_ENDPOINT
from cybereason_modules.logging import get_logger
from cybereason_modules.exceptions import InvalidResponse
from cybereason_modules.logging import get_logger

logger = get_logger()

2 changes: 1 addition & 1 deletion Cybereason/cybereason_modules/metrics.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from prometheus_client import Counter, Histogram, Gauge
from prometheus_client import Counter, Gauge, Histogram

# Declare google prometheus metrics
prom_namespace_cybereason = "symphony_module_cybereason"
272 changes: 241 additions & 31 deletions Cybereason/poetry.lock

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions Cybereason/pyproject.toml
Original file line number Diff line number Diff line change
@@ -6,9 +6,9 @@ authors = []

[tool.poetry.dependencies]
python = ">=3.10,<3.12"
sekoia-automation-sdk = "^1.13.0"
beautifulsoup4 = "^4.11.1"
structlog = "^23.1.0"
sekoia-automation-sdk = "^1.18.1"
beautifulsoup4 = "^4.12.3"
structlog = "^24.4.0"

[tool.poetry.dev-dependencies]
pytest = "*"
11 changes: 10 additions & 1 deletion Cybereason/tests/test_pull_events_connector.py
Original file line number Diff line number Diff line change
@@ -25,6 +25,15 @@ def patch_time(fake_time):
yield mock_time


@pytest.fixture
def patch_datetime(fake_time):
with patch("sekoia_automation.checkpoint.datetime") as mock_datetime:
mock_datetime.now.return_value = datetime.fromtimestamp(fake_time, tz=timezone.utc)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
mock_datetime.fromtimestamp = lambda ts: datetime.fromtimestamp(ts)
yield mock_datetime


@pytest.fixture
def mock_cybereason_api():
with requests_mock.Mocker() as mock:
@@ -174,7 +183,7 @@ def edr_suspicions():


@pytest.fixture
def trigger(symphony_storage, patch_time):
def trigger(symphony_storage, patch_time, patch_datetime):
module = CybereasonModule()
trigger = CybereasonEventConnector(module=module, data_path=symphony_storage)
trigger.log = MagicMock()
16 changes: 12 additions & 4 deletions Cybereason/tests/test_pull_events_connector_new.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import copy
import os
from datetime import datetime, timedelta, timezone
from datetime import datetime, timezone
from typing import Any
from unittest.mock import MagicMock, Mock, patch
from unittest.mock import MagicMock, patch

import pytest
import requests_mock
@@ -25,6 +24,15 @@ def patch_time(fake_time):
yield mock_time


@pytest.fixture
def patch_datetime(fake_time):
with patch("sekoia_automation.checkpoint.datetime") as mock_datetime:
mock_datetime.now.return_value = datetime.fromtimestamp(fake_time, tz=timezone.utc)
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
mock_datetime.fromtimestamp = lambda ts: datetime.fromtimestamp(ts)
yield mock_datetime


@pytest.fixture
def mock_cybereason_api():
with requests_mock.Mocker() as mock:
@@ -174,7 +182,7 @@ def edr_suspicions():


@pytest.fixture
def trigger(symphony_storage, patch_time):
def trigger(symphony_storage, patch_time, patch_datetime):
module = CybereasonModule()
trigger = CybereasonEventConnectorNew(module=module, data_path=symphony_storage)
trigger.log = MagicMock()