Skip to content

Commit

Permalink
feat: New ActivityAuditClientV1 to query Activity Audit events (#197)
Browse files Browse the repository at this point in the history
* feat: New ActivityAuditClientV1 to query Activity Audit events
* chore: Deprecate old command audit methods
* refactor: Import datetime package directly
* fix(lint): Solve linting problems
* fix(ci): Change the image to scan to quay.io/sysdig/agent
  • Loading branch information
tembleking authored Jul 5, 2021
1 parent 5a93a4c commit ff1f2d6
Show file tree
Hide file tree
Showing 6 changed files with 219 additions and 20 deletions.
4 changes: 4 additions & 0 deletions sdcclient/_secure.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,8 @@ def list_commands_audit(self, from_sec=None, to_sec=None, scope_filter=None, com
'''**Description**
List the commands audit.
**DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time.
**Arguments**
- from_sec: the start of the timerange for which to get commands audit.
- end_sec: the end of the timerange for which to get commands audit.
Expand Down Expand Up @@ -528,6 +530,8 @@ def get_command_audit(self, id, metrics=[]):
'''**Description**
Get a command audit.
**DEPRECATED**: Use sdcclient.secure.ActivityAuditClientV1 instead. This is maintained for old on-prem versions, but will be removed over time.
**Arguments**
- id: the id of the command audit to get.
Expand Down
8 changes: 5 additions & 3 deletions sdcclient/secure/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from ._activity_audit_v1 import ActivityAuditClientV1, ActivityAuditDataSource
from ._falco_rules_files_old import FalcoRulesFilesClientOld
from ._policy_events_old import PolicyEventsClientOld
from ._policy_events_v1 import PolicyEventsClientV1
from ._policy_v2 import PolicyClientV2, policy_action_pause, policy_action_stop, policy_action_kill, \
policy_action_capture
from ._policy_v2 import policy_action_capture, policy_action_kill, policy_action_pause, policy_action_stop, \
PolicyClientV2

__all__ = ["PolicyEventsClientOld", "PolicyEventsClientV1", "FalcoRulesFilesClientOld",
"PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture"]
"PolicyClientV2", "policy_action_pause", "policy_action_stop", "policy_action_kill", "policy_action_capture",
"ActivityAuditClientV1", "ActivityAuditDataSource"]
137 changes: 137 additions & 0 deletions sdcclient/secure/_activity_audit_v1.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
import datetime

from sdcclient._common import _SdcCommon


class ActivityAuditDataSource:
CMD = "command"
NET = "connection"
KUBE_EXEC = "kubernetes"
FILE = "fileaccess"


_seconds_to_nanoseconds = 10 ** 9


class ActivityAuditClientV1(_SdcCommon):
def __init__(self, token="", sdc_url='https://secure.sysdig.com', ssl_verify=True, custom_headers=None):
super(ActivityAuditClientV1, self).__init__(token, sdc_url, ssl_verify, custom_headers)
self.product = "SDS"

def list_events(self, from_date=None, to_date=None, scope_filter=None, limit=0,
data_sources=None):
"""
List the events in the Activity Audit.
Args:
from_date (datetime.datetime): the start of the time range from which to get events. The default value is yesterday.
to_date (datetime.datetime): the end of the time range from which to get events. The default value is now.
scope_filter (List): a list of Sysdig Monitor-like filter (e.g `processName in ("ubuntu")`).
limit (int): max number of events to retrieve. A limit of 0 or negative will retrieve all events.
data_sources (List): a list of data sources to retrieve events from. None or an empty list retrieves all events.
Examples:
>>> client = ActivityAuditClientV1(token=SECURE_TOKEN)
>>>
>>> now = datetime.datetime.utcnow()
>>> three_days_ago = now - datetime.timedelta(days=3)
>>> max_event_number_retrieved = 50
>>> data_sources = [ActivityAuditDataSource.CMD, ActivityAuditDataSource.KUBE_EXEC]
>>>
>>> ok, events = client.list_events(from_date=three_days_ago,
>>> to_date=now,
>>> limit=max_event_number_retrieved,
>>> data_sources=data_sources)
Returns:
A list of event objects from the Activity Audit.
"""
number_of_events_per_query = 50

if from_date is None:
from_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
if to_date is None:
to_date = datetime.datetime.utcnow()

filters = scope_filter if scope_filter else []
if data_sources:
quoted_data_sources = [f'"{data_source}"' for data_source in data_sources]
data_source_filter = f'type in ({",".join(quoted_data_sources)})'
filters.append(data_source_filter)

query_params = {
"from": int(from_date.timestamp()) * _seconds_to_nanoseconds,
"to": int(to_date.timestamp()) * _seconds_to_nanoseconds,
"limit": number_of_events_per_query,
"filter": " and ".join(filters),
}

res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify,
params=query_params)
ok, res = self._request_result(res)
if not ok:
return False, res

events = []

# Pagination required by Secure API
while "page" in res and \
"total" in res["page"] and \
res["page"]["total"] > number_of_events_per_query:
events = events + res["data"]

if 0 < limit < len(events):
events = events[0:limit - 1]
break

paginated_query_params = {
"limit": number_of_events_per_query,
"filter": " and ".join(filters),
"cursor": res["page"]["prev"]
}

res = self.http.get(self.url + '/api/v1/activityAudit/events', headers=self.hdrs, verify=self.ssl_verify,
params=paginated_query_params)
ok, res = self._request_result(res)
if not ok:
return False, res
else:
events = events + res["data"]

return True, events

def list_trace(self, traceable_event):
"""
Lists the events from an original traceable event.
Args:
traceable_event(object): an event retrieved from the list_events method. The event must be traceable,
this is, it must have the "traceable" key as true.
Examples:
>>> client = ActivityAuditClientV1(token=SECURE_TOKEN)
>>>
>>> ok, events = client.list_events()
>>> if not ok:
>>> return
>>> traceable_events = [event for event in events if event["traceable"]]
>>>
>>> ok, trace = client.list_trace(traceable_events[0])
>>> if not ok:
>>> return
>>>
>>> for event in trace:
>>> print(event)
Returns:
All the related events that are the trace of the given event.
"""
if not traceable_event or not traceable_event["traceable"]:
return False, "a traceable event must be provided"

endpoint = f'/api/v1/activityAudit/events/{traceable_event["type"]}/{traceable_event["id"]}/trace'
res = self.http.get(self.url + endpoint, headers=self.hdrs, verify=self.ssl_verify)
ok, res = self._request_result(res)
if not ok:
return False, res
return True, res["data"]
58 changes: 58 additions & 0 deletions specs/secure/activitylog_v1_spec.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import datetime
import os

from expects import be_above, be_empty, contain, expect, have_keys, have_len
from mamba import _it, before, context, description, it

from sdcclient.secure import ActivityAuditClientV1 as ActivityAuditClient, ActivityAuditDataSource
from specs import be_successful_api_call

with description("Activity Audit v1", "integration") as self:
with before.all:
self.client = ActivityAuditClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
token=os.getenv("SDC_SECURE_TOKEN"))

with it("is able to list the most recent commands with the default parameters"):
ok, res = self.client.list_events()

expect((ok, res)).to(be_successful_api_call)
expect(res).to_not(be_empty)

with context("when listing the most recent commands with a limit of 5"):
with it("retrieves the 5 events"):
ok, res = self.client.list_events(limit=5)

expect((ok, res)).to(be_successful_api_call)
expect(res).to_not(have_len(5))

with context("when listing the events from the last 3 days"):
with it("retrieves all the events"):
three_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=3)
ok, res = self.client.list_events(from_date=three_days_ago)

expect((ok, res)).to(be_successful_api_call)
expect(res).to_not(be_empty)

with context("when listing events from a specific type"):
with it("retrieves the events of this event type only"):
ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.CMD])

expect((ok, res)).to(be_successful_api_call)
expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD)))
expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.KUBE_EXEC)))
expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.FILE)))
expect(res).to_not(contain(have_keys(type=ActivityAuditDataSource.NET)))

with context("when retrieving the inner events of a traceable event"):
with _it("retrieves the trace of these events"):
ok, res = self.client.list_events(data_sources=[ActivityAuditDataSource.KUBE_EXEC])
expect((ok, res)).to(be_successful_api_call)

expect(res).to(contain(have_keys(traceable=True)))

traceable_events = [event for event in res if event["traceable"]]
ok, res = self.client.list_trace(traceable_events[0])

expect((ok, res)).to(be_successful_api_call)
expect(res).to(contain(have_keys(type=ActivityAuditDataSource.CMD)))
expect(res).to(have_len(be_above(0))) # Not using be_empty, because we want to ensure this is a list
17 changes: 7 additions & 10 deletions specs/secure/scanning/policy_evaluation_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,50 +11,47 @@
with before.all:
self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
token=os.getenv("SDC_SECURE_TOKEN"))
self.image_name = "quay.io/sysdig/agent:latest"

with it("is able to retrieve the results for all the policies"):
image_name = "alpine:latest"
ok, res = self.client.get_image_scanning_results(image_name)
ok, res = self.client.get_image_scanning_results(self.image_name)

expect((ok, res)).to(be_successful_api_call)
expect(res).to(
have_keys("image_digest", "image_id", "stop_results",
total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0),
last_evaluation=be_an(datetime),
status="pass", image_tag="docker.io/alpine:latest",
status="pass", image_tag=self.image_name,
policy_id="*", policy_name="All policies",
warn_results=not_(be_empty))
)

with it("is able to retrieve the results for the default policy"):
image_name = "alpine:latest"
policy_id = "default"
ok, res = self.client.get_image_scanning_results(image_name, policy_id)
ok, res = self.client.get_image_scanning_results(self.image_name, policy_id)

expect((ok, res)).to(be_successful_api_call)
expect(res).to(
have_keys("image_digest", "image_id", "stop_results",
total_warn=be_above_or_equal(0), total_stop=be_above_or_equal(0),
last_evaluation=be_an(datetime),
status="pass", image_tag="docker.io/alpine:latest",
status="pass", image_tag=self.image_name,
policy_id="default", policy_name="DefaultPolicy",
warn_results=not_(be_empty))
)

with context("but the image has not been scanned yet"):
with it("returns an error saying that the image has not been found"):
image_name = "unknown_image"
ok, res = self.client.get_image_scanning_results(image_name)
ok, res = self.client.get_image_scanning_results("unknown_image")

expect((ok, res)).to_not(be_successful_api_call)
expect(res).to(equal("could not retrieve image digest for the given image name, "
"ensure that the image has been scanned"))

with context("but the provided policy id does not exist"):
with it("returns an error saying that the policy id is not found"):
image_name = "alpine"
policy_id = "unknown_policy_id"
ok, res = self.client.get_image_scanning_results(image_name, policy_id)
ok, res = self.client.get_image_scanning_results(self.image_name, policy_id)

expect((ok, res)).to_not(be_successful_api_call)
expect(res).to(equal("the specified policy ID doesn't exist"))
15 changes: 8 additions & 7 deletions specs/secure/scanning/query_image_content_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,40 +10,41 @@
with before.each:
self.client = SdScanningClient(sdc_url=os.getenv("SDC_SECURE_URL", "https://secure.sysdig.com"),
token=os.getenv("SDC_SECURE_TOKEN"))
self.image_to_scan = "quay.io/sysdig/agent:latest"

with it("is able to retrieve the OS contents"):
ok, res = self.client.query_image_content("alpine:latest", "os")
ok, res = self.client.query_image_content(self.image_to_scan, "os")

expect((ok, res)).to(be_successful_api_call)
expect(res["content"]).to(contain(have_keys("license", "origin", "package", "size", "type", "version")))
expect(res["content_type"]).to(equal("os"))

with it("is able to retrieve the npm contents"):
ok, res = self.client.query_image_content("alpine:latest", "npm")
ok, res = self.client.query_image_content(self.image_to_scan, "npm")

expect((ok, res)).to(be_successful_api_call)
expect(res["content_type"]).to(equal("npm"))

with it("is able to retrieve the gem contents"):
ok, res = self.client.query_image_content("alpine:latest", "gem")
ok, res = self.client.query_image_content(self.image_to_scan, "gem")

expect((ok, res)).to(be_successful_api_call)
expect(res["content_type"]).to(equal("gem"))

with it("is able to retrieve the python contents"):
ok, res = self.client.query_image_content("alpine:latest", "python")
ok, res = self.client.query_image_content(self.image_to_scan, "python")

expect((ok, res)).to(be_successful_api_call)
expect(res["content_type"]).to(equal("python"))

with it("is able to retrieve the java contents"):
ok, res = self.client.query_image_content("alpine:latest", "java")
ok, res = self.client.query_image_content(self.image_to_scan, "java")

expect((ok, res)).to(be_successful_api_call)
expect(res["content_type"]).to(equal("java"))

with it("is able to retrieve the files contents"):
ok, res = self.client.query_image_content("alpine:latest", "files")
ok, res = self.client.query_image_content(self.image_to_scan, "files")

expect((ok, res)).to(be_successful_api_call)
expect(res["content"]).to(
Expand All @@ -52,7 +53,7 @@

with context("when the type is not in the supported list"):
with it("returns an error indicating the type is incorrect"):
ok, res = self.client.query_image_content("alpine:latest", "Unknown")
ok, res = self.client.query_image_content(self.image_to_scan, "Unknown")

expect((ok, res)).not_to(be_successful_api_call)
expect(res).to(equal(
Expand Down

0 comments on commit ff1f2d6

Please sign in to comment.