Skip to content

Commit

Permalink
Merge pull request #10 from SEKOIA-IO/feat/custom_tip
Browse files Browse the repository at this point in the history 
feat: add support for custom TIP
  • Loading branch information
@gbossert
gbossert authored May 15, 2022
2 parents 18a9d85 + c4ebbf0 commit 805053a
Show file tree
Hide file tree
Showing 7 changed files with 65 additions and 17 deletions.
1 change: 1 addition & 0 deletions sekoia.io/README/inputs.conf.spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ python.version = python3
api_key = <value>
feed_id = <value>
proxy_url = <value>
api_root_url = <value>
2 changes: 1 addition & 1 deletion sekoia.io/app.manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"id": {
"group": null,
"name": "sekoia.io",
"version": "1.1.2"
"version": "1.2.1"
},
"author": [
{
Expand Down
10 changes: 6 additions & 4 deletions sekoia.io/appserver/static/javascript/views/setup_page.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ define(["backbone", "jquery", "splunkjs/splunk"], function (Backbone, jquery, sp
jquery("#api_key").val(input["api_key"]);
jquery("#feed_id").val(input["feed_id"]);
jquery("#proxy_url").val(input["proxy_url"]);
jquery("#api_root_url").val(input["api_root_url"]);
}

Object.keys(this.current_settings["lookups"]).forEach((name) => {
Expand Down Expand Up @@ -125,7 +126,7 @@ define(["backbone", "jquery", "splunkjs/splunk"], function (Backbone, jquery, sp


// Extract Feed Settings
feed_id = values[1].value.trim();
var feed_id = values[1].value.trim();
if (feed_id == "") {
feed_id = "d6092c37-d8d7-45c3-8aff-c4dc26030608";
}
Expand All @@ -134,15 +135,16 @@ define(["backbone", "jquery", "splunkjs/splunk"], function (Backbone, jquery, sp
const feed_settings = {
api_key: "<nothing to see here>",
feed_id: feed_id,
proxy_url: values[2].value.trim(),
api_root_url: values[2].value.trim(),
proxy_url: values[3].value.trim(),
};

// Extract Lookups
const lookups = new Array();
var i = 0;

while (i * 3 + 3 < values.length) {
const offset = 3 + i * 3;
while (i * 3 + 4 < values.length) {
const offset = 4 + i * 3;
lookups.push({
type: values[offset].value.trim(),
query: values[offset + 1].value.trim(),
Expand Down
8 changes: 8 additions & 0 deletions sekoia.io/appserver/static/javascript/views/setup_page_template.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ function get_template() {
You can find more information on feeds in the <a href="https://docs.sekoia.io/intelligence_center/api/#feeds" target="_blank">documentation</a>.
</p>
<label>
SEKOIA.IO API URL
<input id="api_root_url" type="text" name="api_root_url" placeholder="https://api.sekoia.io" />
</label>
<p class="hint">
(optional) URL root of your SEKOIA.IO TIP API (e.g. https://api.sekoia.io or https://my.sekoiaio.tip.local/api)
</p>
<label>
Proxy URL
<input id="proxy_url" type="text" name="proxy_url" placeholder="http://[username:password@]host:port" />
Expand Down
52 changes: 42 additions & 10 deletions sekoia.io/bin/sekoia_indicators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@

import requests # noqa: E402
import six # noqa: E402
from splunklib.binding import HTTPError # noqa: E402
import splunklib.client as client # noqa: E402
from splunklib.binding import HTTPError # noqa: E402
from splunklib.modularinput import Argument, Scheme, Script # noqa: E402
from stix2patterns.pattern import Pattern # noqa: E402

SEKOIAIO_REALM = "sekoiaio_realm"
MASK = "<nothing to see here>"
DEFAULT_FEED = "d6092c37-d8d7-45c3-8aff-c4dc26030608"
BASE_URL = "https://api.sekoia.io/v2/inthreat/"
BASE_URL = "https://api.sekoia.io"
LIMIT = 300
COLLECTION_NAME = "sekoia_iocs_{}"
SUPPORTED_TYPES = {
Expand Down Expand Up @@ -116,15 +116,17 @@ def _mask_api_key(self, session_key, input_name, feed_id, ew):
item.update(**kwargs).refresh()
ew.log(ew.INFO, "Input succesfully updated")

def get_indicators(self, feed_id, api_key, proxy_url=None, cursor=None, ew=None):
def get_indicators(
self, feed_id, api_key, api_root_url=None, proxy_url=None, cursor=None, ew=None
):
"""
Fetch and yelds indicators from the configuration feed
"""

if ew:
ew.log(
ew.INFO,
f"Fetch indicators from feed_id={feed_id}",
f"Fetch indicators from feed_id={feed_id} (api_root_url={api_root_url})",
)

proxies = None
Expand All @@ -136,9 +138,13 @@ def get_indicators(self, feed_id, api_key, proxy_url=None, cursor=None, ew=None)
ew.DEBUG, f"Configure network proxy access with proxy={proxy_url}"
)

url_root = BASE_URL
if api_root_url:
url_root = api_root_url

url = urljoin(
BASE_URL,
"collections",
url_root+"/",
"v2/inthreat/collections",
feed_id,
"objects?match[type]=indicator&limit={}".format(LIMIT),
)
Expand Down Expand Up @@ -170,7 +176,7 @@ def get_indicators(self, feed_id, api_key, proxy_url=None, cursor=None, ew=None)
break

# Convert a STIX 2.1 Indicator to Splunk key-value objects
def indicator_to_kv(self, indicator):
def indicator_to_kv(self, indicator, api_root_url):
results = defaultdict(list)
pattern_type = indicator.get("pattern_type")

Expand Down Expand Up @@ -232,9 +238,20 @@ def indicator_to_kv(self, indicator):
# Unfortunately, Splunk Accelerated Fields
# cannot be larger than 1024.
if len(value.strip("'")) <= 1024:

if not api_root_url:
server_root_url = "https://app.sekoia.io"
else:
server_root_url = api_root_url
if server_root_url.endswith("/api"):
server_root_url = server_root_url[:-4]
elif server_root_url.endswith("/api/"):
server_root_url = server_root_url[:-5]

result = {
"_key": value.strip("'"),
"indicator_id": indicator["id"],
"server_root_url": server_root_url,
"valid_until": indicator.get("valid_until"),
}

Expand Down Expand Up @@ -274,15 +291,15 @@ def revoke_indicator(self, kv_objects):
except Exception:
pass

def store_indicators(self, indicators, ew):
def store_indicators(self, indicators, ew, api_root_url):
"""
Stores the indicators in the Splunk KV-Stores
"""
objects = defaultdict(list)
now = datetime.utcnow()

for indicator in indicators:
kv_objects = self.indicator_to_kv(indicator)
kv_objects = self.indicator_to_kv(indicator, api_root_url)

if indicator.get("revoked", False):
self.revoke_indicator(kv_objects)
Expand Down Expand Up @@ -333,6 +350,15 @@ def get_scheme(self):
feed_id.required_on_edit = False
scheme.add_argument(feed_id)

# api root url
api_root_url = Argument("api_root_url")
api_root_url.title = "SEKOIA.IO API URL"
api_root_url.data_type = Argument.data_type_string
api_root_url.description = "(optional) URL root of your SEKOIA.IO TIP API (e.g. https://api.sekoia.io or https://my.sekoiaio.tip.local/api)"
api_root_url.required_on_create = False
api_root_url.required_on_edit = False
scheme.add_argument(api_root_url)

# proxy url
proxy_url = Argument("proxy_url")
proxy_url.title = "Proxy URL"
Expand Down Expand Up @@ -360,13 +386,17 @@ def validate_input(self, definition):
try:
feed_id = definition.parameters["feed_id"] or DEFAULT_FEED
api_key = definition.parameters["api_key"]
api_root_url = definition.parameters.get("api_root_url")
proxy_url = definition.parameters.get("proxy_url")
if api_key == MASK:
return True

(cursor, indicators) = next(
self.get_indicators(
feed_id=feed_id,
api_key=api_key,
api_root_url=api_root_url,
proxy_url=proxy_url,
)
)
except requests.exceptions.HTTPError as http_error:
Expand Down Expand Up @@ -400,6 +430,7 @@ def stream_events(self, inputs, ew):
try:
feed_id = input_item.get("feed_id", "") or DEFAULT_FEED
proxy_url = input_item.get("proxy_url")
api_root_url = input_item.get("api_root_url")
api_key = self._get_api_key_from_secured_storage(
feed_id=feed_id
)
Expand All @@ -408,11 +439,12 @@ def stream_events(self, inputs, ew):
for cursor, indicators in self.get_indicators(
feed_id=feed_id,
api_key=api_key,
api_root_url=api_root_url,
proxy_url=proxy_url,
cursor=cursor,
ew=ew,
):
self.store_indicators(indicators, ew)
self.store_indicators(indicators, ew, api_root_url)
self.store_cursor(inputs, feed_id, cursor)

except Exception:
Expand Down
2 changes: 1 addition & 1 deletion sekoia.io/default/app.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ setup_view = setup
[launcher]
author = [email protected]
description = Search your logs with Indicators of Compromise (IoCs) from SEKOIA.IO.
version = 1.1.2
version = 1.2.1

[package]
check_for_updates = 1
Expand Down
7 changes: 6 additions & 1 deletion sekoia.io/default/data/ui/views/dashboard.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,12 @@
<option name="refresh.display">progressbar</option>
<drilldown>
<condition field="matched_ioc">
<link target="_blank">https://app.sekoia.io/inthreatv2/objects/$row.indicator_id$</link>
<condition match="$row.server_root_url$ != &quot;&quot;">
<link target="_blank">$row.server_root_url$/inthreatv2/objects/$row.indicator_id$</link>
</condition>
<condition match="$row.server_root_url$ == &quot;&quot;">
<link target="_blank">https://app.sekoia.io/inthreatv2/objects/$row.indicator_id$</link>
</condition>
</condition>
</drilldown>
</table>
Expand Down

0 comments on commit 805053a

Please sign in to comment.