Skip to content

Commit

Permalink
Merge pull request #3 from SEKOIA-IO/feat-valid-indicators-and-persis…
Browse files Browse the repository at this point in the history
…tence

App improvements
  • Loading branch information
gaelmuller authored Dec 17, 2020
2 parents 848a337 + 10149f7 commit 7f43014
Show file tree
Hide file tree
Showing 257 changed files with 38 additions and 18 deletions.
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# Splunk local configuration files
local/
metadata/local.meta
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@ For each lookup, a saved search is automatically created and scheduled to run on

Sightings are stored in the `sekoia_alerts` KV-store and listed on the home Dashboard.

Clicking on the `matched_ioc` will open the Intelligence Center to see context aroung matched indicator.
Clicking on the `matched_ioc` will open the Intelligence Center to see context around matched indicator.
1 change: 0 additions & 1 deletion README/inputs.conf.spec
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
[sekoia_indicators://<name>]

interval = 1800
api_key = <value>
feed_id = <value>
45 changes: 29 additions & 16 deletions bin/sekoia_indicators.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@
}


def from_rfc3339(date_string):
try:
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ")
except ValueError:
return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ")


class SEKOIAIndicators(Script):
def __init__(self):
self._splunk = None
Expand Down Expand Up @@ -169,22 +176,26 @@ def revoke_indicator(self, kv_objects):
for obj in objects:
try:
self.get_kvstore(ioc_type).delete_by_id(obj["_key"])
# TODO: improve error handling here
except Exception:
pass

# Store indicators in Splunk KV-Stores
def store_indicators(self, indicators):
objects = defaultdict(list)
now = datetime.utcnow()

for indicator in indicators:
kv_objects = self.indicator_to_kv(indicator)

if indicator.get("revoked", False):
self.revoke_indicator(kv_objects)
else:
for ioc_type, dicts in six.iteritems(kv_objects):
objects[ioc_type] += dicts
# Only import IOCs with a Valid Until date set
elif indicator.get("valid_until"):
# Ignore expired indicators
valid_until = from_rfc3339(indicator["valid_until"])
if valid_until > now:
for ioc_type, dicts in six.iteritems(kv_objects):
objects[ioc_type] += dicts

for ioc_type, batch in six.iteritems(objects):
self.get_kvstore(ioc_type).batch_save(*batch)
Expand Down Expand Up @@ -225,20 +236,22 @@ def validate_input(self, validation_definition):

# Method called by Splunk to get new events
def stream_events(self, inputs, ew):
try:
self._splunk = client.connect(
token=self._input_definition.metadata["session_key"], owner="nobody"
)

for indicators in self.get_indicators(inputs):
self.store_indicators(indicators)
except Exception:
exception = traceback.format_exc()
while True:
try:
self._splunk = client.connect(
token=self._input_definition.metadata["session_key"], owner="nobody"
)

for line in exception.splitlines():
print("ERROR {}".format(line), file=sys.stderr)
for indicators in self.get_indicators(inputs):
self.store_indicators(indicators)
except Exception:
exception = traceback.format_exc()

raise
for line in exception.splitlines():
print("ERROR {}".format(line), file=sys.stderr)
finally:
print("INFO Done fetching indicators, sleeping for 10 minutes.")
time.sleep(600)


if __name__ == "__main__":
Expand Down
Binary file removed lib/antlr4/BufferedTokenStream.pyc
Binary file not shown.
Binary file removed lib/antlr4/CodePoints.pyc
Binary file not shown.
Binary file removed lib/antlr4/CommonTokenFactory.pyc
Binary file not shown.
Binary file removed lib/antlr4/CommonTokenStream.pyc
Binary file not shown.
Binary file removed lib/antlr4/FileStream.pyc
Binary file not shown.
Binary file removed lib/antlr4/InputStream.pyc
Binary file not shown.
Binary file removed lib/antlr4/IntervalSet.pyc
Binary file not shown.
Binary file removed lib/antlr4/LL1Analyzer.pyc
Binary file not shown.
Binary file removed lib/antlr4/Lexer.pyc
Binary file not shown.
Binary file removed lib/antlr4/ListTokenSource.pyc
Binary file not shown.
Binary file removed lib/antlr4/Parser.pyc
Binary file not shown.
Binary file removed lib/antlr4/ParserInterpreter.pyc
Binary file not shown.
Binary file removed lib/antlr4/ParserRuleContext.pyc
Binary file not shown.
Binary file removed lib/antlr4/PredictionContext.pyc
Binary file not shown.
Binary file removed lib/antlr4/Recognizer.pyc
Binary file not shown.
Binary file removed lib/antlr4/RuleContext.pyc
Binary file not shown.
Binary file removed lib/antlr4/StdinStream.pyc
Binary file not shown.
Binary file removed lib/antlr4/Token.pyc
Binary file not shown.
Binary file removed lib/antlr4/TokenStreamRewriter.pyc
Binary file not shown.
Binary file removed lib/antlr4/Utils.pyc
Binary file not shown.
Binary file removed lib/antlr4/__init__.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATN.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNConfig.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNConfigSet.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNDeserializationOptions.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNDeserializer.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNSimulator.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNState.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ATNType.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/LexerATNSimulator.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/LexerAction.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/LexerActionExecutor.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/ParserATNSimulator.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/PredictionMode.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/SemanticContext.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/Transition.pyc
Binary file not shown.
Binary file removed lib/antlr4/atn/__init__.pyc
Binary file not shown.
Binary file removed lib/antlr4/dfa/DFA.pyc
Binary file not shown.
Binary file removed lib/antlr4/dfa/DFASerializer.pyc
Binary file not shown.
Binary file removed lib/antlr4/dfa/DFAState.pyc
Binary file not shown.
Binary file removed lib/antlr4/dfa/__init__.pyc
Binary file not shown.
Binary file removed lib/antlr4/error/DiagnosticErrorListener.pyc
Binary file not shown.
Binary file removed lib/antlr4/error/ErrorListener.pyc
Binary file not shown.
Binary file removed lib/antlr4/error/ErrorStrategy.pyc
Binary file not shown.
Binary file removed lib/antlr4/error/Errors.pyc
Binary file not shown.
Binary file removed lib/antlr4/error/__init__.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/Chunk.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/ParseTreeMatch.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/ParseTreePattern.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/ParseTreePatternMatcher.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/RuleTagToken.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/TokenTagToken.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/Tree.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/Trees.pyc
Binary file not shown.
Binary file removed lib/antlr4/tree/__init__.pyc
Binary file not shown.
Binary file removed lib/antlr4/xpath/XPath.pyc
Binary file not shown.
Binary file removed lib/antlr4/xpath/__init__.pyc
Binary file not shown.
Binary file removed lib/certifi/__init__.pyc
Binary file not shown.
Binary file removed lib/certifi/__main__.pyc
Binary file not shown.
Binary file removed lib/certifi/core.pyc
Binary file not shown.
Binary file removed lib/chardet/__init__.pyc
Binary file not shown.
Binary file removed lib/chardet/big5freq.pyc
Binary file not shown.
Binary file removed lib/chardet/big5prober.pyc
Binary file not shown.
Binary file removed lib/chardet/chardistribution.pyc
Binary file not shown.
Binary file removed lib/chardet/charsetgroupprober.pyc
Binary file not shown.
Binary file removed lib/chardet/charsetprober.pyc
Binary file not shown.
Binary file removed lib/chardet/cli/__init__.pyc
Binary file not shown.
Binary file removed lib/chardet/cli/chardetect.pyc
Binary file not shown.
Binary file removed lib/chardet/codingstatemachine.pyc
Binary file not shown.
Binary file removed lib/chardet/compat.pyc
Binary file not shown.
Binary file removed lib/chardet/cp949prober.pyc
Binary file not shown.
Binary file removed lib/chardet/enums.pyc
Binary file not shown.
Binary file removed lib/chardet/escprober.pyc
Binary file not shown.
Binary file removed lib/chardet/escsm.pyc
Binary file not shown.
Binary file removed lib/chardet/eucjpprober.pyc
Binary file not shown.
Binary file removed lib/chardet/euckrfreq.pyc
Binary file not shown.
Binary file removed lib/chardet/euckrprober.pyc
Binary file not shown.
Binary file removed lib/chardet/euctwfreq.pyc
Binary file not shown.
Binary file removed lib/chardet/euctwprober.pyc
Binary file not shown.
Binary file removed lib/chardet/gb2312freq.pyc
Binary file not shown.
Binary file removed lib/chardet/gb2312prober.pyc
Binary file not shown.
Binary file removed lib/chardet/hebrewprober.pyc
Binary file not shown.
Binary file removed lib/chardet/jisfreq.pyc
Binary file not shown.
Binary file removed lib/chardet/jpcntx.pyc
Binary file not shown.
Binary file removed lib/chardet/langbulgarianmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langcyrillicmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langgreekmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langhebrewmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langhungarianmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langthaimodel.pyc
Binary file not shown.
Binary file removed lib/chardet/langturkishmodel.pyc
Binary file not shown.
Binary file removed lib/chardet/latin1prober.pyc
Binary file not shown.
Binary file removed lib/chardet/mbcharsetprober.pyc
Binary file not shown.
Binary file removed lib/chardet/mbcsgroupprober.pyc
Binary file not shown.
Binary file removed lib/chardet/mbcssm.pyc
Binary file not shown.
Binary file removed lib/chardet/sbcharsetprober.pyc
Binary file not shown.
Binary file removed lib/chardet/sbcsgroupprober.pyc
Binary file not shown.
Binary file removed lib/chardet/sjisprober.pyc
Binary file not shown.
Binary file removed lib/chardet/universaldetector.pyc
Binary file not shown.
Binary file removed lib/chardet/utf8prober.pyc
Binary file not shown.
Binary file removed lib/chardet/version.pyc
Binary file not shown.
Binary file removed lib/enum/__init__.pyc
Binary file not shown.
Binary file removed lib/idna/__init__.pyc
Binary file not shown.
Binary file removed lib/idna/codec.pyc
Binary file not shown.
Binary file removed lib/idna/compat.pyc
Binary file not shown.
Binary file removed lib/idna/core.pyc
Binary file not shown.
Binary file removed lib/idna/idnadata.pyc
Binary file not shown.
Binary file removed lib/idna/intranges.pyc
Binary file not shown.
Binary file removed lib/idna/package_data.pyc
Binary file not shown.
Binary file removed lib/idna/uts46data.pyc
Binary file not shown.
Binary file removed lib/requests/__init__.pyc
Binary file not shown.
Binary file removed lib/requests/__version__.pyc
Binary file not shown.
Binary file removed lib/requests/_internal_utils.pyc
Binary file not shown.
Binary file removed lib/requests/adapters.pyc
Binary file not shown.
Binary file removed lib/requests/api.pyc
Binary file not shown.
Binary file removed lib/requests/auth.pyc
Binary file not shown.
Binary file removed lib/requests/certs.pyc
Binary file not shown.
Binary file removed lib/requests/compat.pyc
Binary file not shown.
Binary file removed lib/requests/cookies.pyc
Binary file not shown.
Binary file removed lib/requests/exceptions.pyc
Binary file not shown.
Binary file removed lib/requests/help.pyc
Binary file not shown.
Binary file removed lib/requests/hooks.pyc
Binary file not shown.
Binary file removed lib/requests/models.pyc
Binary file not shown.
Binary file removed lib/requests/packages.pyc
Binary file not shown.
Binary file removed lib/requests/sessions.pyc
Binary file not shown.
Binary file removed lib/requests/status_codes.pyc
Binary file not shown.
Binary file removed lib/requests/structures.pyc
Binary file not shown.
Binary file removed lib/requests/utils.pyc
Binary file not shown.
Binary file removed lib/six.pyc
Binary file not shown.
Binary file removed lib/splunklib/__init__.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/__init__.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/__init__.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/binding.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/binding.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/client.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/client.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/data.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/data.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/results.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/results.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/six.cpython-36.pyc
Binary file not shown.
Binary file removed lib/splunklib/__pycache__/six.cpython-37.pyc
Binary file not shown.
Binary file removed lib/splunklib/binding.pyc
Binary file not shown.
Binary file removed lib/splunklib/client.pyc
Binary file not shown.
Binary file removed lib/splunklib/data.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/__init__.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/argument.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/event.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/event_writer.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/input_definition.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/scheme.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/script.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/utils.pyc
Binary file not shown.
Binary file removed lib/splunklib/modularinput/validation_definition.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file removed lib/splunklib/six.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/exceptions.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/grammars/STIXPatternLexer.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/grammars/STIXPatternListener.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/grammars/STIXPatternParser.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/grammars/STIXPatternVisitor.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/grammars/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/helpers.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/inspector.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/pattern.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/test_helpers.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v20/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v20/test_inspector.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v20/test_validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v21/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v21/test_inspector.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/test/v21/test_validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/grammars/STIXPatternLexer.pyc
Binary file not shown.
Binary file not shown.
Binary file removed lib/stix2patterns/v20/grammars/STIXPatternParser.pyc
Binary file not shown.
Binary file not shown.
Binary file removed lib/stix2patterns/v20/grammars/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/inspector.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/object_validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/pattern.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v20/validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/grammars/STIXPatternLexer.pyc
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file removed lib/stix2patterns/v21/grammars/__init__.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/inspector.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/object_validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/pattern.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/v21/validator.pyc
Binary file not shown.
Binary file removed lib/stix2patterns/validator.pyc
Binary file not shown.
Binary file removed lib/urllib3/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/_collections.pyc
Binary file not shown.
Binary file removed lib/urllib3/_version.pyc
Binary file not shown.
Binary file removed lib/urllib3/connection.pyc
Binary file not shown.
Binary file removed lib/urllib3/connectionpool.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/_appengine_environ.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/_securetransport/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/_securetransport/bindings.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/_securetransport/low_level.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/appengine.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/ntlmpool.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/pyopenssl.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/securetransport.pyc
Binary file not shown.
Binary file removed lib/urllib3/contrib/socks.pyc
Binary file not shown.
Binary file removed lib/urllib3/exceptions.pyc
Binary file not shown.
Binary file removed lib/urllib3/fields.pyc
Binary file not shown.
Binary file removed lib/urllib3/filepost.pyc
Binary file not shown.
Binary file removed lib/urllib3/packages/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/packages/backports/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/packages/backports/makefile.pyc
Binary file not shown.
Binary file removed lib/urllib3/packages/six.pyc
Binary file not shown.
Binary file removed lib/urllib3/packages/ssl_match_hostname/__init__.pyc
Binary file not shown.
Binary file not shown.
Binary file removed lib/urllib3/poolmanager.pyc
Binary file not shown.
Binary file removed lib/urllib3/request.pyc
Binary file not shown.
Binary file removed lib/urllib3/response.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/__init__.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/connection.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/queue.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/request.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/response.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/retry.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/ssl_.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/timeout.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/url.pyc
Binary file not shown.
Binary file removed lib/urllib3/util/wait.pyc
Binary file not shown.

0 comments on commit 7f43014

Please sign in to comment.