diff --git a/README/inputs.conf.spec b/README/inputs.conf.spec index 273b849..cc6191c 100644 --- a/README/inputs.conf.spec +++ b/README/inputs.conf.spec @@ -1,5 +1,4 @@ [sekoia_indicators://] -interval = 1800 api_key = feed_id = diff --git a/bin/sekoia_indicators.py b/bin/sekoia_indicators.py index d11b399..8b626e1 100644 --- a/bin/sekoia_indicators.py +++ b/bin/sekoia_indicators.py @@ -29,6 +29,13 @@ } +def from_rfc3339(date_string): + try: + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%SZ") + + class SEKOIAIndicators(Script): def __init__(self): self._splunk = None @@ -169,22 +176,26 @@ def revoke_indicator(self, kv_objects): for obj in objects: try: self.get_kvstore(ioc_type).delete_by_id(obj["_key"]) - # TODO: improve error handling here except Exception: pass # Store indicators in Splunk KV-Stores def store_indicators(self, indicators): objects = defaultdict(list) + now = datetime.utcnow() for indicator in indicators: kv_objects = self.indicator_to_kv(indicator) if indicator.get("revoked", False): self.revoke_indicator(kv_objects) - else: - for ioc_type, dicts in six.iteritems(kv_objects): - objects[ioc_type] += dicts + # Only import IOCs with a Valid Until date set + elif indicator.get("valid_until"): + # Ignore expired indicators + valid_until = from_rfc3339(indicator["valid_until"]) + if valid_until > now: + for ioc_type, dicts in six.iteritems(kv_objects): + objects[ioc_type] += dicts for ioc_type, batch in six.iteritems(objects): self.get_kvstore(ioc_type).batch_save(*batch) @@ -225,20 +236,22 @@ def validate_input(self, validation_definition): # Method called by Splunk to get new events def stream_events(self, inputs, ew): - try: - self._splunk = client.connect( - token=self._input_definition.metadata["session_key"], owner="nobody" - ) - - for indicators in self.get_indicators(inputs): - self.store_indicators(indicators) - except Exception: - exception = traceback.format_exc() + while True: + try: + self._splunk = client.connect( + token=self._input_definition.metadata["session_key"], owner="nobody" + ) - for line in exception.splitlines(): - print("ERROR {}".format(line), file=sys.stderr) + for indicators in self.get_indicators(inputs): + self.store_indicators(indicators) + except Exception: + exception = traceback.format_exc() - raise + for line in exception.splitlines(): + print("ERROR {}".format(line), file=sys.stderr) + finally: + print("INFO Done fetching indicators, sleeping for 10 minutes.") + time.sleep(600) if __name__ == "__main__":