From d26a31a51ed316e50b1be8f97e84a152aaaec2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Guillermin?= Date: Wed, 7 Jul 2021 17:59:11 +0200 Subject: [PATCH 1/3] 1st version working with bad sigma rules --- timesketch/lib/analyzers/sigma_tagger.py | 10 ++++++---- timesketch/lib/datastores/elastic.py | 4 ++++ timesketch/lib/sigma_util.py | 16 ++++++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py index 8414ce495b..7c42ecb6c0 100644 --- a/timesketch/lib/analyzers/sigma_tagger.py +++ b/timesketch/lib/analyzers/sigma_tagger.py @@ -75,6 +75,8 @@ def run(self): for rule in sigma_rules: tags_applied[rule.get('file_name')] = 0 try: + if not rule.get('es_query'): + continue sigma_rule_counter += 1 tagged_events_counter = self.run_sigma_rule( rule.get('es_query'), rule.get('file_name'), @@ -84,14 +86,14 @@ def run(self): logger.debug('Rule {0:d}/{1:d}'.format( sigma_rule_counter, len(sigma_rules))) except elasticsearch.TransportError as e: - logger.error( - 'Timeout executing search for {0:s}: ' - '{1!s} waiting for 10 seconds'.format( - rule.get('file_name'), e), exc_info=True) # this is caused by too many ES queries in short time range # TODO: https://github.com/google/timesketch/issues/1782 sleep_time = current_app.config.get( 'SIGMA_TAG_DELAY', 15) + logger.error( + 'Timeout executing search for {0:s}: ' + '{1!s} waiting for {2} seconds'.format( + rule.get('file_name'), e, sleep_time), exc_info=True) time.sleep(sleep_time) tagged_events_counter = self.run_sigma_rule( rule.get('es_query'), rule.get('file_name'), diff --git a/timesketch/lib/datastores/elastic.py b/timesketch/lib/datastores/elastic.py index 4ac4b7842e..4d978572bf 100644 --- a/timesketch/lib/datastores/elastic.py +++ b/timesketch/lib/datastores/elastic.py @@ -125,6 +125,8 @@ def __init__(self, host='127.0.0.1', port=9200): if self.user and self.password: parameters['http_auth'] = (self.user, self.password) + parameters['timeout'] = 30 + self.client = Elasticsearch( [{'host': host, 'port': port}], **parameters) @@ -655,6 +657,8 @@ def search_stream(self, sketch_id=None, query_string=None, for event in result['hits']['hits']: yield event + self.client.clear_scroll(scroll_id=scroll_id) + def get_filter_labels(self, sketch_id, indices): """Aggregate labels for a sketch. diff --git a/timesketch/lib/sigma_util.py b/timesketch/lib/sigma_util.py index d1d56322ec..61e5ced68f 100644 --- a/timesketch/lib/sigma_util.py +++ b/timesketch/lib/sigma_util.py @@ -202,24 +202,36 @@ def get_sigma_rule(filepath, sigma_config=None): abs_path, 'r', encoding='utf-8', errors='replace') as file: try: rule_return = {} + parsed_sigma_rules = {} + logsource = dict() rule_yaml_data = yaml.safe_load_all(file.read()) for doc in rule_yaml_data: + if 'logsource' not in doc: + logsource = next(rule_yaml_data) + if 'detection' not in logsource: + logsource['detection'] = dict() + if 'detection' in doc: + for k, v in doc['detection'].items(): + logsource['detection'][k] = v + doc.update(logsource) rule_return.update(doc) parser = sigma_collection.SigmaCollectionParser( yaml.safe_dump(doc), sigma_conf_obj, None) parsed_sigma_rules = parser.generate(sigma_backend) + if logsource: + break except NotImplementedError as exception: logger.error( 'Error generating rule in file {0:s}: {1!s}' .format(abs_path, exception)) - raise + # raise except sigma_exceptions.SigmaParseError as exception: logger.error( 'Sigma parsing error generating rule in file {0:s}: {1!s}' .format(abs_path, exception)) - raise + # raise except yaml.parser.ParserError as exception: logger.error( From 608c1ced408937c37f9b2db9ddc1144e441c0fe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Guillermin?= Date: Thu, 19 Aug 2021 09:32:41 +0200 Subject: [PATCH 2/3] Back to master code for new features --- timesketch/lib/analyzers/sigma_tagger.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py index 7c42ecb6c0..6753e6cd49 100644 --- a/timesketch/lib/analyzers/sigma_tagger.py +++ b/timesketch/lib/analyzers/sigma_tagger.py @@ -88,11 +88,13 @@ def run(self): except elasticsearch.TransportError as e: # this is caused by too many ES queries in short time range # TODO: https://github.com/google/timesketch/issues/1782 - sleep_time = current_app.config.get( + sleep_time : int = current_app.config.get( 'SIGMA_TAG_DELAY', 15) logger.error( 'Timeout executing search for {0:s}: ' - '{1!s} waiting for {2} seconds'.format( + '{1!s} waiting for {2:d} seconds ' + '(https://github.com/google/timesketch/issues/1782)' + .format( rule.get('file_name'), e, sleep_time), exc_info=True) time.sleep(sleep_time) tagged_events_counter = self.run_sigma_rule( From bad29f706480cbab2e6caf1effc2745aa1221e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Guillermin?= Date: Thu, 19 Aug 2021 09:34:42 +0200 Subject: [PATCH 3/3] Back to master code for new features --- timesketch/lib/analyzers/sigma_tagger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timesketch/lib/analyzers/sigma_tagger.py b/timesketch/lib/analyzers/sigma_tagger.py index 6753e6cd49..fda2973b33 100644 --- a/timesketch/lib/analyzers/sigma_tagger.py +++ b/timesketch/lib/analyzers/sigma_tagger.py @@ -86,8 +86,6 @@ def run(self): logger.debug('Rule {0:d}/{1:d}'.format( sigma_rule_counter, len(sigma_rules))) except elasticsearch.TransportError as e: - # this is caused by too many ES queries in short time range - # TODO: https://github.com/google/timesketch/issues/1782 sleep_time : int = current_app.config.get( 'SIGMA_TAG_DELAY', 15) logger.error( @@ -96,6 +94,8 @@ def run(self): '(https://github.com/google/timesketch/issues/1782)' .format( rule.get('file_name'), e, sleep_time), exc_info=True) + # this is caused by too many ES queries in short time range + # TODO: https://github.com/google/timesketch/issues/1782 time.sleep(sleep_time) tagged_events_counter = self.run_sigma_rule( rule.get('es_query'), rule.get('file_name'),