Improve and patch sigma rules analyzer

timesketch/lib/analyzers/sigma_tagger.py

@@ -75,6 +75,8 @@ def run(self):
         for rule in sigma_rules:
             tags_applied[rule.get('file_name')] = 0
             try:
+                if not rule.get('es_query'):
+                    continue
                 sigma_rule_counter += 1
                 tagged_events_counter = self.run_sigma_rule(
                     rule.get('es_query'), rule.get('file_name'),

timesketch/lib/datastores/elastic.py

@@ -125,6 +125,8 @@ def __init__(self, host='127.0.0.1', port=9200):
         if self.user and self.password:
             parameters['http_auth'] = (self.user, self.password)
 
+        parameters['timeout'] = 30
+
         self.client = Elasticsearch(
             [{'host': host, 'port': port}], **parameters)
 
@@ -655,6 +657,8 @@ def search_stream(self, sketch_id=None, query_string=None,
             for event in result['hits']['hits']:
                 yield event
 
+        self.client.clear_scroll(scroll_id=scroll_id)
+
     def get_filter_labels(self, sketch_id, indices):
         """Aggregate labels for a sketch.
 

timesketch/lib/sigma_util.py

@@ -202,24 +202,36 @@ def get_sigma_rule(filepath, sigma_config=None):
             abs_path, 'r', encoding='utf-8', errors='replace') as file:
         try:
             rule_return = {}
+            parsed_sigma_rules = {}
+            logsource = dict()
             rule_yaml_data = yaml.safe_load_all(file.read())
             for doc in rule_yaml_data:
+                if 'logsource' not in doc:
+                    logsource = next(rule_yaml_data)
+                    if 'detection' not in logsource:
+                        logsource['detection'] = dict()
+                    if 'detection' in doc:
+                        for k, v in doc['detection'].items():
+                            logsource['detection'][k] = v
+                    doc.update(logsource)
                 rule_return.update(doc)
                 parser = sigma_collection.SigmaCollectionParser(
                     yaml.safe_dump(doc), sigma_conf_obj, None)
                 parsed_sigma_rules = parser.generate(sigma_backend)
+                if logsource:
+                    break
 
         except NotImplementedError as exception:
             logger.error(
                 'Error generating rule in file {0:s}: {1!s}'
                 .format(abs_path, exception))
-            raise
+            # raise
 
         except sigma_exceptions.SigmaParseError as exception:
             logger.error(
                 'Sigma parsing error generating rule in file {0:s}: {1!s}'
                 .format(abs_path, exception))
-            raise
+            # raise
 
         except yaml.parser.ParserError as exception:
             logger.error(