Update cardinality field in schema for threshold rules (elastic#1349)

* Make cardinality array in schema for threshold rules * update master, 7.12, 7.13, and 7.14 schemas with cardinality fix * fix 7.12 downgrade to handle cardinality as an array * Add two new rules to detect agent spoofing Co-authored-by: Ross Wolf <[email protected]>
shashank-elastic · Jul 21, 2021 · 163d9e3 · 163d9e3
1 parent 95e6458
commit 163d9e3
Show file tree

Hide file tree

Showing 13 changed files with 216 additions and 72 deletions.
diff --git a/detection_rules/rule.py b/detection_rules/rule.py
@@ -183,7 +183,7 @@ class BaseRuleData(MarshmallowDataclassMixin):
     def save_schema(cls):
         """Save the schema as a jsonschema."""
         fields: List[dataclasses.Field] = dataclasses.fields(cls)
-        type_field = next(field for field in fields if field.name == "type")
+        type_field = next(f for f in fields if f.name == "type")
         rule_type = typing.get_args(type_field.type)[0] if cls != BaseRuleData else "base"
         schema = cls.jsonschema()
         version_dir = SCHEMA_DIR / "master"
@@ -256,9 +256,9 @@ class ThresholdCardinality:
             field: str
             value: definitions.ThresholdValue
 
-        field: List[definitions.NonEmptyStr]
+        field: definitions.CardinalityFields
         value: definitions.ThresholdValue
-        cardinality: Optional[ThresholdCardinality]
+        cardinality: Optional[List[ThresholdCardinality]]
 
     type: Literal["threshold"]
     threshold: ThresholdMapping

diff --git a/detection_rules/rule_formatter.py b/detection_rules/rule_formatter.py
@@ -206,5 +206,5 @@ def _do_write(_data, _contents):
             _do_write(data, _contents)
 
     finally:
-        if needs_close:
+        if needs_close and hasattr(outfile, "close"):
             outfile.close()
diff --git a/detection_rules/schemas/__init__.py b/detection_rules/schemas/__init__.py
@@ -123,7 +123,7 @@ def downgrade_threshold_to_7_11(version: Version, api_contents: dict) -> dict:
         if len(threshold_field) > 1:
             raise ValueError('Cannot downgrade a threshold rule that has multiple threshold fields defined')
 
-        if threshold.get('cardinality', {}).get('field') or threshold.get('cardinality', {}).get('value'):
+        if threshold.get('cardinality'):
             raise ValueError('Cannot downgrade a threshold rule that has a defined cardinality')
 
         api_contents = api_contents.copy()

diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
@@ -5,7 +5,7 @@
 
 """Custom shared definitions for schemas."""
 
-from typing import Literal, Final
+from typing import List, Literal, Final
 
 from marshmallow import validate
 from marshmallow_dataclass import NewType
@@ -44,19 +44,21 @@
 }
 
 
+NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
+
 BranchVer = NewType('BranchVer', str, validate=validate.Regexp(BRANCH_PATTERN))
+CardinalityFields = NewType('CardinalityFields', List[NonEmptyStr], validate=validate.Length(min=0, max=3))
 CodeString = NewType("CodeString", str)
 ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
 Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
 FilterLanguages = Literal["kuery", "lucene"]
 Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN))
-PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
 Markdown = NewType("MarkdownField", CodeString)
 Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated']
 MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1))
-NonEmptyStr = NewType('NonEmptyStr', str, validate=validate.Length(min=1))
 Operator = Literal['equals']
 OSType = Literal['windows', 'linux', 'macos']
+PositiveInteger = NewType('PositiveInteger', int, validate=validate.Range(min=1))
 RiskScore = NewType("MaxSignals", int, validate=validate.Range(min=1, max=100))
 RuleType = Literal['query', 'saved_query', 'machine_learning', 'eql', 'threshold', 'threat_match']
 SemVer = NewType('SemVer', str, validate=validate.Regexp(VERSION_PATTERN))

diff --git a/etc/api_schemas/7.12/7.12.threshold.json b/etc/api_schemas/7.12/7.12.threshold.json
@@ -919,35 +919,46 @@
       "additionalProperties": false,
       "properties": {
         "cardinality": {
-          "additionalProperties": false,
-          "properties": {
-            "field": {
-              "type": "string"
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "field": {
+                "type": "string"
+              },
+              "value": {
+                "description": "ThresholdValue",
+                "format": "integer",
+                "minimum": 1,
+                "type": "number"
+              }
             },
-            "value": {
-              "minimum": 1,
-              "type": "integer"
-            }
+            "required": [
+              "field",
+              "value"
+            ],
+            "type": "object"
           },
-          "required": [
-            "field",
-            "value"
-          ],
-          "type": "object"
+          "type": "array"
         },
         "field": {
+          "description": "CardinalityFields",
           "items": {
-            "default": "",
+            "description": "NonEmptyStr",
+            "minLength": 1,
             "type": "string"
           },
+          "maxItems": 3,
           "type": "array"
         },
         "value": {
+          "description": "ThresholdValue",
+          "format": "integer",
           "minimum": 1,
-          "type": "integer"
+          "type": "number"
         }
       },
       "required": [
+        "field",
         "value"
       ],
       "type": "object"

diff --git a/etc/api_schemas/7.13/7.13.threshold.json b/etc/api_schemas/7.13/7.13.threshold.json
@@ -112,6 +112,9 @@
             "type": "string"
           },
           "operator": {
+            "enum": [
+              "equals"
+            ],
             "type": "string"
           },
           "value": {
@@ -151,6 +154,9 @@
             "type": "string"
           },
           "operator": {
+            "enum": [
+              "equals"
+            ],
             "type": "string"
           },
           "severity": {
@@ -178,6 +184,9 @@
         "additionalProperties": false,
         "properties": {
           "framework": {
+            "enum": [
+              "MITRE ATT&CK"
+            ],
             "type": "string"
           },
           "tactic": {
@@ -265,30 +274,35 @@
       "additionalProperties": false,
       "properties": {
         "cardinality": {
-          "additionalProperties": false,
-          "properties": {
-            "field": {
-              "type": "string"
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "field": {
+                "type": "string"
+              },
+              "value": {
+                "description": "ThresholdValue",
+                "format": "integer",
+                "minimum": 1,
+                "type": "number"
+              }
             },
-            "value": {
-              "description": "ThresholdValue",
-              "format": "integer",
-              "minimum": 1,
-              "type": "number"
-            }
+            "required": [
+              "field",
+              "value"
+            ],
+            "type": "object"
           },
-          "required": [
-            "field",
-            "value"
-          ],
-          "type": "object"
+          "type": "array"
         },
         "field": {
+          "description": "CardinalityFields",
           "items": {
             "description": "NonEmptyStr",
             "minLength": 1,
             "type": "string"
           },
+          "maxItems": 3,
           "type": "array"
         },
         "value": {
@@ -336,6 +350,9 @@
       "type": "string"
     },
     "type": {
+      "enum": [
+        "threshold"
+      ],
       "type": "string"
     }
   },

diff --git a/etc/api_schemas/7.14/7.14.threshold.json b/etc/api_schemas/7.14/7.14.threshold.json
@@ -274,30 +274,35 @@
       "additionalProperties": false,
       "properties": {
         "cardinality": {
-          "additionalProperties": false,
-          "properties": {
-            "field": {
-              "type": "string"
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "field": {
+                "type": "string"
+              },
+              "value": {
+                "description": "ThresholdValue",
+                "format": "integer",
+                "minimum": 1,
+                "type": "number"
+              }
             },
-            "value": {
-              "description": "ThresholdValue",
-              "format": "integer",
-              "minimum": 1,
-              "type": "number"
-            }
+            "required": [
+              "field",
+              "value"
+            ],
+            "type": "object"
           },
-          "required": [
-            "field",
-            "value"
-          ],
-          "type": "object"
+          "type": "array"
         },
         "field": {
+          "description": "CardinalityFields",
           "items": {
             "description": "NonEmptyStr",
             "minLength": 1,
             "type": "string"
           },
+          "maxItems": 3,
           "type": "array"
         },
         "value": {

diff --git a/etc/api_schemas/master/master.threshold.json b/etc/api_schemas/master/master.threshold.json
@@ -274,30 +274,35 @@
       "additionalProperties": false,
       "properties": {
         "cardinality": {
-          "additionalProperties": false,
-          "properties": {
-            "field": {
-              "type": "string"
+          "items": {
+            "additionalProperties": false,
+            "properties": {
+              "field": {
+                "type": "string"
+              },
+              "value": {
+                "description": "ThresholdValue",
+                "format": "integer",
+                "minimum": 1,
+                "type": "number"
+              }
             },
-            "value": {
-              "description": "ThresholdValue",
-              "format": "integer",
-              "minimum": 1,
-              "type": "number"
-            }
+            "required": [
+              "field",
+              "value"
+            ],
+            "type": "object"
           },
-          "required": [
-            "field",
-            "value"
-          ],
-          "type": "object"
+          "type": "array"
         },
         "field": {
+          "description": "CardinalityFields",
           "items": {
             "description": "NonEmptyStr",
             "minLength": 1,
             "type": "string"
           },
+          "maxItems": 3,
           "type": "array"
         },
         "value": {

diff --git a/etc/stack-schema-map.yaml b/etc/stack-schema-map.yaml
@@ -9,4 +9,4 @@
 
 "7.14.0":
   beats: "master"  # TODO: 7.14.x
-  ecs: "1.10.0"
+  ecs: "master"  # TODO: master came out after 7.13.0 release
diff --git a/requirements.txt b/requirements.txt
@@ -8,7 +8,7 @@ PyYAML~=5.3
 eql==0.9.9
 elasticsearch~=7.9
 XlsxWriter~=1.3.6
-marshmallow~=3.10.0
+marshmallow~=3.12.2
 marshmallow-dataclass[union]~=8.4
 
 # test deps

diff --git a/rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml b/rules/cross-platform/defense_evasion_agent_spoofing_mismatched_id.toml
@@ -0,0 +1,48 @@
+[metadata]
+creation_date = "2021/07/14"
+maturity = "production"
+updated_date = "2021/07/14"
+min_stack_version = "7.14.0"
+
+[rule]
+author = ["Elastic"]
+description = """Detects events which have a mismatch on the expected event agent ID. The status "agent_id_mismatch"
+occurs when the expected agent ID associated with the API key does not match the actual agent ID in an event. This could
+indicate attempts to spoof events in order to masquerade actual activity to evade detection.
+"""
+false_positives = [
+    """
+    This is meant to run only on datasources using agents v7.14+ since versions prior to that will be missing the
+    necessary field, resulting in false positives.
+    """,
+]
+from = "now-9m"
+index = ["logs-*", "metrics-*", "traces-*"]
+language = "kuery"
+license = "Elastic License v2"
+name = "Agent Spoofing - Mismatched Agent ID"
+risk_score = 73
+rule_id = "3115bd2c-0baa-4df0-80ea-45e474b5ef93"
+severity = "high"
+tags = ["Elastic", "Threat Detection", "Defense Evasion"]
+timestamp_override = "event.ingested"
+type = "query"
+
+query = '''
+event.agent_id_status:agent_id_mismatch
+'''
+
+
+[[rule.threat]]
+framework = "MITRE ATT&CK"
+[[rule.threat.technique]]
+id = "T1036"
+name = "Masquerading"
+reference = "https://attack.mitre.org/techniques/T1036/"
+
+
+[rule.threat.tactic]
+id = "TA0005"
+name = "Defense Evasion"
+reference = "https://attack.mitre.org/tactics/TA0005/"
+