[Security Content] Introduce Investigate Plugin in Investigation Guid…

…es (#3080) * [Security Content] Introduce Investigate Plugin in Investigation Guides * Add compatibility note * Update Transform format * update transform unit tests for investigate * updated docs with transform --------- Co-authored-by: brokensound77 <[email protected]> Co-authored-by: Justin Ibarra <[email protected]> (cherry picked from commit aeb1f91)
elastic · Dec 8, 2023 · 7529897 · 7529897
1 parent bb36349
commit 7529897
Show file tree

Hide file tree

Showing 9 changed files with 247 additions and 26 deletions.
diff --git a/detection_rules/rule.py b/detection_rules/rule.py
@@ -91,34 +91,30 @@ class OsQuery:
         ecs_mapping: Optional[Dict[str, Dict[Literal['field', 'value'], str]]]
 
     @dataclass(frozen=True)
-    class Insight:
+    class Investigate:
         @dataclass(frozen=True)
         class Provider:
+            excluded: bool
             field: str
+            queryType: definitions.InvestigateProviderQueryType
             value: str
-            type: str
+            valueType: definitions.InvestigateProviderValueType
 
         label: str
+        description: Optional[str]
         providers: List[List[Provider]]
+        relativeFrom: Optional[str]
+        relativeTo: Optional[str]
 
     # these must be lists in order to have more than one. Their index in the list is how they will be referenced in the
     # note string templates
     osquery: Optional[List[OsQuery]]
-    insight: Optional[List[Insight]]
+    investigate: Optional[List[Investigate]]
 
-    @validates_schema
-    def validate_transforms(self, value: dict, **kwargs):
-        """Validate transform fields."""
-        # temporarily invalidate insights until schema stabilizes
-        insight = value.get('insight')
-        if insight is not None:
-            raise NotImplementedError('Insights are not stable yet.')
-        return
-
-    def render_insight_osquery_to_string(self) -> Dict[Literal['osquery', 'insight'], List[str]]:
+    def render_investigate_osquery_to_string(self) -> Dict[definitions.TransformTypes, List[str]]:
         obj = self.to_dict()
 
-        rendered: Dict[Literal['osquery', 'insight'], List[str]] = {'osquery': [], 'insight': []}
+        rendered: Dict[definitions.TransformTypes, List[str]] = {'osquery': [], 'investigate': []}
         for plugin, entries in obj.items():
             for entry in entries:
                 rendered[plugin].append(f'!{{{plugin}{json.dumps(entry, sort_keys=True, separators=(",", ":"))}}}')
@@ -343,12 +339,12 @@ def process_transforms(cls, transform: RuleTransform, obj: dict) -> dict:
         # only create functions that CAREFULLY mutate the obj dict
 
         def process_note_plugins():
-            """Format the note field with osquery and insight plugin strings."""
+            """Format the note field with osquery and investigate plugin strings."""
             note = obj.get('note')
             if not note:
                 return
 
-            rendered = transform.render_insight_osquery_to_string()
+            rendered = transform.render_investigate_osquery_to_string()
             rendered_patterns = {}
             for plugin, entries in rendered.items():
                 rendered_patterns.update(**{f'{plugin}_{i}': e for i, e in enumerate(entries)})

diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py
@@ -140,6 +140,8 @@
 Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
 FilterLanguages = Literal["kuery", "lucene", "eql", "esql"]
 Interval = NewType('Interval', str, validate=validate.Regexp(INTERVAL_PATTERN))
+InvestigateProviderQueryType = Literal["phrase", "range"]
+InvestigateProviderValueType = Literal["string", "boolean"]
 Markdown = NewType("MarkdownField", CodeString)
 Maturity = Literal['development', 'experimental', 'beta', 'production', 'deprecated']
 MaxSignals = NewType("MaxSignals", int, validate=validate.Range(min=1))
@@ -160,6 +162,7 @@
 ThresholdValue = NewType("ThresholdValue", int, validate=validate.Range(min=1))
 TimelineTemplateId = NewType('TimelineTemplateId', str, validate=validate.OneOf(list(TIMELINE_TEMPLATES)))
 TimelineTemplateTitle = NewType('TimelineTemplateTitle', str, validate=validate.OneOf(TIMELINE_TEMPLATES.values()))
+TransformTypes = Literal["osquery", "investigate"]
 UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))
 BuildingBlockType = Literal['default']
 

diff --git a/docs/developing.md b/docs/developing.md
@@ -0,0 +1,38 @@
+# Developing
+
+Notes for managing and internal development
+
+## Transforms
+
+Transforms are data structures within rules which will be integrated into other fields at build
+time for rules, meaning they are not directly converted.
+
+### CLI
+
+There are some helper commands to assist with converting transforms into the excpected rule TOML format
+
+- create transform in Kibana
+- export it (or copy it)
+- run the following commmand and paste them (multiple)
+- copy and paste into rule, with minor format changes if needed
+
+```console
+(detection_dev) ➜  detection-rules git:(initial_inv_queries) python -m detection_rules dev transforms guide-plugin-convert
+
+█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄   ▄      █▀▀▄ ▄  ▄ ▄   ▄▄▄ ▄▄▄
+█  █ █▄▄  █  █▄▄ █    █   █  █ █ █▀▄ █      █▄▄▀ █  █ █   █▄▄ █▄▄
+█▄▄▀ █▄▄  █  █▄▄ █▄▄  █  ▄█▄ █▄█ █ ▀▄█      █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█
+
+Enter plugin contents []: !{investigate{"label":"Alerts associated with the host in the last 48h","providers":[[{"field":"event.kind","excluded":false,"queryType":"phrase","value":"signal","valueType":"string"},{"field":"host.name","excluded":false,"queryType":"phrase","value":"{{host.name}}","valueType":"string"}]],"relativeFrom":"now-48h/h","relativeTo":"now"}}
+[transform]
+
+[[transform.investigate]]
+label = "Alerts associated with the host in the last 48h"
+providers = [[{field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"}, {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}]]
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+```
+
+Other transform suppoprt can be found under
+
+`python -m detection-rules dev transforms -h`
diff --git a/rules/windows/command_and_control_common_webservices.toml b/rules/windows/command_and_control_common_webservices.toml
@@ -4,7 +4,7 @@ integration = ["endpoint"]
 maturity = "production"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
-updated_date = "2023/10/16"
+updated_date = "2023/12/07"
 
 [transform]
 [[transform.osquery]]
@@ -31,6 +31,38 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
 authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
 """
 
+[[transform.investigate]]
+label = "Alerts associated with the user in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+  [
+    {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+    {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
+  ]
+]
+
+
+[[transform.investigate]]
+label = "Alerts associated with the host in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+  [
+    {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+    {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}
+  ]
+]
+
+[[transform.investigate]]
+label = "Investigate the Subject Process Network Events"
+providers = [
+  [
+    {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
+    {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
+  ]
+]
+
 
 [rule]
 author = ["Elastic"]
@@ -55,18 +87,22 @@ This rule looks for processes outside known legitimate program locations communi
 
 > **Note**:
 > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
+> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
 
 #### Possible investigation steps
 
-- Investigate the script execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
+- Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
 - Investigate other alerts associated with the user/host during the past 48 hours.
+  - $investigate_0
+  - $investigate_1
 - Verify whether the digital signature exists in the executable.
 - Identify the operation type (upload, download, tunneling, etc.).
 - Examine the host for derived artifacts that indicate suspicious activities:
   - Analyze the process executable using a private sandboxed analysis system.
   - Observe and collect information about the following activities in both the sandbox and the alert subject host:
     - Attempts to contact external domains and addresses.
       - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
+        - $investigate_2
       - Examine the DNS cache for suspicious or anomalous entries.
         - $osquery_0
     - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.
@@ -75,7 +111,6 @@ This rule looks for processes outside known legitimate program locations communi
       - $osquery_2
       - $osquery_3
   - Retrieve the files' SHA-256 hash values using the PowerShell `Get-FileHash` cmdlet and search for the existence and reputation of the hashes in resources like VirusTotal, Hybrid-Analysis, CISCO Talos, Any.run, etc.
-- Investigate potentially compromised accounts. Analysts can do this by searching for login events (for example, 4624) to the target host after the registry modification.
 
 ### False positive analysis
 

diff --git a/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml b/rules/windows/command_and_control_remote_file_copy_desktopimgdownldr.toml
@@ -2,7 +2,7 @@
 creation_date = "2020/09/03"
 integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2023/10/23"
+updated_date = "2023/12/07"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
 
@@ -31,6 +31,37 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
 authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
 """
 
+[[transform.investigate]]
+label = "Alerts associated with the user in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+    [
+        {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+        {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
+    ]
+]
+
+[[transform.investigate]]
+label = "Alerts associated with the host in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+    [
+        {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+        {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"},
+    ]
+]
+
+[[transform.investigate]]
+label = "Investigate the Subject Process Network Events"
+providers = [
+    [
+        {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
+        {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
+    ]
+]
+
 
 [rule]
 author = ["Elastic"]
@@ -53,20 +84,24 @@ The `Desktopimgdownldr.exe` utility is used to to configure lockscreen/desktop i
 
 > **Note**:
 > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
+> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
 
 #### Possible investigation steps
 
 - Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
 - Identify the user account that performed the action and whether it should perform this kind of action.
 - Contact the account owner and confirm whether they are aware of this activity.
 - Investigate other alerts associated with the user/host during the past 48 hours.
+  - $investigate_0
+  - $investigate_1
 - Assess whether this behavior is prevalent in the environment by looking for similar occurrences across hosts.
 - Check the reputation of the domain or IP address used to host the downloaded file or if the user downloaded the file from an internal system.
 - Examine the host for derived artifacts that indicate suspicious activities:
   - Analyze the file using a private sandboxed analysis system.
   - Observe and collect information about the following activities in both the sandbox and the alert subject host:
     - Attempts to contact external domains and addresses.
       - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
+        - $investigate_2
       - Examine the DNS cache for suspicious or anomalous entries.
         - $osquery_0
     - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.

diff --git a/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml b/rules/windows/command_and_control_remote_file_copy_mpcmdrun.toml
@@ -2,7 +2,7 @@
 creation_date = "2020/09/03"
 integration = ["endpoint", "windows"]
 maturity = "production"
-updated_date = "2023/10/23"
+updated_date = "2023/12/07"
 min_stack_comments = "New fields added: required_fields, related_integrations, setup"
 min_stack_version = "8.3.0"
 
@@ -31,6 +31,37 @@ services.path FROM services JOIN authenticode ON services.path = authenticode.pa
 authenticode.path JOIN hash ON services.path = hash.path WHERE authenticode.result != 'trusted'
 """
 
+[[transform.investigate]]
+label = "Alerts associated with the user in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+    [
+        {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+        {field = "user.id", excluded = false, queryType = "phrase", value = "{{user.id}}", valueType = "string"}
+    ]
+]
+
+[[transform.investigate]]
+label = "Alerts associated with the host in the last 48h"
+relativeFrom = "now-48h/h"
+relativeTo = "now"
+providers = [
+    [
+        {field = "event.kind", excluded = false, queryType = "phrase", value = "signal", valueType = "string"},
+        {field = "host.name", excluded = false, queryType = "phrase", value = "{{host.name}}", valueType = "string"}
+    ]
+]
+
+[[transform.investigate]]
+label = "Investigate the Subject Process Network Events"
+providers = [
+    [
+        {field = "process.entity_id", excluded = false, queryType = "phrase", value = "{{process.entity_id}}", valueType = "string"},
+        {field = "event.category", excluded = false, queryType = "phrase", value = "network", valueType = "string"}
+    ]
+]
+
 
 [rule]
 author = ["Elastic"]
@@ -50,19 +81,23 @@ The `MpCmdRun.exe` is a command-line tool part of Windows Defender and is used t
 
 > **Note**:
 > This investigation guide uses the [Osquery Markdown Plugin](https://www.elastic.co/guide/en/security/master/invest-guide-run-osquery.html) introduced in Elastic Stack version 8.5.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
+> This investigation guide uses the [Investigate Markdown Plugin](https://www.elastic.co/guide/en/security/master/interactive-investigation-guides.html) introduced in Elastic Stack version 8.8.0. Older Elastic Stack versions will display unrendered Markdown in this guide.
 
 #### Possible investigation steps
 
 - Investigate the process execution chain (parent process tree) for unknown processes. Examine their executable files for prevalence, whether they are located in expected locations, and if they are signed with valid digital signatures.
 - Identify the user account that performed the action and whether it should perform this kind of action.
 - Contact the account owner and confirm whether they are aware of this activity.
 - Investigate other alerts associated with the user/host during the past 48 hours.
+  - $investigate_0
+  - $investigate_1
 - Check the reputation of the domain or IP address used to host the downloaded file.
 - Examine the host for derived artifacts that indicate suspicious activities:
   - Analyze the file using a private sandboxed analysis system.
   - Observe and collect information about the following activities in both the sandbox and the alert subject host:
     - Attempts to contact external domains and addresses.
       - Use the Elastic Defend network events to determine domains and addresses contacted by the subject process by filtering by the process' `process.entity_id`.
+        - $investigate_2
       - Examine the DNS cache for suspicious or anomalous entries.
         - $osquery_0
     - Use the Elastic Defend registry events to examine registry keys accessed, modified, or created by the related processes in the process tree.