From 2e75cf09e4a819e9a7ac947d68286ab00f88f86f Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 11 Sep 2023 16:14:29 -0700 Subject: [PATCH 1/6] Add new terms rule type to Detections API create/update rule docs --- .../api/rules/rules-api-create.asciidoc | 25 ++++++++++++++++--- .../api/rules/rules-api-update.asciidoc | 23 ++++++++++++++--- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index b2855928c6..6f09c919b0 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -100,6 +100,7 @@ A JSON object that defines the rule's values: * <> * <> * <> +* <> * <> * <> * <> @@ -145,11 +146,12 @@ occurred {es} index. * `threshold`: rule based on the number of times a `query` matches the specified field. +* `new_terms`: rule that alerts on values that have not been seen before |============================================== [[req-fields-query-threshold]] -===== Required field for query, indicator match and threshold rules +===== Required field for query, indicator match, threshold, and new terms rules [width="100%",options="header"] |============================================== @@ -250,6 +252,21 @@ uses both `and` and `or` logic. |============================================== +[[req-fields-new-terms]] +===== Required fields for new terms rules + +[width="100%",options="header"] +|============================================== +|Name |Type |Description + +|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1-3 field names. + +|history_window_start |String |Start date to use when checking if a term has been seen before. +Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term +is new. + +|============================================== + [[opt-fields-all]] ===== Optional fields for all rule types @@ -336,7 +353,7 @@ Required when `actions` are used to send notifications. |============================================== [[opt-fields-query-eql]] -===== Optional fields for query, indicator match, and event correlation rules +===== Optional fields for query, indicator match, event correlation, and new terms rules [width="100%",options="header"] |============================================== @@ -372,7 +389,7 @@ documents from the {es} index containing the threat values. |============================================== [[opt-fields-query-threshold]] -===== Optional fields for query, indicator match, and threshold rules +===== Optional fields for query, indicator match, threshold, and new terms rules [width="100%",options="header"] |============================================== @@ -383,7 +400,7 @@ documents from the {es} index containing the threat values. |============================================== [[opt-fields-eql-query-threshold]] -===== Optional fields for event correlation, query, and threshold rules +===== Optional fields for event correlation, query, threshold, and new terms rules [width="100%",options="header"] |============================================== diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index ba5ee39a4f..461be98c33 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -67,10 +67,11 @@ occurred {es} index. * `threshold`: rule based on the number of times a `query` matches the specified field. +* `new_terms`: rule that alerts on values that have not been seen before |============================================== -===== Field required for query, threat-match and threshold rules `PUT` calls +===== Field required for query, threat-match, threshold, and new terms rules `PUT` calls [width="100%",options="header"] |============================================== @@ -166,6 +167,20 @@ uses both `and` and `or` logic. |============================================== +===== Fields required for new terms rules `PUT` calls + +[width="100%",options="header"] +|============================================== +|Name |Type |Description + +|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1-3 field names. + +|history_window_start |String |Start date to use when checking if a term has been seen before. +Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term +is new. + +|============================================== + ===== Optional fields for all rule types [width="100%",options="header"] @@ -244,7 +259,7 @@ number. |============================================== -===== Optional fields for query, threat-match and EQL rules +===== Optional fields for query, threat-match, EQL, and new terms rules [width="100%",options="header"] |============================================== @@ -278,7 +293,7 @@ documents from the {es} index containing the threat values. |Much like an ingest processor, users can use this field to define where their threat indicator can be found on their indicator documents. Defaults to `threatintel.indicator`. |============================================== -===== Optional fields for query, threat-match and threshold rules +===== Optional fields for query, threat-match, threshold, and new terms rules [width="100%",options="header"] |============================================== @@ -288,7 +303,7 @@ documents from the {es} index containing the threat values. `kuery` or `lucene`. Defaults to `kuery`. |============================================== -===== Optional fields for EQL, query and threshold rules +===== Optional fields for EQL, query, threshold, and new terms rules [width="100%",options="header"] |============================================== From a01fda6d0edc46d5f8f483f04541390169f64ea9 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Wed, 13 Sep 2023 11:39:41 -0700 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: natasha-moore-elastic <137783811+natasha-moore-elastic@users.noreply.github.com> --- docs/detections/api/rules/rules-api-create.asciidoc | 2 +- docs/detections/api/rules/rules-api-update.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 6f09c919b0..4d46e14903 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -259,7 +259,7 @@ uses both `and` and `or` logic. |============================================== |Name |Type |Description -|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1-3 field names. +|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index 461be98c33..03a02c30c8 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -173,7 +173,7 @@ uses both `and` and `or` logic. |============================================== |Name |Type |Description -|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1-3 field names. +|new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term From 443db435fe54024caa670a5743cf53ea04e55564 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Thu, 14 Sep 2023 15:11:08 -0700 Subject: [PATCH 3/6] Update copy for history_window_start recommended format --- docs/detections/api/rules/rules-api-create.asciidoc | 5 +++-- docs/detections/api/rules/rules-api-update.asciidoc | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 4d46e14903..2e700328aa 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -262,8 +262,9 @@ uses both `and` and `or` logic. |new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. -Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term -is new. +Supports relative dates – for example, now-30d will search the last 30 days of data when checking if a term +is new. We do not recommend using absolute dates, which can cause issues with rule performance +due to querying increasing amounts of data over time. |============================================== diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index 03a02c30c8..c7565debe6 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -176,8 +176,9 @@ uses both `and` and `or` logic. |new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. -Supports relative dates, e.g. `now-30d` will search the last 30 days of data when checking if a term -is new. +Supports relative dates – for example, now-30d will search the last 30 days of data when checking if a term +is new. We do not recommend using absolute dates, which can cause issues with rule performance +due to querying increasing amounts of data over time. |============================================== From d169fa72a2828b0291db9306be0fe1baa3024703 Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Mon, 18 Sep 2023 13:08:49 -0700 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: natasha-moore-elastic <137783811+natasha-moore-elastic@users.noreply.github.com> --- docs/detections/api/rules/rules-api-create.asciidoc | 2 +- docs/detections/api/rules/rules-api-update.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 2e700328aa..d6f53d85e0 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -262,7 +262,7 @@ uses both `and` and `or` logic. |new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. -Supports relative dates – for example, now-30d will search the last 30 days of data when checking if a term +Supports relative dates – for example, `now-30d` will search the last 30 days of data when checking if a term is new. We do not recommend using absolute dates, which can cause issues with rule performance due to querying increasing amounts of data over time. diff --git a/docs/detections/api/rules/rules-api-update.asciidoc b/docs/detections/api/rules/rules-api-update.asciidoc index c7565debe6..8552edf606 100644 --- a/docs/detections/api/rules/rules-api-update.asciidoc +++ b/docs/detections/api/rules/rules-api-update.asciidoc @@ -176,7 +176,7 @@ uses both `and` and `or` logic. |new_terms_fields |String[] |Fields to monitor for new values. Must contain 1–3 field names. |history_window_start |String |Start date to use when checking if a term has been seen before. -Supports relative dates – for example, now-30d will search the last 30 days of data when checking if a term +Supports relative dates – for example, `now-30d` will search the last 30 days of data when checking if a term is new. We do not recommend using absolute dates, which can cause issues with rule performance due to querying increasing amounts of data over time. From c662b4015de79571a2415ca8b3636328ef8177a1 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Mon, 18 Sep 2023 14:46:55 -0700 Subject: [PATCH 5/6] Add example request and response for new terms rule --- .../api/rules/rules-api-create.asciidoc | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index d6f53d85e0..8fbd323d9f 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -33,6 +33,7 @@ an event's `destination.ip` equals a value in the index. The index's field mappings should be {ecs-ref}[ECS-compliant]. * *{ml-cap} rules*: Creates an alert when a {ml} job discovers an anomaly above the defined threshold (see <>). +* *New terms*: Generates an alert for each new term detected in source documents within a specified time range. IMPORTANT: To create {ml} rules, you must have the https://www.elastic.co/subscriptions[appropriate license] or use a @@ -941,6 +942,29 @@ logic. In this example, both the event's `destination.ip` and <4> Sibling `entries` are evaluated using `or` logic. An alert is generated when at least one `entries` object evaluates to `true`. +*Example 6* + +New terms rule that creates alerts when the Windows `rundll32.exe` process makes +unusual network connections: + +[source,console] +-------------------------------------------------- +POST api/detection_engine/rules +{ + "risk_score": 21, + "description": "Detects a user associated with a new IP address", + "name": "New User IP Detected", + "severity": "medium", + "type": "new_terms", + "language": "kuery", + "query": "*", + "new_terms_fields": ["user.id", "source.ip"], + "history_window_start": "now-30d", + "index": ["auditbeat*"] +} +-------------------------------------------------- +// KIBANA + ==== Response code `200`:: @@ -1259,3 +1283,48 @@ Example response for an indicator match rule: } -------------------------------------------------- <1> dev:[] These fields are under development and their usage may change: `related_integrations`, `required_fields`, and `setup`. + +Example response for a new terms rule: + +[source,json] +-------------------------------------------------- +{ + "author": [], + "created_at": "2020-10-06T07:07:58.227Z", + "updated_at": "2020-10-06T07:07:58.237Z", + "created_by": "elastic", + "description": "Detects a user associated with a new IP address", + "enabled": true, + "false_positives": [], + "from": "now-6m", + "id": "eb7225c0-566b-11ee-8b4f-bbf3afdeb9f4", + "immutable": false, + "interval": "5m", + "rule_id": "c6f5d0bc-7be9-47d4-b2f3-073d22641e30", + "max_signals": 100, + "risk_score": 50, + "risk_score_mapping": [], + "name": "New User IP Detected", + "references": [], + "severity": "medium", + "severity_mapping": [], + "updated_by": "elastic", + "tags": [], + "to": "now", + "type": "threat_match", + "threat": [], + "version": 1, + "exceptions_list": [], + "index": [ + "auditbeat*" + ], + "query": "*", + "language": "kuery", + "new_terms_fields": ["user.id", "source.ip"], + "history_window_start": "now-30d", + "related_integrations": [], <1> + "required_fields": [], <1> + "setup": "" <1> +} +-------------------------------------------------- +<1> dev:[] These fields are under development and their usage may change: `related_integrations`, `required_fields`, and `setup`. From e8ddd2013144542fa3bec33ffc40667f31bcacd2 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 19 Sep 2023 08:00:27 -0700 Subject: [PATCH 6/6] Fix copy paste errors --- docs/detections/api/rules/rules-api-create.asciidoc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 8fbd323d9f..e9ca5aaae4 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -944,8 +944,7 @@ at least one `entries` object evaluates to `true`. *Example 6* -New terms rule that creates alerts when the Windows `rundll32.exe` process makes -unusual network connections: +New terms rule that creates alerts a new IP address is detected for a user: [source,console] -------------------------------------------------- @@ -1302,7 +1301,7 @@ Example response for a new terms rule: "interval": "5m", "rule_id": "c6f5d0bc-7be9-47d4-b2f3-073d22641e30", "max_signals": 100, - "risk_score": 50, + "risk_score": 21, "risk_score_mapping": [], "name": "New User IP Detected", "references": [], @@ -1311,7 +1310,7 @@ Example response for a new terms rule: "updated_by": "elastic", "tags": [], "to": "now", - "type": "threat_match", + "type": "new_terms", "threat": [], "version": 1, "exceptions_list": [],