From 036ee54e14fc3664e8cec3824b40f1f29e680951 Mon Sep 17 00:00:00 2001 From: "nastasha.solomon" Date: Mon, 6 May 2024 17:14:47 -0400 Subject: [PATCH 1/3] First draft --- docs/detections/rules-ui-create.asciidoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/detections/rules-ui-create.asciidoc b/docs/detections/rules-ui-create.asciidoc index 16f2a712dc..54d92ce039 100644 --- a/docs/detections/rules-ui-create.asciidoc +++ b/docs/detections/rules-ui-create.asciidoc @@ -318,11 +318,11 @@ Non-aggregating queries doesn't use `STATS...BY` functions and doesn't aggregate Here is an example non-aggregating query: [source,esql] ----- -FROM logs-* [metadata _id, _index, _version] +FROM logs-* METADATA _id, _index, _version | WHERE event.category == "process" AND event.id == "8a4f500d" | LIMIT 10 ----- -- This query starts by querying logs from indices that match the pattern `logs-*`. The `[metadata _id, _index, _version]` operator allows <>. +- This query starts by querying logs from indices that match the pattern `logs-*`. The `METADATA _id, _index, _version` operator allows <>. - Next, the query filters events where the `event.category` is a process and the `event.id` is `8a4f500d`. - Then, it limits the output to the top 10 results. @@ -330,11 +330,11 @@ FROM logs-* [metadata _id, _index, _version] [[esql-non-agg-query-dedupe]] ===== Turn on alert deduplication for rules using non-aggregating queries -To deduplicate alerts, a query needs access to the `_id`, `_index`, and `_version` metadata fields of the queried source event documents. You can allow this by adding the `[metadata _id, _index, _version]` operator after the `FROM` source command, for example: +To deduplicate alerts, a query needs access to the `_id`, `_index`, and `_version` metadata fields of the queried source event documents. You can allow this by adding the `METADATA _id, _index, _version` operator after the `FROM` source command, for example: [source,esql] ----- -FROM logs-* [metadata _id, _index, _version] +FROM logs-* METADATA _id, _index, _version | WHERE event.category == "process" AND event.id == "8a4f500d" | LIMIT 10 ----- @@ -347,7 +347,7 @@ Here is an example of a query that fails to deduplicate alerts. It uses the `DRO [source,esql] ----- -FROM logs-* [metadata _id, _index, _version] +FROM logs-* METADATA _id, _index, _version | WHERE event.category == "process" AND event.id == "8a4f500d" | DROP _id | LIMIT 10 @@ -357,7 +357,7 @@ Here is another example of an invalid query that uses the `KEEP` command to only [source,esql] ----- -FROM logs-* [metadata _id, _index, _version] +FROM logs-* METADATA _id, _index, _version | WHERE event.category == "process" AND event.id == "8a4f500d" | KEEP event.* | LIMIT 10 From 9ca2d588813c79a0831e54e7c5ef02e26ce4905e Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Tue, 7 May 2024 09:19:46 +0100 Subject: [PATCH 2/3] update metadata for API create --- docs/detections/api/rules/rules-api-create.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/detections/api/rules/rules-api-create.asciidoc b/docs/detections/api/rules/rules-api-create.asciidoc index 66278ae3ce..e85c4b6848 100644 --- a/docs/detections/api/rules/rules-api-create.asciidoc +++ b/docs/detections/api/rules/rules-api-create.asciidoc @@ -1066,7 +1066,7 @@ POST api/detection_engine/rules { "type": "esql", "language": "esql", - "query": "from auditbeat-8.10.2 [metadata _id, _version, _index] | where process.parent.name == \"EXCEL.EXE\"", + "query": "from auditbeat-8.10.2 METADATA _id, _version, _index | where process.parent.name == \"EXCEL.EXE\"", "name": "Find Excel events", "description": "Find Excel events", "tags": [], @@ -1527,7 +1527,7 @@ Example response for an {esql} rule: "setup": "", "type": "esql", "language": "esql", - "query": "from auditbeat-8.10.2 [metadata _id] | where process.parent.name == \"EXCEL.EXE\"" + "query": "from auditbeat-8.10.2 METADATA _id | where process.parent.name == \"EXCEL.EXE\"" } -------------------------------------------------- <1> dev:[] These fields are under development and their usage may change: `related_integrations` and `required_fields`. From 0195f62c80c684a11c60151ee77565b9b5c56eed Mon Sep 17 00:00:00 2001 From: "nastasha.solomon" Date: Fri, 10 May 2024 09:44:17 -0400 Subject: [PATCH 3/3] Nat's edits --- docs/detections/rules-ui-create.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/detections/rules-ui-create.asciidoc b/docs/detections/rules-ui-create.asciidoc index c35fda61ca..46bb76974e 100644 --- a/docs/detections/rules-ui-create.asciidoc +++ b/docs/detections/rules-ui-create.asciidoc @@ -311,7 +311,7 @@ NOTE: Rules that use aggregating queries might create duplicate alerts. This can [float] [[esql-non-agg-query]] ===== Non-aggregating query -Non-aggregating queries doesn't use `STATS...BY` functions and doesn't aggregate source event data. Alerts generated by an {esql} rule with a non-aggregating query only contain the fields returned by the query. +Non-aggregating queries don't use `STATS...BY` functions and don't aggregate source event data. Alerts generated by an {esql} rule with a non-aggregating query only contain the fields returned by the query. Here is an example non-aggregating query: [source,esql] @@ -367,7 +367,7 @@ FROM logs-* METADATA _id, _index, _version When writing your query, consider the following: -- The {ref}/esql-commands.html#esql-limit[`LIMIT`] command specifies the number of rows an {esql} query returns and the number of alerts created per rule execution. Similarly, a detection rule's <> setting specifies the maximum number of alerts it can create every time it runs. +- The {ref}/esql-commands.html#esql-limit[`LIMIT`] command specifies the maximum number of rows an {esql} query returns and the maximum number of alerts created per rule execution. Similarly, a detection rule's <> setting specifies the maximum number of alerts it can create every time it runs. + If the `LIMIT` value is lower than the `max_signals` value, the rule uses the `LIMIT` value to determine the maximum number of alerts the rule generates. If the `LIMIT` value is higher than the `max_signals` value, the rule uses the `max_signals` value. + @@ -382,7 +382,7 @@ NOTE: The `max_signals` default value is 100. You can modify it using the <>. +- If your {esql} query creates new fields that aren’t part of the ECS schema, they won’t be mapped to the alerts index, and you can't search or filter for them from the Alerts table. As a workaround, create <>. - If your {esql} query creates new fields that aren’t in the query’s source index, they can’t be added to the rule’s <>. [float]