From 0840e2db9219927e0c711f8e80a65d4e7f44a552 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Oct 2024 11:23:37 -0400 Subject: [PATCH 1/5] Add README and helper util functions for moving data to frozen quickly --- .../quickstart/modules/frozen/README.md | 119 ++++++++++++++++++ .../quickstart/modules/frozen/index.ts | 77 ++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md create mode 100644 x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/index.ts diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md new file mode 100644 index 0000000000000..71925ca33e763 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md @@ -0,0 +1,119 @@ +## Creating Frozen Data + +This module provides utilities to quickly move data to the frozen tier for testing. The main API call that accomplishes this is `ilm.moveToStep`, however, there's setup that must be done first to make the index ready. The basic process to go from nothing to having data in frozen is: 0. Set `{"persistent": {"indices.lifecycle.poll_interval": "10s"}}` in the cluster settings. This setting is 10m by default, which makes it take a long time for indices to fully move to frozen. + +1. Create an ILM policy with at least `hot` and `frozen` phases +2. Create an index template that uses the ILM policy from step 1 (ideally make it a data stream template) +3. Index documents into the index/datastream controlled by the index template from step 2 +4. Rollover the index/datastream +5. Move the ...-000001 index that contains the documents we indexed to the next ILM step +6. Wait until the response from `GET /_ilm/explain` shows `phase: frozen`, `step: complete`, and `action: complete` - there are ~5 steps/actions it will go through automatically while you wait. If you didn't set the poll_interval in step 0, each of these steps/actions will take at least 10 minutes. For small data sets, the step/action should change every 10 seconds. + +As an example, dev tools commands below execute the steps above in order. + +``` +PUT /_cluster/settings +{ + "persistent": { + "indices.lifecycle.poll_interval": "10s" + } +} + +GET /_cluster/settings + +PUT _ilm/policy/my-lifecycle-policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "50gb" + } + } + }, + "frozen": { + "min_age": "0d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + } + } + } +} + +PUT _component_template/my-mappings +{ + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "message": { + "type": "wildcard" + } + } + } + }, + "_meta": { + "description": "Mappings for @timestamp and message fields", + "my-custom-meta-field": "More arbitrary metadata" + } +} + +# Creates a component template for index settings +PUT _component_template/my-settings +{ + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + } + }, + "_meta": { + "description": "Settings for ILM", + "my-custom-meta-field": "More arbitrary metadata" + } +} + +PUT _index_template/my-index-template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { }, + "composed_of": [ "my-mappings", "my-settings" ], + "priority": 500, + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + } +} + +PUT my-data-stream/_bulk +{ "create":{ } } +{ "@timestamp": "2099-05-06T16:21:15.000Z", "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } +{ "create":{ } } +{ "@timestamp": "2099-05-06T16:25:42.000Z", "message": "192.0.2.255 - - [06/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" } + +POST my-data-stream/_rollover + +// The `action` and `name` in the _ilm/move request below must match the `action` and `step` values from _ilm/explain - they'll probably be `rollover` and `check-rollover-ready` after rolling the index over, but if `_ilm/move` fails check and make sure the values match +GET my-data-stream/_ilm/explain + +POST _ilm/move/.ds-my-data-stream-2024.10.22-000001 +{ + "current_step": { + "phase": "hot", + "action": "rollover", + "name": "check-rollover-ready" + }, + "next_step": { + "phase": "frozen" + } +} + +// Wait for `phase`, `action`, `step`, to be `frozen`, `complete`, `complete` respectively +GET my-data-stream/_ilm/explain +``` diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/index.ts b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/index.ts new file mode 100644 index 0000000000000..b7e9537b0a2a3 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/index.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { + IlmExplainLifecycleLifecycleExplain, + IlmExplainLifecycleLifecycleExplainManaged, + IlmMoveToStepStepKey, +} from '@elastic/elasticsearch/lib/api/types'; +import type { ToolingLog } from '@kbn/tooling-log'; + +export const setIlmPollInterval = async ({ + client, + interval = '10s', +}: { + client: Client; + interval: string; +}) => { + client.cluster.putSettings({ + persistent: { 'indices.lifecycle.poll_interval': interval }, + }); +}; + +const getActionAndStep = ({ + ilmExplain, +}: { + ilmExplain: IlmExplainLifecycleLifecycleExplain; +}): { action: string | undefined; step: string | undefined } => { + return { + action: (ilmExplain as IlmExplainLifecycleLifecycleExplainManaged).action, + step: (ilmExplain as IlmExplainLifecycleLifecycleExplainManaged).step, + }; +}; + +/** + * Immediately move the write index for an index alias to frozen. This works, but in testing weird behaviors happen sometimes like rollover skipping + * over numbers for backing indices. Use with caution. + */ +export const moveWriteIndexToFrozen = async ({ + alias, + client, + log, +}: { + alias: string; + client: Client; + log: ToolingLog; +}) => { + const rolloverResp = await client.indices.rollover({ alias }); + log.info( + `Rolled over index, old index: ${rolloverResp.old_index}, new index: ${rolloverResp.new_index}` + ); + const oldIndex = rolloverResp.old_index; + + const ilmExplainResp = await client.ilm.explainLifecycle({ index: oldIndex }); + const expectedAction = 'rollover'; + const expectedStep = 'check-rollover-ready'; + let actionAndStep = getActionAndStep({ ilmExplain: ilmExplainResp.indices[oldIndex] }); + + while (actionAndStep.action !== expectedAction || actionAndStep.step !== expectedStep) { + log.info( + `Index not ready after rollover: index: ${oldIndex}, action: ${actionAndStep.action}, step: ${actionAndStep.step}` + ); + await new Promise((r) => setTimeout(r, 1000)); + const nextIlmExplainResp = await client.ilm.explainLifecycle({ index: oldIndex }); + actionAndStep = getActionAndStep({ ilmExplain: nextIlmExplainResp.indices[oldIndex] }); + } + + await client.ilm.moveToStep({ + index: rolloverResp.old_index, + current_step: { phase: 'hot', action: expectedAction, name: expectedStep }, + next_step: { phase: 'frozen' } as IlmMoveToStepStepKey, // Client types require action and name for next_step but ES doesn't actually need them. + }); +}; From 9238cff8af3a358ddcfc14f4bca4ae1296c2985d Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Tue, 22 Oct 2024 11:24:59 -0400 Subject: [PATCH 2/5] Update readme formatting --- .../scripts/quickstart/modules/frozen/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md index 71925ca33e763..04e8c04bb588f 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md @@ -1,7 +1,8 @@ ## Creating Frozen Data -This module provides utilities to quickly move data to the frozen tier for testing. The main API call that accomplishes this is `ilm.moveToStep`, however, there's setup that must be done first to make the index ready. The basic process to go from nothing to having data in frozen is: 0. Set `{"persistent": {"indices.lifecycle.poll_interval": "10s"}}` in the cluster settings. This setting is 10m by default, which makes it take a long time for indices to fully move to frozen. +This module provides utilities to quickly move data to the frozen tier for testing. The main API call that accomplishes this is `ilm.moveToStep`, however, there's setup that must be done first to make the index ready. The basic process to go from nothing to having data in frozen is: +0. Set `{"persistent": {"indices.lifecycle.poll_interval": "10s"}}` in the cluster settings. This setting is 10m by default, which makes it take a long time for indices to fully move to frozen. 1. Create an ILM policy with at least `hot` and `frozen` phases 2. Create an index template that uses the ILM policy from step 1 (ideally make it a data stream template) 3. Index documents into the index/datastream controlled by the index template from step 2 From d2104275f062be9c9db6f3b0b1b1b52e5330aa75 Mon Sep 17 00:00:00 2001 From: Marshall Main Date: Wed, 13 Nov 2024 15:17:51 -0500 Subject: [PATCH 3/5] Update README clarifying environment setup --- .../scripts/quickstart/modules/frozen/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md index 04e8c04bb588f..af3e237f1860f 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/frozen/README.md @@ -1,4 +1,6 @@ -## Creating Frozen Data +## Creating Frozen Data On Cloud + +The following instructions assume you have a cluster set up on ECH with at least one frozen tier node. To test frozen locally, follow instructions in https://docs.elastic.dev/security-soution/analyst-experience-team/eng-prod/how-to/configure-local-frozen-tier instead. Note that results may differ locally since ECH puts frozen shards on logically separate nodes whereas locally everything is running on your machine. This module provides utilities to quickly move data to the frozen tier for testing. The main API call that accomplishes this is `ilm.moveToStep`, however, there's setup that must be done first to make the index ready. The basic process to go from nothing to having data in frozen is: From 44b1658a67e0d660d4a4b43b822f0650bbfd456c Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:58:40 +0000 Subject: [PATCH 4/5] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.serverless.yaml | 78 ++++++-------------------- oas_docs/output/kibana.yaml | 78 ++++++-------------------- 2 files changed, 36 insertions(+), 120 deletions(-) diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 117e52586c5ad..4f54e401b14c2 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -1018,24 +1018,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1610,24 +1603,17 @@ paths: type: boolean flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1945,24 +1931,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2540,24 +2519,17 @@ paths: - active flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2847,24 +2819,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -3902,24 +3867,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 82acbdb311f2f..0e2811d1a4647 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -1367,24 +1367,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -1958,24 +1951,17 @@ paths: type: boolean flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2293,24 +2279,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -2887,24 +2866,17 @@ paths: - active flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch states - in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -3194,24 +3166,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number @@ -4241,24 +4206,17 @@ paths: - last_execution_date flapping: additionalProperties: false - description: >- - When flapping detection is turned on, alerts that switch - quickly between active and recovered states are identified - as “flapping” and notifications are reduced. + description: When flapping detection is turned on, alerts that switch quickly between active and recovered states are identified as “flapping” and notifications are reduced. nullable: true type: object properties: look_back_window: - description: >- - The minimum number of runs in which the threshold must - be met. + description: The minimum number of runs in which the threshold must be met. maximum: 20 minimum: 2 type: number status_change_threshold: - description: >- - The minimum number of times an alert must switch - states in the look back window. + description: The minimum number of times an alert must switch states in the look back window. maximum: 20 minimum: 2 type: number From bc04404a93dd7a349c8068b1efb4fe4a88a89bd1 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:45:40 +0000 Subject: [PATCH 5/5] [CI] Auto-commit changed files from 'make api-docs' --- oas_docs/output/kibana.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index e17bfda16de56..cb7d39cae0cab 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -6666,14 +6666,9 @@ paths: - cases /api/cases/{caseId}/files: post: - description: > - Attach a file to a case. You must have `all` privileges for the - **Cases** feature in the **Management**, **Observability**, or - **Security** section of the Kibana feature privileges, depending on the - owner of the case you're updating. The request must include: - + description: | + Attach a file to a case. You must have `all` privileges for the **Cases** feature in the **Management**, **Observability**, or **Security** section of the Kibana feature privileges, depending on the owner of the case you're updating. The request must include: - The `Content-Type: multipart/form-data` HTTP header. - - The location of the file that is being uploaded. operationId: addCaseFileDefaultSpace parameters: @@ -43673,9 +43668,7 @@ components: - $ref: '#/components/schemas/Cases_add_user_comment_request_properties' title: Add case comment request Cases_add_case_file_request: - description: >- - Defines the file that will be attached to the case. Optional parameters - will be generated automatically from the file metadata if not defined. + description: Defines the file that will be attached to the case. Optional parameters will be generated automatically from the file metadata if not defined. type: object properties: file: @@ -43683,10 +43676,7 @@ components: format: binary type: string filename: - description: >- - The desired name of the file being attached to the case, it can be - different than the name of the file in the filesystem. **This should - not include the file extension.** + description: The desired name of the file being attached to the case, it can be different than the name of the file in the filesystem. **This should not include the file extension.** type: string required: - file