From 24375200bfcf5d920597a7e804a2af17ac70d857 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 13 Nov 2024 14:01:59 +0000 Subject: [PATCH] Add default ILM policies and switch to ILM for apm-data plugin (#115687) (cherry picked from commit 6325e46231c15da744b3ad28811279b03f1299d0) --- docs/changelog/115687.yaml | 5 ++ .../test/rest/ESRestTestCase.java | 21 +++++- .../logs-apm.app-fallback@ilm.yaml | 1 - .../logs-apm.error-fallback@ilm.yaml | 1 - .../metrics-apm.app-fallback@ilm.yaml | 1 - .../metrics-apm.internal-fallback@ilm.yaml | 1 - ....service_destination.10m-fallback@ilm.yaml | 1 - ...m.service_destination.1m-fallback@ilm.yaml | 1 - ....service_destination.60m-fallback@ilm.yaml | 1 - ...-apm.service_summary.10m-fallback@ilm.yaml | 1 - ...s-apm.service_summary.1m-fallback@ilm.yaml | 1 - ...-apm.service_summary.60m-fallback@ilm.yaml | 1 - ....service_transaction.10m-fallback@ilm.yaml | 1 - ...m.service_transaction.1m-fallback@ilm.yaml | 1 - ....service_transaction.60m-fallback@ilm.yaml | 1 - ...rics-apm.transaction.10m-fallback@ilm.yaml | 1 - ...trics-apm.transaction.1m-fallback@ilm.yaml | 1 - ...rics-apm.transaction.60m-fallback@ilm.yaml | 1 - .../traces-apm-fallback@ilm.yaml | 1 - .../traces-apm.rum-fallback@ilm.yaml | 1 - .../traces-apm.sampled-fallback@ilm.yaml | 1 - .../logs-apm.app_logs-default_policy.yaml | 16 +++++ .../logs-apm.error_logs-default_policy.yaml | 16 +++++ ...etrics-apm.app_metrics-default_policy.yaml | 16 +++++ ...s-apm.internal_metrics-default_policy.yaml | 16 +++++ ...estination_10m_metrics-default_policy.yaml | 16 +++++ ...destination_1m_metrics-default_policy.yaml | 16 +++++ ...estination_60m_metrics-default_policy.yaml | 16 +++++ ...ce_summary_10m_metrics-default_policy.yaml | 16 +++++ ...ice_summary_1m_metrics-default_policy.yaml | 16 +++++ ...ce_summary_60m_metrics-default_policy.yaml | 16 +++++ ...ransaction_10m_metrics-default_policy.yaml | 16 +++++ ...transaction_1m_metrics-default_policy.yaml | 16 +++++ ...ransaction_60m_metrics-default_policy.yaml | 16 +++++ ...ransaction_10m_metrics-default_policy.yaml | 16 +++++ ...transaction_1m_metrics-default_policy.yaml | 16 +++++ ...ransaction_60m_metrics-default_policy.yaml | 16 +++++ .../traces-apm.rum_traces-default_policy.yaml | 16 +++++ ...ces-apm.sampled_traces-default_policy.yaml | 13 ++++ .../traces-apm.traces-default_policy.yaml | 16 +++++ .../src/main/resources/resources.yaml | 23 ++++++- .../APMIndexTemplateRegistryTests.java | 66 +++++++++++++++++-- .../xpack/core/ilm/LifecyclePolicyUtils.java | 30 ++++++--- .../core/template/YamlTemplateRegistry.java | 39 ++++++++++- 44 files changed, 468 insertions(+), 36 deletions(-) create mode 100644 docs/changelog/115687.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml diff --git a/docs/changelog/115687.yaml b/docs/changelog/115687.yaml new file mode 100644 index 0000000000000..1180b4627c635 --- /dev/null +++ b/docs/changelog/115687.yaml @@ -0,0 +1,5 @@ +pr: 115687 +summary: Add default ILM policies and switch to ILM for apm-data plugin +area: Data streams +type: feature +issues: [] diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 29dbdc823305c..5076f97ec96d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -820,7 +820,26 @@ protected Set preserveILMPolicyIds() { ".fleet-file-tohost-meta-ilm-policy", ".deprecation-indexing-ilm-policy", ".monitoring-8-ilm-policy", - "behavioral_analytics-events-default_policy" + "behavioral_analytics-events-default_policy", + "logs-apm.app_logs-default_policy", + "logs-apm.error_logs-default_policy", + "metrics-apm.app_metrics-default_policy", + "metrics-apm.internal_metrics-default_policy", + "metrics-apm.service_destination_10m_metrics-default_policy", + "metrics-apm.service_destination_1m_metrics-default_policy", + "metrics-apm.service_destination_60m_metrics-default_policy", + "metrics-apm.service_summary_10m_metrics-default_policy", + "metrics-apm.service_summary_1m_metrics-default_policy", + "metrics-apm.service_summary_60m_metrics-default_policy", + "metrics-apm.service_transaction_10m_metrics-default_policy", + "metrics-apm.service_transaction_1m_metrics-default_policy", + "metrics-apm.service_transaction_60m_metrics-default_policy", + "metrics-apm.transaction_10m_metrics-default_policy", + "metrics-apm.transaction_1m_metrics-default_policy", + "metrics-apm.transaction_60m_metrics-default_policy", + "traces-apm.rum_traces-default_policy", + "traces-apm.sampled_traces-default_policy", + "traces-apm.traces-default_policy" ); } diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml index 627d6345d6b77..07b1bd9cbcd7e 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: logs-apm.app_logs-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml index a97c004fa1707..85d8452506493 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: logs-apm.error_logs-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml index 23130ef8400c2..9610b38923bbb 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.app_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml index 7fbf7941ea538..625db0ddf063d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.internal_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml index a7fe53f56474b..aff33171c4b58 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_destination_10m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml index 274c8c604582c..46f0e74d66d6c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_destination_1m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml index 2d894dec48ac4..01b5057fb4124 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_destination_60m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml index 612bf6ff7c1d0..9a2c8cc4e0f0b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_summary_10m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml index e86eb803de63f..011380ea40c1f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_summary_1m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml index 4b4e14eb711e0..32b4840d26a4c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_summary_60m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml index fc03e62bcc4cd..80118df29877f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_transaction_10m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml index 9021506be3d33..673c17d972c5e 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_transaction_1m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml index 961b0a35543a7..a04870d4224ca 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.service_transaction_60m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml index e2504def2505c..abadcbf58bd62 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.transaction_10m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml index 7bfbcc7bb8052..b8af9a8b96f56 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.transaction_1m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml index 48e6ee5a09c20..3d13284934ade 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: metrics-apm.transaction_60m_metrics-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml index 360693e97ae2b..7fc2ca2343ea5 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: traces-apm.traces-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml index 6dfd79341424f..207307b396dc6 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: traces-apm.rum_traces-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml index 2193dbf58488b..975e19693b656 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml @@ -8,4 +8,3 @@ template: index: lifecycle: name: traces-apm.sampled_traces-default_policy - prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml new file mode 100644 index 0000000000000..ab73c1c357897 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 10d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml new file mode 100644 index 0000000000000..ab73c1c357897 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 10d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml new file mode 100644 index 0000000000000..19fbd66e954cb --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml new file mode 100644 index 0000000000000..19fbd66e954cb --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..15c067d6720af --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 14d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 180d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..4f618ce4ff51b --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 7d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..277ef59f11300 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 390d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..15c067d6720af --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 14d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 180d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..4f618ce4ff51b --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 7d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..277ef59f11300 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 390d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..15c067d6720af --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 14d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 180d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..4f618ce4ff51b --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 7d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..277ef59f11300 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 390d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..15c067d6720af --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 14d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 180d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..4f618ce4ff51b --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 7d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml new file mode 100644 index 0000000000000..277ef59f11300 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 390d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml new file mode 100644 index 0000000000000..19fbd66e954cb --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 90d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml new file mode 100644 index 0000000000000..2c25f5ec568c6 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml @@ -0,0 +1,13 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 1h + delete: + min_age: 1h + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml new file mode 100644 index 0000000000000..ab73c1c357897 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml @@ -0,0 +1,16 @@ +--- +_meta: + description: Default ILM policy for APM managed datastreams + managed: true +phases: + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + delete: + min_age: 10d + actions: + delete: {} diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index a178b768c4fe9..fa209cdec3695 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 10 +version: 11 component-templates: # Data lifecycle. @@ -97,3 +97,24 @@ ingest-pipelines: - metrics-apm@pipeline: dependencies: - apm@pipeline + +lifecycle-policies: + - logs-apm.app_logs-default_policy + - logs-apm.error_logs-default_policy + - metrics-apm.app_metrics-default_policy + - metrics-apm.internal_metrics-default_policy + - metrics-apm.service_destination_10m_metrics-default_policy + - metrics-apm.service_destination_1m_metrics-default_policy + - metrics-apm.service_destination_60m_metrics-default_policy + - metrics-apm.service_summary_10m_metrics-default_policy + - metrics-apm.service_summary_1m_metrics-default_policy + - metrics-apm.service_summary_60m_metrics-default_policy + - metrics-apm.service_transaction_10m_metrics-default_policy + - metrics-apm.service_transaction_1m_metrics-default_policy + - metrics-apm.service_transaction_60m_metrics-default_policy + - metrics-apm.transaction_10m_metrics-default_policy + - metrics-apm.transaction_1m_metrics-default_policy + - metrics-apm.transaction_60m_metrics-default_policy + - traces-apm.rum_traces-default_policy + - traces-apm.sampled_traces-default_policy + - traces-apm.traces-default_policy diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index ff1debdea79b1..4a2b9265b3b05 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -44,6 +44,8 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; import org.elasticsearch.xpack.stack.StackTemplateRegistry; import org.elasticsearch.xpack.stack.StackTemplateRegistryAccessor; @@ -57,6 +59,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -133,6 +136,7 @@ public void testThatDisablingRegistryDoesNothing() throws Exception { assertThat(apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet(), hasSize(0)); assertThat(apmIndexTemplateRegistry.getComposableTemplateConfigs().entrySet(), hasSize(0)); assertThat(apmIndexTemplateRegistry.getIngestPipelines(), hasSize(0)); + assertThat(apmIndexTemplateRegistry.getLifecyclePolicies(), hasSize(0)); client.setVerifier((a, r, l) -> { fail("if the registry is disabled nothing should happen"); @@ -145,6 +149,7 @@ public void testThatDisablingRegistryDoesNothing() throws Exception { assertThat(apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet(), not(hasSize(0))); assertThat(apmIndexTemplateRegistry.getComposableTemplateConfigs().entrySet(), not(hasSize(0))); assertThat(apmIndexTemplateRegistry.getIngestPipelines(), not(hasSize(0))); + assertThat(apmIndexTemplateRegistry.getLifecyclePolicies(), not(hasSize(0))); } public void testThatIndependentTemplatesAreAddedImmediatelyIfMissing() throws Exception { @@ -154,23 +159,26 @@ public void testThatIndependentTemplatesAreAddedImmediatelyIfMissing() throws Ex AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + AtomicInteger actualILMPolicies = new AtomicInteger(0); client.setVerifier( (action, request, listener) -> verifyActions( actualInstalledIndexTemplates, actualInstalledComponentTemplates, actualInstalledIngestPipelines, + actualILMPolicies, action, request, listener ) ); - apmIndexTemplateRegistry.clusterChanged(createClusterChangedEvent(Map.of(), Map.of(), nodes)); + apmIndexTemplateRegistry.clusterChanged(createClusterChangedEvent(Map.of(), Map.of(), List.of(), Map.of(), nodes)); assertBusy(() -> assertThat(actualInstalledIngestPipelines.get(), equalTo(getIndependentPipelineConfigs().size()))); assertBusy(() -> assertThat(actualInstalledComponentTemplates.get(), equalTo(getIndependentComponentTemplateConfigs().size()))); + assertBusy(() -> assertThat(actualILMPolicies.get(), equalTo(getIndependentLifecyclePolicies().size()))); - // index templates should not be installed as they are dependent in component templates and ingest pipelines + // index templates should not be installed as they are dependent on component templates and ingest pipelines assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); } @@ -201,6 +209,31 @@ public void testIngestPipelines() throws Exception { }); } + public void testILMLifecyclePolicies() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + final List lifecyclePolicies = apmIndexTemplateRegistry.getLifecyclePolicies(); + assertThat(lifecyclePolicies, is(not(empty()))); + + final Set expectedILMPolicies = apmIndexTemplateRegistry.getLifecyclePolicies() + .stream() + .map(LifecyclePolicy::getName) + .collect(Collectors.toSet()); + final Set installedILMPolicies = ConcurrentHashMap.newKeySet(lifecyclePolicies.size()); + client.setVerifier((a, r, l) -> { + if (a == ILMActions.PUT && r instanceof PutLifecycleRequest putLifecycleRequest) { + if (expectedILMPolicies.contains(putLifecycleRequest.getPolicy().getName())) { + installedILMPolicies.add(putLifecycleRequest.getPolicy().getName()); + } + } + return AcknowledgedResponse.TRUE; + }); + + apmIndexTemplateRegistry.clusterChanged(createClusterChangedEvent(Map.of(), Map.of(), List.of(), Map.of(), nodes)); + assertBusy(() -> { assertThat(installedILMPolicies, equalTo(expectedILMPolicies)); }); + } + public void testComponentTemplates() throws Exception { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); @@ -208,12 +241,14 @@ public void testComponentTemplates() throws Exception { AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + AtomicInteger actualILMPolicies = new AtomicInteger(0); client.setVerifier( (action, request, listener) -> verifyActions( actualInstalledIndexTemplates, actualInstalledComponentTemplates, actualInstalledIngestPipelines, + actualILMPolicies, action, request, listener @@ -224,6 +259,9 @@ public void testComponentTemplates() throws Exception { Map.of(), Map.of(), apmIndexTemplateRegistry.getIngestPipelines().stream().map(IngestPipelineConfig::getId).collect(Collectors.toList()), + apmIndexTemplateRegistry.getLifecyclePolicies() + .stream() + .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity())), nodes ) ); @@ -237,8 +275,10 @@ public void testComponentTemplates() throws Exception { // ingest pipelines should not have been installed as we used a cluster state that includes them already assertThat(actualInstalledIngestPipelines.get(), equalTo(0)); - // index templates should not be installed as they are dependent in component templates and ingest pipelines + // index templates should not be installed as they are dependent on component templates and ingest pipelines assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); + // ilm policies should not have been installed as we used a cluster state that includes them already + assertThat(actualILMPolicies.get(), equalTo(0)); } public void testIndexTemplates() throws Exception { @@ -248,12 +288,14 @@ public void testIndexTemplates() throws Exception { AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + AtomicInteger actualILMPolicies = new AtomicInteger(0); client.setVerifier( (action, request, listener) -> verifyActions( actualInstalledIndexTemplates, actualInstalledComponentTemplates, actualInstalledIngestPipelines, + actualILMPolicies, action, request, listener @@ -272,6 +314,9 @@ public void testIndexTemplates() throws Exception { componentTemplates, Map.of(), apmIndexTemplateRegistry.getIngestPipelines().stream().map(IngestPipelineConfig::getId).collect(Collectors.toList()), + apmIndexTemplateRegistry.getLifecyclePolicies() + .stream() + .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity())), nodes ) ); @@ -280,9 +325,11 @@ public void testIndexTemplates() throws Exception { () -> assertThat(actualInstalledIndexTemplates.get(), equalTo(apmIndexTemplateRegistry.getComposableTemplateConfigs().size())) ); - // ingest pipelines and component templates should not have been installed as we used a cluster state that includes them already + // ingest pipelines, component templates, and lifecycle policies should not have been installed as we used a cluster state that + // includes them already assertThat(actualInstalledComponentTemplates.get(), equalTo(0)); assertThat(actualInstalledIngestPipelines.get(), equalTo(0)); + assertThat(actualILMPolicies.get(), equalTo(0)); } public void testIndexTemplateConventions() throws Exception { @@ -408,10 +455,18 @@ private List getIndependentPipelineConfigs() { .collect(Collectors.toList()); } + private Map getIndependentLifecyclePolicies() { + // All lifecycle policies are independent + return apmIndexTemplateRegistry.getLifecyclePolicies() + .stream() + .collect(Collectors.toMap(LifecyclePolicy::getName, Function.identity())); + } + private ActionResponse verifyActions( AtomicInteger indexTemplatesCounter, AtomicInteger componentTemplatesCounter, AtomicInteger ingestPipelinesCounter, + AtomicInteger ilmPolicyCounter, ActionType action, ActionRequest request, ActionListener listener @@ -430,6 +485,9 @@ private ActionResponse verifyActions( } else if (action == PutPipelineTransportAction.TYPE) { ingestPipelinesCounter.incrementAndGet(); return AcknowledgedResponse.TRUE; + } else if (action == ILMActions.PUT) { + ilmPolicyCounter.incrementAndGet(); + return AcknowledgedResponse.TRUE; } else { fail("client called with unexpected request:" + request.toString()); return null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java index 4fb94dce1dcd0..8fe8c8835b98d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java @@ -23,6 +23,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.template.resources.TemplateResources; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -48,19 +49,32 @@ public static LifecyclePolicy loadPolicy( source = replaceVariables(source, variables); validate(source); - try ( - XContentParser parser = XContentType.JSON.xContent() - .createParser(XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry), source) - ) { - LifecyclePolicy policy = LifecyclePolicy.parse(parser, name); - policy.validate(); - return policy; - } + return parsePolicy(source, name, xContentRegistry, XContentType.JSON); } catch (Exception e) { throw new IllegalArgumentException("unable to load policy [" + name + "] from [" + resource + "]", e); } } + /** + * Parses lifecycle policy based on the provided content type without doing any variable substitution. + * It is caller's responsibility to do any variable substitution if required. + */ + public static LifecyclePolicy parsePolicy( + String rawPolicy, + String name, + NamedXContentRegistry xContentRegistry, + XContentType contentType + ) throws IOException { + try ( + XContentParser parser = contentType.xContent() + .createParser(XContentParserConfiguration.EMPTY.withRegistry(xContentRegistry), rawPolicy) + ) { + LifecyclePolicy policy = LifecyclePolicy.parse(parser, name); + policy.validate(); + return policy; + } + } + private static String replaceVariables(String template, Map variables) { for (Map.Entry variable : variables.entrySet()) { template = replaceVariable(template, variable.getKey(), variable.getValue()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index c8ddd46c5912f..a30236b2fef28 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.metadata.ComponentTemplate; @@ -22,7 +23,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.yaml.YamlXContent; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyUtils; import java.io.IOException; import java.util.Collections; @@ -48,6 +52,7 @@ public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { private final Map componentTemplates; private final Map composableIndexTemplates; private final List ingestPipelines; + private final List lifecyclePolicies; private final FeatureService featureService; private volatile boolean enabled; @@ -84,6 +89,7 @@ public YamlTemplateRegistry( final List componentTemplateNames = (List) resources.get("component-templates"); final List indexTemplateNames = (List) resources.get("index-templates"); final List ingestPipelineConfigs = (List) resources.get("ingest-pipelines"); + final List lifecyclePolicyConfigs = (List) resources.get("lifecycle-policies"); componentTemplates = Optional.ofNullable(componentTemplateNames) .orElse(Collections.emptyList()) @@ -110,9 +116,16 @@ public YamlTemplateRegistry( ); }) .collect(Collectors.toList()); + lifecyclePolicies = Optional.ofNullable(lifecyclePolicyConfigs) + .orElse(Collections.emptyList()) + .stream() + .map(o -> (String) o) + .filter(templateFilter) + .map(this::loadLifecyclePolicy) + .collect(Collectors.toList()); this.featureService = featureService; } catch (IOException e) { - throw new RuntimeException(e); + throw new ElasticsearchException(e); } } @@ -178,6 +191,15 @@ public List getIngestPipelines() { } } + @Override + public List getLifecyclePolicies() { + if (enabled) { + return lifecyclePolicies; + } else { + return Collections.emptyList(); + } + } + protected abstract String getVersionProperty(); private ComponentTemplate loadComponentTemplate(String name, int version) { @@ -192,7 +214,7 @@ private ComponentTemplate loadComponentTemplate(String name, int version) { return ComponentTemplate.parse(parser); } } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); + throw new ElasticsearchException("failed to load " + getName() + " Ingest plugin's component template: " + name, e); } } @@ -208,7 +230,7 @@ private ComposableIndexTemplate loadIndexTemplate(String name, int version) { return ComposableIndexTemplate.parse(parser); } } catch (Exception e) { - throw new RuntimeException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); + throw new ElasticsearchException("failed to load " + getName() + " Ingest plugin's index template: " + name, e); } } @@ -226,6 +248,17 @@ private IngestPipelineConfig loadIngestPipeline(String name, int version, @Nulla ); } + // IndexTemplateRegistry ensures that ILM lifecycle policies are not loaded + // when in DSL only mode. + private LifecyclePolicy loadLifecyclePolicy(String name) { + try { + var rawPolicy = loadResource(this.getClass(), "/lifecycle-policies/" + name + ".yaml"); + return LifecyclePolicyUtils.parsePolicy(rawPolicy, name, LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY, XContentType.YAML); + } catch (IOException e) { + throw new ElasticsearchException(e); + } + } + @Override protected boolean applyRolloverAfterTemplateV2Update() { return true;