From 317d57f8d4b9ca29691d628d428781d317ee04c2 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Fri, 6 Sep 2024 20:35:28 +0100 Subject: [PATCH 01/14] [plugin/apm-data]Fallback to ILM for non-dsl-only mode if DSL unavailable --- .../test/ClusterServiceUtils.java | 10 +- .../logs-apm.app-fallback@ilm.yaml | 11 ++ .../logs-apm.error-fallback@ilm.yaml | 11 ++ .../metrics-apm.app-fallback@ilm.yaml | 11 ++ .../metrics-apm.internal-fallback@ilm.yaml | 11 ++ ....service_destination.10m-fallback@ilm.yaml | 11 ++ ...m.service_destination.1m-fallback@ilm.yaml | 11 ++ ....service_destination.60m-fallback@ilm.yaml | 11 ++ ...-apm.service_summary.10m-fallback@ilm.yaml | 11 ++ ...s-apm.service_summary.1m-fallback@ilm.yaml | 11 ++ ...-apm.service_summary.60m-fallback@ilm.yaml | 11 ++ ....service_transaction.10m-fallback@ilm.yaml | 11 ++ ...m.service_transaction.1m-fallback@ilm.yaml | 11 ++ ....service_transaction.60m-fallback@ilm.yaml | 11 ++ ...rics-apm.transaction.10m-fallback@ilm.yaml | 11 ++ ...trics-apm.transaction.1m-fallback@ilm.yaml | 11 ++ ...rics-apm.transaction.60m-fallback@ilm.yaml | 11 ++ .../traces-apm-fallback@ilm.yaml | 11 ++ .../traces-apm.rum-fallback@ilm.yaml | 11 ++ .../traces-apm.sampled-fallback@ilm.yaml | 11 ++ .../logs-apm.app@template.yaml | 1 + .../logs-apm.error@template.yaml | 1 + .../metrics-apm.app@template.yaml | 1 + .../metrics-apm.internal@template.yaml | 1 + ...-apm.service_destination.10m@template.yaml | 1 + ...s-apm.service_destination.1m@template.yaml | 1 + ...-apm.service_destination.60m@template.yaml | 1 + ...rics-apm.service_summary.10m@template.yaml | 1 + ...trics-apm.service_summary.1m@template.yaml | 1 + ...rics-apm.service_summary.60m@template.yaml | 1 + ...-apm.service_transaction.10m@template.yaml | 1 + ...s-apm.service_transaction.1m@template.yaml | 1 + ...-apm.service_transaction.60m@template.yaml | 1 + .../metrics-apm.transaction.10m@template.yaml | 1 + .../metrics-apm.transaction.1m@template.yaml | 1 + .../metrics-apm.transaction.60m@template.yaml | 1 + .../traces-apm.rum@template.yaml | 1 + .../traces-apm.sampled@template.yaml | 1 + .../index-templates/traces-apm@template.yaml | 1 + .../src/main/resources/resources.yaml | 22 +++- .../xpack/apmdata/APMDSLOnlyTests.java | 106 ++++++++++++++++++ .../APMIndexTemplateRegistryTests.java | 14 +++ .../xpack/core/template/ResourceUtils.java | 3 + .../core/template/YamlTemplateRegistry.java | 16 ++- 44 files changed, 394 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml create mode 100644 x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index fc4a39acf853b..8b723bbebfcd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -121,11 +121,17 @@ public static ClusterService createClusterService(ThreadPool threadPool, Cluster public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode, ClusterSettings clusterSettings) { Settings settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").build(); + return createClusterService(threadPool, localNode, settings, clusterSettings); + } + + public static ClusterService createClusterService( + ThreadPool threadPool, DiscoveryNode localNode, Settings nodeSettings, ClusterSettings clusterSettings + ) { ClusterService clusterService = new ClusterService( - settings, + nodeSettings, clusterSettings, threadPool, - new TaskManager(settings, threadPool, Collections.emptySet(), Tracer.NOOP) + new TaskManager(nodeSettings, threadPool, Collections.emptySet(), Tracer.NOOP) ); clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService()); ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName())) diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml new file mode 100644 index 0000000000000..9d49bb027c217 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for logs-apm.app if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: logs-apm.app_logs-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml new file mode 100644 index 0000000000000..b3e2bd2a631a9 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for logs-apm.error if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: logs-apm.error_logs-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml new file mode 100644 index 0000000000000..99acbea08ec04 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.app if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.app_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml new file mode 100644 index 0000000000000..cbc93e77bc1b0 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.internal if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.internal_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml new file mode 100644 index 0000000000000..8395cc713fd5e --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_destination.10m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_destination_10m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml new file mode 100644 index 0000000000000..81f3a5da9803d --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_destination.1m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_destination_1m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml new file mode 100644 index 0000000000000..5278114a7cc1f --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_destination.60m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_destination_60m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml new file mode 100644 index 0000000000000..18752839eba47 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_summary.10m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_summary_10m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml new file mode 100644 index 0000000000000..118b76438e763 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_summary.1m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_summary_1m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml new file mode 100644 index 0000000000000..c0d825d6cbb31 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_summary.60m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_summary_60m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml new file mode 100644 index 0000000000000..7f4c0310ed007 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_transaction.10m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_transaction_10m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml new file mode 100644 index 0000000000000..44ea69d476954 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_transaction.1m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_transaction_1m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml new file mode 100644 index 0000000000000..5be8368ab1efb --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.service_transaction.60m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.service_transaction_60m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml new file mode 100644 index 0000000000000..6bd837af53e79 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.transaction.10m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.transaction_10m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml new file mode 100644 index 0000000000000..468dff5b81a8d --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.transaction.1m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.transaction_1m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml new file mode 100644 index 0000000000000..17cf9c3316200 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for metrics-apm.transaction.60m if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: metrics-apm.transaction_60m_metrics-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml new file mode 100644 index 0000000000000..7febfb0dbdd1e --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for traces-apm if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: traces-apm.traces-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml new file mode 100644 index 0000000000000..3c894601f6717 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for traces-apm.rum if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: traces-apm.rum_traces-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml new file mode 100644 index 0000000000000..bd892f8e64e83 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Fallback to ILM policy for traces-apm.sampled if DSL not defined + managed: true +template: + settings: + index: + lifecycle: + name: traces-apm.sampled_traces-default_policy + prefer_ilm: false diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml index 21cad50f3fe90..a84d5b899d75c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml @@ -12,6 +12,7 @@ composed_of: - apm@settings - apm-10d@lifecycle - logs-apm@settings +- logs-apm.app-fallback@ilm - logs@custom - logs-apm.app@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml index 2cfa7b454722f..71b7859230d16 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml @@ -14,6 +14,7 @@ composed_of: - apm-10d@lifecycle - logs-apm@settings - logs-apm.error@mappings +- logs-apm.error-fallback@ilm - logs@custom - logs-apm.error@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml index a3c7ab7c05193..e8f8ce3b25760 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml @@ -13,6 +13,7 @@ composed_of: - apm-90d@lifecycle - metrics-apm@mappings - metrics-apm@settings +- metrics-apm.app-fallback@ilm - metrics@custom - metrics-apm.app@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml index 4c7df377a6cfa..7e73dde13156d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml @@ -14,6 +14,7 @@ composed_of: - apm-90d@lifecycle - metrics-apm@mappings - metrics-apm@settings +- metrics-apm.internal-fallback@ilm - metrics@custom - metrics-apm.internal@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml index 63c9ff9c3b988..f10da8d87e025 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings +- metrics-apm.service_destination.10m-fallback@ilm - metrics@custom - metrics-apm.service_destination.10m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml index 6995a2d09b12e..7e5ab0f2d87b2 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml @@ -15,6 +15,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings +- metrics-apm.service_destination.1m-fallback@ilm - metrics@custom - metrics-apm.service_destination.1m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml index b39d0beca3740..a087e8ea8b403 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_destination@mappings +- metrics-apm.service_destination.60m-fallback@ilm - metrics@custom - metrics-apm.service_destination.60m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml index 8d92b21866bb8..37781991d9c4d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings +- metrics-apm.service_summary.10m-fallback@ilm - metrics@custom - metrics-apm.service_summary.10m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml index de19df330aa0e..0609b6af02d05 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml @@ -15,6 +15,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings +- metrics-apm.service_summary.1m-fallback@ilm - metrics@custom - metrics-apm.service_summary.1m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml index 002676eb08cc1..f9eca377482c3 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_summary@mappings +- metrics-apm.service_summary.60m-fallback@ilm - metrics@custom - metrics-apm.service_summary.60m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml index 549af3942dcd3..47605b19424c3 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings +- metrics-apm.service_transaction.10m-fallback@ilm - metrics@custom - metrics-apm.service_transaction.10m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml index 9bdacfc337663..ad8a983535d3c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml @@ -15,6 +15,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings +- metrics-apm.service_transaction.1m-fallback@ilm - metrics@custom - metrics-apm.service_transaction.1m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml index 8bcbeb53c74fe..dea2c3382ecca 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.service_transaction@mappings +- metrics-apm.service_transaction.60m-fallback@ilm - metrics@custom - metrics-apm.service_transaction.60m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml index 68c1dc0f31c1e..08c0494d4cf58 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings +- metrics-apm.transaction.10m-fallback@ilm - metrics@custom - metrics-apm.transaction.10m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml index 6065f6e12f999..221a988e2371f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml @@ -15,6 +15,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings +- metrics-apm.transaction.1m-fallback@ilm - metrics@custom - metrics-apm.transaction.1m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml index d8889ceb63f87..2f50fe63875fe 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml @@ -16,6 +16,7 @@ composed_of: - metrics-apm@mappings - metrics-apm@settings - metrics-apm.transaction@mappings +- metrics-apm.transaction.60m-fallback@ilm - metrics@custom - metrics-apm.transaction.60m@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml index d299481ff6e21..a6aeaff2b6f40 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml @@ -14,6 +14,7 @@ composed_of: - apm-90d@lifecycle - traces-apm@mappings - traces-apm.rum@mappings +- traces-apm.rum-fallback@ilm - traces@custom - traces-apm.rum@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml index 81457e2f204cb..9ec276fdad580 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml @@ -11,6 +11,7 @@ composed_of: - traces@mappings - apm@mappings - apm@settings +- traces-apm.sampled-fallback@ilm - traces@custom - traces-apm.sampled@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml index fda953171b793..3808f132d0106 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml @@ -13,6 +13,7 @@ composed_of: - apm@settings - apm-10d@lifecycle - traces-apm@mappings +- traces-apm-fallback@ilm - traces@custom - traces-apm@custom - ecs@mappings diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index 0502a8c559ff6..a178b768c4fe9 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 9 +version: 10 component-templates: # Data lifecycle. @@ -25,6 +25,26 @@ component-templates: - metrics-apm.transaction@mappings - traces-apm@mappings - traces-apm.rum@mappings + # The below files allow to fallback to ILM if DSL configurations are not present + - logs-apm.app-fallback@ilm + - logs-apm.error-fallback@ilm + - metrics-apm.app-fallback@ilm + - metrics-apm.internal-fallback@ilm + - metrics-apm.service_destination.10m-fallback@ilm + - metrics-apm.service_destination.1m-fallback@ilm + - metrics-apm.service_destination.60m-fallback@ilm + - metrics-apm.service_summary.10m-fallback@ilm + - metrics-apm.service_summary.1m-fallback@ilm + - metrics-apm.service_summary.60m-fallback@ilm + - metrics-apm.service_transaction.10m-fallback@ilm + - metrics-apm.service_transaction.1m-fallback@ilm + - metrics-apm.service_transaction.60m-fallback@ilm + - metrics-apm.transaction.10m-fallback@ilm + - metrics-apm.transaction.1m-fallback@ilm + - metrics-apm.transaction.60m-fallback@ilm + - traces-apm-fallback@ilm + - traces-apm.rum-fallback@ilm + - traces-apm.sampled-fallback@ilm index-templates: - logs-apm.app@template diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java new file mode 100644 index 0000000000000..43ac8e9f381a2 --- /dev/null +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamFeatures; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xpack.stack.StackTemplateRegistry; +import org.elasticsearch.xpack.stack.StackTemplateRegistryAccessor; +import org.junit.After; +import org.junit.Before; + +import java.util.Set; +import java.util.Map; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.common.settings.Setting.boolSetting; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.not; + +public class APMDSLOnlyTests extends ESTestCase { + private APMIndexTemplateRegistry apmIndexTemplateRegistry; + private StackTemplateRegistryAccessor stackTemplateRegistryAccessor; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + final ClusterSettings clusterSettings = new ClusterSettings( + Settings.builder().put( + DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, + "true" + ).build(), + Stream.concat( + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), + Set.of( + APMPlugin.APM_DATA_REGISTRY_ENABLED + ).stream() + ).collect(Collectors.toSet()) + ); + + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("node", "node"); + Settings nodeSettings = Settings.builder() + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, true) + .build(); + ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, nodeSettings, clusterSettings); + FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures())); + stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor( + new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService) + ); + + apmIndexTemplateRegistry = new APMIndexTemplateRegistry( + Settings.EMPTY, + clusterService, + threadPool, + client, + NamedXContentRegistry.EMPTY, + featureService + ); + apmIndexTemplateRegistry.setEnabled(true); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testNoILMComponentTemplateInstalled() throws Exception { + for (Map.Entry entry : apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet()) { + final String name = entry.getKey(); + final int atIndex = name.lastIndexOf('@'); + assertThat(atIndex, not(equalTo(-1))); + // No ILM templates should have been loaded + assertThat(name.substring(atIndex + 1), not(equalTo("ilm"))); + } + } +} diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index 1d6faa0f403d4..ba32e8194bf7f 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -376,6 +376,20 @@ public void testThatNothingIsInstalledWhenAllNodesAreNotUpdated() { apmIndexTemplateRegistry.clusterChanged(event); } + public void testILMComponentTemplatesInstalled() throws Exception { + int ilmFallbackCount = 0; + for (Map.Entry entry : apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet()) { + final String name = entry.getKey(); + final int atIndex = name.lastIndexOf('@'); + assertThat(atIndex, not(equalTo(-1))); + if ("ilm".equals(name.substring(atIndex + 1))) { + ilmFallbackCount++; + } + } + // Each index template should have a corresponding ILM fallback policy + assertThat(apmIndexTemplateRegistry.getComposableTemplateConfigs().size(), equalTo(ilmFallbackCount)); + } + private Map getIndependentComponentTemplateConfigs() { return apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet().stream().filter(template -> { Settings settings = template.getValue().template().settings(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java index 9840535989a7c..fc1c65be32ec3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java @@ -7,12 +7,15 @@ package org.elasticsearch.xpack.core.template; +import org.elasticsearch.xpack.core.template.resources.TemplateResources; + import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Map; public class ResourceUtils { + static byte[] loadVersionedResourceUTF8(Class clazz, String name, int version, String versionProperty) { return loadVersionedResourceUTF8(clazz, name, version, versionProperty, Map.of()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index a4e5d7cdc4b44..655dc309b4065 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -33,9 +33,12 @@ import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; /** - * Creates index templates and ingest pipelines based on YAML files from resources. + * Creates index templates and ingest pipelines based on YAML files defined in resources.yaml. + * It also allows ILM files to be managed separately as per the DSL only mode for the cluster. + * The ILM names MUST end with `@ilm` for this class to mark the resource as ILM resource. */ public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class); @@ -60,7 +63,7 @@ public YamlTemplateRegistry( FeatureService featureService ) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - + final boolean dslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); try { final Map resources = XContentHelper.convertToMap( YamlXContent.yamlXContent, @@ -77,11 +80,13 @@ public YamlTemplateRegistry( .orElse(Collections.emptyList()) .stream() .map(o -> (String) o) + .filter(tpl -> shouldLoadTemplate(tpl, dslOnlyMode)) .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); composableIndexTemplates = Optional.ofNullable(indexTemplateNames) .orElse(Collections.emptyList()) .stream() .map(o -> (String) o) + .filter(tpl -> shouldLoadTemplate(tpl, dslOnlyMode)) .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) .orElse(Collections.emptyList()) @@ -166,6 +171,13 @@ public List getIngestPipelines() { protected abstract String getVersionProperty(); + private boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { + if (dslOnlyMode && name.endsWith("@ilm")) { + return false; + } + return true; + } + private ComponentTemplate loadComponentTemplate(String name, int version) { try { final byte[] content = loadVersionedResourceUTF8( From 21ee4f24002e0f979af338fdc544621c04e6ca2f Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 18:46:26 +0100 Subject: [PATCH 02/14] Ignore fallback component templates if not found --- .../main/resources/index-templates/logs-apm.app@template.yaml | 1 + .../main/resources/index-templates/logs-apm.error@template.yaml | 1 + .../main/resources/index-templates/metrics-apm.app@template.yaml | 1 + .../resources/index-templates/metrics-apm.internal@template.yaml | 1 + .../metrics-apm.service_destination.10m@template.yaml | 1 + .../metrics-apm.service_destination.1m@template.yaml | 1 + .../metrics-apm.service_destination.60m@template.yaml | 1 + .../metrics-apm.service_summary.10m@template.yaml | 1 + .../index-templates/metrics-apm.service_summary.1m@template.yaml | 1 + .../metrics-apm.service_summary.60m@template.yaml | 1 + .../metrics-apm.service_transaction.10m@template.yaml | 1 + .../metrics-apm.service_transaction.1m@template.yaml | 1 + .../metrics-apm.service_transaction.60m@template.yaml | 1 + .../index-templates/metrics-apm.transaction.10m@template.yaml | 1 + .../index-templates/metrics-apm.transaction.1m@template.yaml | 1 + .../index-templates/metrics-apm.transaction.60m@template.yaml | 1 + .../main/resources/index-templates/traces-apm.rum@template.yaml | 1 + .../resources/index-templates/traces-apm.sampled@template.yaml | 1 + .../src/main/resources/index-templates/traces-apm@template.yaml | 1 + 19 files changed, 19 insertions(+) diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml index a84d5b899d75c..ff9fe7df03ab0 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml @@ -19,6 +19,7 @@ composed_of: ignore_missing_component_templates: - logs@custom - logs-apm.app@custom +- logs-apm.app-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml index 71b7859230d16..5f510df4ee065 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml @@ -21,6 +21,7 @@ composed_of: ignore_missing_component_templates: - logs@custom - logs-apm.error@custom +- logs-apm.error-fallback@ilm template: mappings: properties: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml index e8f8ce3b25760..ff40c533aad66 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml @@ -20,6 +20,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.app@custom +- metrics-apm.app-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml index 7e73dde13156d..4d1466f743672 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml @@ -21,6 +21,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.internal@custom +- metrics-apm.internal-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml index f10da8d87e025..d42f55ed0e67d 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_destination.10m@custom +- metrics-apm.service_destination.10m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml index 7e5ab0f2d87b2..1c2920febc159 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml @@ -22,6 +22,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_destination.1m@custom +- metrics-apm.service_destination.1m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml index a087e8ea8b403..44948e820a7b7 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_destination.60m@custom +- metrics-apm.service_destination.60m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml index 37781991d9c4d..ad18d23a2d928 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_summary.10m@custom +- metrics-apm.service_summary.10m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml index 0609b6af02d05..8f0faa960ff48 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml @@ -22,6 +22,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_summary.1m@custom +- metrics-apm.service_summary.1m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml index f9eca377482c3..97438c80bb92e 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_summary.60m@custom +- metrics-apm.service_summary.60m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml index 47605b19424c3..a941e3b41d775 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_transaction.10m@custom +- metrics-apm.service_transaction.10m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml index ad8a983535d3c..51d06f8bbbec7 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml @@ -22,6 +22,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_transaction.1m@custom +- metrics-apm.service_transaction.1m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml index dea2c3382ecca..f1432e7208776 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.service_transaction.60m@custom +- metrics-apm.service_transaction.60m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml index 08c0494d4cf58..21ed837d2107e 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.transaction.10m@custom +- metrics-apm.transaction.10m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml index 221a988e2371f..8dc59895cdbbd 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml @@ -22,6 +22,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.transaction.1m@custom +- metrics-apm.transaction.1m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml index 2f50fe63875fe..e0d8db2c3481f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml @@ -23,6 +23,7 @@ composed_of: ignore_missing_component_templates: - metrics@custom - metrics-apm.transaction.60m@custom +- metrics-apm.transaction.60m-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml index a6aeaff2b6f40..6421fcca80275 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml @@ -21,6 +21,7 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-apm.rum@custom +- traces-apm.rum-fallback@ilm template: settings: index: diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml index 9ec276fdad580..9170306f3e8c3 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml @@ -18,6 +18,7 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-apm.sampled@custom +- traces-apm.sampled-fallback@ilm template: lifecycle: data_retention: 1h diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml index 3808f132d0106..d431c9c2a17eb 100644 --- a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml @@ -20,6 +20,7 @@ composed_of: ignore_missing_component_templates: - traces@custom - traces-apm@custom +- traces-apm-fallback@ilm template: settings: index: From 5cde6b08e4f248103cdf499cf5f7c52070477ba0 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 19:00:03 +0100 Subject: [PATCH 03/14] Remove unused imports --- .../org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java index 43ac8e9f381a2..3dbc636456901 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java @@ -8,13 +8,11 @@ package org.elasticsearch.xpack.apmdata; import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.datastreams.DataStreamFeatures; import org.elasticsearch.features.FeatureService; @@ -34,11 +32,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.common.settings.Setting.boolSetting; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; public class APMDSLOnlyTests extends ESTestCase { From a1d3bb610a8f9032d9c18ee95e9a34aafbaf5fcc Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 21:51:09 +0100 Subject: [PATCH 04/14] make it spotless --- .../elasticsearch/test/ClusterServiceUtils.java | 5 ++++- .../xpack/apmdata/APMDSLOnlyTests.java | 17 +++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index 8b723bbebfcd7..43d423374559a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -125,7 +125,10 @@ public static ClusterService createClusterService(ThreadPool threadPool, Discove } public static ClusterService createClusterService( - ThreadPool threadPool, DiscoveryNode localNode, Settings nodeSettings, ClusterSettings clusterSettings + ThreadPool threadPool, + DiscoveryNode localNode, + Settings nodeSettings, + ClusterSettings clusterSettings ) { ClusterService clusterService = new ClusterService( nodeSettings, diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java index 3dbc636456901..5c775116dc329 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java @@ -26,9 +26,9 @@ import org.junit.After; import org.junit.Before; -import java.util.Set; -import java.util.Map; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -44,16 +44,9 @@ public class APMDSLOnlyTests extends ESTestCase { @Before public void createRegistryAndClient() { final ClusterSettings clusterSettings = new ClusterSettings( - Settings.builder().put( - DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, - "true" - ).build(), - Stream.concat( - ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), - Set.of( - APMPlugin.APM_DATA_REGISTRY_ENABLED - ).stream() - ).collect(Collectors.toSet()) + Settings.builder().put(DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, "true").build(), + Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Set.of(APMPlugin.APM_DATA_REGISTRY_ENABLED).stream()) + .collect(Collectors.toSet()) ); threadPool = new TestThreadPool(this.getClass().getName()); From 3df9e7279bb34e8cb992aafe2a4b036a9e0bf1e7 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 22:05:53 +0100 Subject: [PATCH 05/14] fix style check --- .../org/elasticsearch/xpack/core/template/ResourceUtils.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java index fc1c65be32ec3..f52013ea5aa89 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/ResourceUtils.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.core.template; -import org.elasticsearch.xpack.core.template.resources.TemplateResources; - import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; From 2ba83babc0f031fffc3373c2ee61447d01ab57ef Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 22:31:36 +0100 Subject: [PATCH 06/14] Fix convention test --- .../xpack/apmdata/APMIndexTemplateRegistryTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index ba32e8194bf7f..ff1debdea79b1 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -312,11 +312,15 @@ public void testIndexTemplateConventions() throws Exception { // Each index template should be composed of the following optional component templates: // @custom // -@custom + // -fallback@ilm final List optionalComponentTemplates = template.composedOf() .stream() .filter(t -> template.getIgnoreMissingComponentTemplates().contains(t)) .toList(); - assertThat(optionalComponentTemplates, containsInAnyOrder(namePrefix + "@custom", dataStreamType + "@custom")); + assertThat( + optionalComponentTemplates, + containsInAnyOrder(namePrefix + "@custom", dataStreamType + "@custom", namePrefix + "-fallback@ilm") + ); // There should be no required custom component templates. final List requiredCustomComponentTemplates = template.getRequiredComponentTemplates() From 5da96b83f57121630f45896322e4c06eedd6d610 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Wed, 11 Sep 2024 22:44:06 +0100 Subject: [PATCH 07/14] Another spotless apply --- .../elasticsearch/xpack/core/template/YamlTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index 655dc309b4065..620d4e1a62729 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -31,9 +31,9 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; /** * Creates index templates and ingest pipelines based on YAML files defined in resources.yaml. From f86bec0a4082677fd4a2d1bb4ba7bbc9993530f4 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Thu, 12 Sep 2024 10:59:36 +0100 Subject: [PATCH 08/14] Add yaml rest test for ILM fallback --- .../resources/rest-api-spec/test/10_apm.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml index 4b45fda66835c..c591668f3549d 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml @@ -81,3 +81,27 @@ setup: - length: {hits.hits: 1} - match: {hits.hits.0.fields.event\.success_count: [1]} - match: {hits.hits.0.fields.span\.duration\.us: [123]} + +--- +"Test ILM fallback component-template installation with dsl-only mode false": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + cluster.get_component_template: + name: "*-fallback@ilm" + - length: {component_templates: 19} + - contains: {component_templates: {name: metrics-apm.app-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.internal-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_destination.1m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_destination.10m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_destination.60m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_summary.1m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_summary.10m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_summary.60m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_transaction.1m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_transaction.10m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.service_transaction.60m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.transaction.1m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.transaction.10m-fallback@ilm}} + - contains: {component_templates: {name: metrics-apm.transaction.60m-fallback@ilm}} From e3c33446fd6204730631efa032d407e194af5c8a Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Fri, 13 Sep 2024 14:54:19 +0100 Subject: [PATCH 09/14] Update DSL -> data stream lifecycle --- .../component-templates/logs-apm.app-fallback@ilm.yaml | 2 +- .../component-templates/logs-apm.error-fallback@ilm.yaml | 2 +- .../component-templates/metrics-apm.app-fallback@ilm.yaml | 2 +- .../component-templates/metrics-apm.internal-fallback@ilm.yaml | 2 +- .../metrics-apm.service_destination.10m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_destination.1m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_destination.60m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_summary.10m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_summary.1m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_summary.60m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_transaction.10m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_transaction.1m-fallback@ilm.yaml | 2 +- .../metrics-apm.service_transaction.60m-fallback@ilm.yaml | 2 +- .../metrics-apm.transaction.10m-fallback@ilm.yaml | 2 +- .../metrics-apm.transaction.1m-fallback@ilm.yaml | 2 +- .../metrics-apm.transaction.60m-fallback@ilm.yaml | 2 +- .../resources/component-templates/traces-apm-fallback@ilm.yaml | 2 +- .../component-templates/traces-apm.rum-fallback@ilm.yaml | 2 +- .../component-templates/traces-apm.sampled-fallback@ilm.yaml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml index 9d49bb027c217..627d6345d6b77 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for logs-apm.app if DSL not defined + description: Fallback to ILM policy for logs-apm.app if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml index b3e2bd2a631a9..a97c004fa1707 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for logs-apm.error if DSL not defined + description: Fallback to ILM policy for logs-apm.error if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml index 99acbea08ec04..23130ef8400c2 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.app if DSL not defined + description: Fallback to ILM policy for metrics-apm.app if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml index cbc93e77bc1b0..7fbf7941ea538 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.internal if DSL not defined + description: Fallback to ILM policy for metrics-apm.internal if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml index 8395cc713fd5e..a7fe53f56474b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_destination.10m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_destination.10m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml index 81f3a5da9803d..274c8c604582c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_destination.1m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_destination.1m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml index 5278114a7cc1f..2d894dec48ac4 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_destination.60m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_destination.60m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml index 18752839eba47..612bf6ff7c1d0 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_summary.10m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_summary.10m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml index 118b76438e763..e86eb803de63f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_summary.1m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_summary.1m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml index c0d825d6cbb31..4b4e14eb711e0 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_summary.60m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_summary.60m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml index 7f4c0310ed007..fc03e62bcc4cd 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_transaction.10m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_transaction.10m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml index 44ea69d476954..9021506be3d33 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_transaction.1m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_transaction.1m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml index 5be8368ab1efb..961b0a35543a7 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.service_transaction.60m if DSL not defined + description: Fallback to ILM policy for metrics-apm.service_transaction.60m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml index 6bd837af53e79..e2504def2505c 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.transaction.10m if DSL not defined + description: Fallback to ILM policy for metrics-apm.transaction.10m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml index 468dff5b81a8d..7bfbcc7bb8052 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.transaction.1m if DSL not defined + description: Fallback to ILM policy for metrics-apm.transaction.1m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml index 17cf9c3316200..48e6ee5a09c20 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for metrics-apm.transaction.60m if DSL not defined + description: Fallback to ILM policy for metrics-apm.transaction.60m if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml index 7febfb0dbdd1e..360693e97ae2b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for traces-apm if DSL not defined + description: Fallback to ILM policy for traces-apm if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml index 3c894601f6717..6dfd79341424f 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for traces-apm.rum if DSL not defined + description: Fallback to ILM policy for traces-apm.rum if data stream lifecycle not defined managed: true template: settings: diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml index bd892f8e64e83..2193dbf58488b 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml @@ -1,7 +1,7 @@ --- version: ${xpack.apmdata.template.version} _meta: - description: Fallback to ILM policy for traces-apm.sampled if DSL not defined + description: Fallback to ILM policy for traces-apm.sampled if data stream lifecycle not defined managed: true template: settings: From a586807ab485b1b5132070a98dc715abcf43d6a2 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Fri, 13 Sep 2024 14:54:56 +0100 Subject: [PATCH 10/14] Make shouldLoadTemplate overridable --- .../apmdata/APMIndexTemplateRegistry.java | 10 ++++++++++ .../core/template/YamlTemplateRegistry.java | 18 +++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 6f5d4e13dc56b..620cbd6b4c020 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; import static org.elasticsearch.xpack.apmdata.APMPlugin.APM_DATA_REGISTRY_ENABLED; /** @@ -58,4 +59,13 @@ protected String getVersionProperty() { protected String getOrigin() { return ClientHelper.APM_ORIGIN; } + + @Override + protected boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { + // Load ILM files only when the server supports ILM i.e. dsl-only-mode is false + if (dslOnlyMode && name.endsWith("@ilm")) { + return false; + } + return true; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index 620d4e1a62729..a1cc76ddac27b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -37,8 +37,6 @@ /** * Creates index templates and ingest pipelines based on YAML files defined in resources.yaml. - * It also allows ILM files to be managed separately as per the DSL only mode for the cluster. - * The ILM names MUST end with `@ilm` for this class to mark the resource as ILM resource. */ public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(YamlTemplateRegistry.class); @@ -113,7 +111,7 @@ public int getVersion() { /*** * - * @return A friendly, human readable name of the index template regisry + * @return A friendly, human-readable name of the index template registry */ public abstract String getName(); @@ -171,10 +169,16 @@ public List getIngestPipelines() { protected abstract String getVersionProperty(); - private boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { - if (dslOnlyMode && name.endsWith("@ilm")) { - return false; - } + /** + * shouldLoadTemplate provides a way to filter out templates conditionally as required by the plugin. + * One common use of this method could be to filter out ILM based templates when data stream only mode + * is enabled which is passed as a parameter to the shouldLoadTemplate function. + * + * @param name of the resource currently being processed + * @param dslOnlyMode represents if server is running in data stream lifecycle only mode + * @return boolean indicating if the resource should be loaded by the plugin or not. + */ + protected boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { return true; } From f123211bfd64804e7005d3696f82ad919f6267b3 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Fri, 13 Sep 2024 15:01:37 +0100 Subject: [PATCH 11/14] Remove unused import --- .../elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 620cbd6b4c020..fc144e67dc7e5 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; import static org.elasticsearch.xpack.apmdata.APMPlugin.APM_DATA_REGISTRY_ENABLED; /** From 53602e5867d88191070e5223e9c26167ec156a52 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Mon, 16 Sep 2024 10:12:57 +0100 Subject: [PATCH 12/14] Use predicate and refactor YamlTemplateRegistry interface --- .../apmdata/APMIndexTemplateRegistry.java | 21 ++++++++---- .../core/template/YamlTemplateRegistry.java | 34 +++++++++---------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index fc144e67dc7e5..466c9e4f006dc 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -16,6 +16,9 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.template.YamlTemplateRegistry; +import java.util.function.Predicate; + +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; import static org.elasticsearch.xpack.apmdata.APMPlugin.APM_DATA_REGISTRY_ENABLED; /** @@ -33,7 +36,15 @@ public APMIndexTemplateRegistry( NamedXContentRegistry xContentRegistry, FeatureService featureService ) { - super(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService); + super( + nodeSettings, + clusterService, + threadPool, + client, + xContentRegistry, + featureService, + templateFilter(isDataStreamsLifecycleOnlyMode(clusterService.getSettings())) + ); } @Override @@ -59,12 +70,8 @@ protected String getOrigin() { return ClientHelper.APM_ORIGIN; } - @Override - protected boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { + private static Predicate templateFilter(boolean dslOnlyMode) { // Load ILM files only when the server supports ILM i.e. dsl-only-mode is false - if (dslOnlyMode && name.endsWith("@ilm")) { - return false; - } - return true; + return templateName -> dslOnlyMode == false || templateName.endsWith("@ilm") == false; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java index a1cc76ddac27b..183251f39a029 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java @@ -29,9 +29,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadResource; import static org.elasticsearch.xpack.core.template.ResourceUtils.loadVersionedResourceUTF8; @@ -51,7 +51,6 @@ public abstract class YamlTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean enabled; - @SuppressWarnings({ "unchecked", "this-escape" }) public YamlTemplateRegistry( Settings nodeSettings, ClusterService clusterService, @@ -59,9 +58,21 @@ public YamlTemplateRegistry( Client client, NamedXContentRegistry xContentRegistry, FeatureService featureService + ) { + this(nodeSettings, clusterService, threadPool, client, xContentRegistry, featureService, ignored -> true); + } + + @SuppressWarnings({ "unchecked", "this-escape" }) + public YamlTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + FeatureService featureService, + Predicate templateFilter ) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry); - final boolean dslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); try { final Map resources = XContentHelper.convertToMap( YamlXContent.yamlXContent, @@ -78,13 +89,13 @@ public YamlTemplateRegistry( .orElse(Collections.emptyList()) .stream() .map(o -> (String) o) - .filter(tpl -> shouldLoadTemplate(tpl, dslOnlyMode)) + .filter(templateFilter) .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); composableIndexTemplates = Optional.ofNullable(indexTemplateNames) .orElse(Collections.emptyList()) .stream() .map(o -> (String) o) - .filter(tpl -> shouldLoadTemplate(tpl, dslOnlyMode)) + .filter(templateFilter) .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); ingestPipelines = Optional.ofNullable(ingestPipelineConfigs) .orElse(Collections.emptyList()) @@ -169,19 +180,6 @@ public List getIngestPipelines() { protected abstract String getVersionProperty(); - /** - * shouldLoadTemplate provides a way to filter out templates conditionally as required by the plugin. - * One common use of this method could be to filter out ILM based templates when data stream only mode - * is enabled which is passed as a parameter to the shouldLoadTemplate function. - * - * @param name of the resource currently being processed - * @param dslOnlyMode represents if server is running in data stream lifecycle only mode - * @return boolean indicating if the resource should be loaded by the plugin or not. - */ - protected boolean shouldLoadTemplate(String name, boolean dslOnlyMode) { - return true; - } - private ComponentTemplate loadComponentTemplate(String name, int version) { try { final byte[] content = loadVersionedResourceUTF8( From 08395c6d7d35e99e11ee2ec906d605d7f2c01d4f Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Mon, 16 Sep 2024 12:30:00 +0100 Subject: [PATCH 13/14] Refactor tests --- .../test/ClusterServiceUtils.java | 14 ++++++++----- .../xpack/apmdata/APMDSLOnlyTests.java | 21 +++++++------------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index 6d892cbe65c6f..b80978a0bc676 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -121,21 +121,25 @@ public static ClusterService createClusterService(ThreadPool threadPool, Cluster } public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode, ClusterSettings clusterSettings) { - Settings settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").build(); - return createClusterService(threadPool, localNode, settings, clusterSettings); + return createClusterService(threadPool, localNode, Settings.EMPTY, clusterSettings); } public static ClusterService createClusterService( ThreadPool threadPool, DiscoveryNode localNode, - Settings nodeSettings, + Settings providedSettings, ClusterSettings clusterSettings ) { + Settings settings = Settings.builder() + .put("node.name", "test") + .put("cluster.name", "ClusterServiceTests") + .put(providedSettings) + .build(); ClusterService clusterService = new ClusterService( - nodeSettings, + settings, clusterSettings, threadPool, - new TaskManager(nodeSettings, threadPool, Collections.emptySet(), Tracer.NOOP) + new TaskManager(providedSettings, threadPool, Collections.emptySet(), Tracer.NOOP) ); clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService()); ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName())) diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java index 5c775116dc329..476b504339e62 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMDSLOnlyTests.java @@ -21,8 +21,6 @@ import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.stack.StackTemplateRegistry; -import org.elasticsearch.xpack.stack.StackTemplateRegistryAccessor; import org.junit.After; import org.junit.Before; @@ -37,14 +35,13 @@ public class APMDSLOnlyTests extends ESTestCase { private APMIndexTemplateRegistry apmIndexTemplateRegistry; - private StackTemplateRegistryAccessor stackTemplateRegistryAccessor; private ThreadPool threadPool; private VerifyingClient client; @Before public void createRegistryAndClient() { final ClusterSettings clusterSettings = new ClusterSettings( - Settings.builder().put(DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, "true").build(), + Settings.EMPTY, Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Set.of(APMPlugin.APM_DATA_REGISTRY_ENABLED).stream()) .collect(Collectors.toSet()) ); @@ -52,16 +49,14 @@ public void createRegistryAndClient() { threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("node", "node"); - Settings nodeSettings = Settings.builder() - .put("node.name", "test") - .put("cluster.name", "ClusterServiceTests") - .put(DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, true) - .build(); - ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, nodeSettings, clusterSettings); - FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures())); - stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor( - new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService) + Settings additionalSettings = Settings.builder().put(DataStreamLifecycle.DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME, true).build(); + ClusterService clusterService = ClusterServiceUtils.createClusterService( + threadPool, + discoveryNode, + additionalSettings, + clusterSettings ); + FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures())); apmIndexTemplateRegistry = new APMIndexTemplateRegistry( Settings.EMPTY, From 38c3fae7a92ee56fe6be3e5eaadc4c37ac1ce4b3 Mon Sep 17 00:00:00 2001 From: Vishal Raj Date: Mon, 16 Sep 2024 12:32:38 +0100 Subject: [PATCH 14/14] Fix incorrect usage of settings --- .../main/java/org/elasticsearch/test/ClusterServiceUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index b80978a0bc676..7fb480952356e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -139,7 +139,7 @@ public static ClusterService createClusterService( settings, clusterSettings, threadPool, - new TaskManager(providedSettings, threadPool, Collections.emptySet(), Tracer.NOOP) + new TaskManager(settings, threadPool, Collections.emptySet(), Tracer.NOOP) ); clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService()); ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName()))