Merge branch '8.x' into backport/8.x/pr-114810

pmpailis · Oct 16, 2024 · 8729755 · 8729755
2 parents a17ca52 + 6de48b1
commit 8729755
Show file tree

Hide file tree

Showing 29 changed files with 352 additions and 216 deletions.
diff --git a/docs/changelog/114856.yaml b/docs/changelog/114856.yaml
@@ -0,0 +1,5 @@
+pr: 114856
+summary: "OTel mappings: avoid metrics to be rejected when attributes are malformed"
+area: Data streams
+type: bug
+issues: []
diff --git a/...ms/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/...ms/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
@@ -132,9 +132,7 @@ public void setup() throws Exception {
         // Initialize the failure store.
         RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null);
         rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions())
-                .failureStoreOptions(b -> b.includeRegularIndices(false).includeFailureIndices(true))
-                .build()
+            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build()
         );
         response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get();
         assertTrue(response.isAcknowledged());

diff --git a/...c/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/...c/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
@@ -198,9 +198,7 @@ public void testRejectionFromFailureStore() throws IOException {
         // Initialize failure store.
         var rolloverRequest = new RolloverRequest(dataStream, null);
         rolloverRequest.setIndicesOptions(
-            IndicesOptions.builder(rolloverRequest.indicesOptions())
-                .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
-                .build()
+            IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build()
         );
         var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
         var failureStoreIndex = rolloverResponse.getNewIndex();

diff --git a/...ams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/...ams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
@@ -946,7 +946,7 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
                 UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest();
                 updateMergePolicySettingsRequest.indicesOptions(
                     IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(true, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
                         .build()
                 );
                 updateMergePolicySettingsRequest.indices(indexName);
@@ -1409,7 +1409,7 @@ static RolloverRequest getDefaultRolloverRequest(
         if (rolloverFailureStore) {
             rolloverRequest.setIndicesOptions(
                 IndicesOptions.builder(rolloverRequest.indicesOptions())
-                    .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false))
+                    .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
                     .build()
             );
         }

diff --git a/...ta-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/...ta-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java
@@ -43,7 +43,7 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
                 IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
                 "verbose"
             ),
-            DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FailureStoreOptions.FAILURE_STORE) : Set.of()
+            DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FAILURE_STORE_QUERY_PARAM) : Set.of()
         )
     );
 

diff --git a/...rc/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/...rc/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
@@ -225,17 +225,11 @@ public void testOperationsExecutedOnce() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(true, false))
-        );
+        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
         assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(false, true))
-        );
+        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES));
         List<DeleteIndexRequest> deleteRequests = clientSeenRequests.subList(2, 5)
             .stream()
             .map(transportRequest -> (DeleteIndexRequest) transportRequest)
@@ -1549,17 +1543,11 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
         assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
         assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverBackingIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(true, false))
-        );
+        assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA));
         assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
         RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
         assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
-        assertThat(
-            rolloverFailureIndexRequest.indicesOptions().failureStoreOptions(),
-            equalTo(new IndicesOptions.FailureStoreOptions(false, true))
-        );
+        assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES));
         assertThat(
             ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
             is(dataStream.getFailureIndices().getIndices().get(0).getName())

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -384,6 +384,12 @@ tests:
 - class: org.elasticsearch.xpack.enrich.EnrichRestIT
   method: test {p0=enrich/20_standard_index/enrich documents over _bulk via an alias}
   issue: https://github.com/elastic/elasticsearch/issues/114763
+- class: org.elasticsearch.xpack.eql.EqlRestValidationIT
+  method: testAllowNoIndicesOption
+  issue: https://github.com/elastic/elasticsearch/issues/114789
+- class: org.elasticsearch.xpack.enrich.EnrichRestIT
+  method: test {p0=enrich/20_standard_index/enrich documents over _bulk}
+  issue: https://github.com/elastic/elasticsearch/issues/114768
 
 # Examples:
 #

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -245,6 +245,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion QUERY_RULE_TEST_API = def(8_769_00_0);
     public static final TransportVersion ESQL_PER_AGGREGATE_FILTER = def(8_770_00_0);
     public static final TransportVersion ML_INFERENCE_ATTACH_TO_EXISTSING_DEPLOYMENT = def(8_771_00_0);
+    public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
@@ -98,9 +98,7 @@ public GetIndexRequest() {
         super(
             DataStream.isFailureStoreFeatureFlagEnabled()
                 ? IndicesOptions.builder(IndicesOptions.strictExpandOpen())
-                    .failureStoreOptions(
-                        IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true)
-                    )
+                    .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
                     .build()
                 : IndicesOptions.strictExpandOpen()
         );

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -147,8 +147,8 @@ public ActionRequestValidationException validate() {
             );
         }
 
-        var failureStoreOptions = indicesOptions.failureStoreOptions();
-        if (failureStoreOptions.includeRegularIndices() && failureStoreOptions.includeFailureIndices()) {
+        var selectors = indicesOptions.selectorOptions().defaultSelectors();
+        if (selectors.size() > 1) {
             validationException = addValidationError(
                 "rollover cannot be applied to both regular and failure indices at the same time",
                 validationException
@@ -188,7 +188,7 @@ public IndicesOptions indicesOptions() {
      * @return true of the rollover request targets the failure store, false otherwise.
      */
     public boolean targetsFailureStore() {
-        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.failureStoreOptions().includeFailureIndices();
+        return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices();
     }
 
     public void setIndicesOptions(IndicesOptions indicesOptions) {

diff --git a/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -150,7 +150,7 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
                 .matchClosed(request.indicesOptions().expandWildcardsClosed())
                 .build(),
             IndicesOptions.GatekeeperOptions.DEFAULT,
-            request.indicesOptions().failureStoreOptions()
+            request.indicesOptions().selectorOptions()
         );
 
         return state.blocks()
@@ -247,7 +247,7 @@ protected void masterOperation(
             IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS,
             IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(),
             IndicesOptions.GatekeeperOptions.DEFAULT,
-            rolloverRequest.indicesOptions().failureStoreOptions()
+            rolloverRequest.indicesOptions().selectorOptions()
         );
         IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(rolloverRequest.getRolloverTarget())
             .clear()

diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
@@ -212,7 +212,7 @@ private void rollOverFailureStores(Runnable runnable) {
                 RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null);
                 rolloverRequest.setIndicesOptions(
                     IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
                         .build()
                 );
                 // We are executing a lazy rollover because it is an action specialised for this situation, when we want an

diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -418,7 +418,7 @@ private void rollOverDataStreams(
             if (targetFailureStore) {
                 rolloverRequest.setIndicesOptions(
                     IndicesOptions.builder(rolloverRequest.indicesOptions())
-                        .failureStoreOptions(new IndicesOptions.FailureStoreOptions(false, true))
+                        .selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
                         .build()
                 );
             }

diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
@@ -61,9 +61,7 @@ public Request() {
                             .allowFailureIndices(true)
                             .build()
                     )
-                    .failureStoreOptions(
-                        IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build()
-                    )
+                    .selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
                     .build()
             );
         }

diff --git a/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java b/server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.support;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * We define as index components the two different sets of indices a data stream could consist of:
+ * - DATA: represents the backing indices
+ * - FAILURES: represent the failing indices
+ * Note: An index is its own DATA component, but it cannot have a FAILURE component.
+ */
+public enum IndexComponentSelector {
+    DATA("data"),
+    FAILURES("failures");
+
+    private final String key;
+
+    IndexComponentSelector(String key) {
+        this.key = key;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    private static final Map<String, IndexComponentSelector> REGISTRY;
+
+    static {
+        Map<String, IndexComponentSelector> registry = new HashMap<>(IndexComponentSelector.values().length);
+        for (IndexComponentSelector value : IndexComponentSelector.values()) {
+            registry.put(value.getKey(), value);
+        }
+        REGISTRY = Collections.unmodifiableMap(registry);
+    }
+
+    public static IndexComponentSelector getByKey(String key) {
+        return REGISTRY.get(key);
+    }
+}