Merge branch 'main' into add_pit_info_to_collapse_request

docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc

@@ -50,7 +50,7 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1>
 be used and ELSER creates sparse vectors. The `inference_id` is
 `my-elser-endpoint`.
 <2> The `elser` service is used in this example.
-<3> This setting enables and configures {ml-docs}/ml-nlp-elser.html#elser-adaptive-allocations[adaptive allocations].
+<3> This setting enables and configures adaptive allocations.
 Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process.
 
 [NOTE]

modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.action.datastreams.GetDataStreamAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.set.Sets;
@@ -36,12 +35,12 @@ public class RestGetDataStreamsAction extends BaseRestHandler {
             Set.of(
                 "name",
                 "include_defaults",
+                "timeout",
                 "master_timeout",
                 IndicesOptions.WildcardOptions.EXPAND_WILDCARDS,
                 IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE,
                 IndicesOptions.WildcardOptions.ALLOW_NO_INDICES,
                 IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED,
-                DataStream.isFailureStoreFeatureFlagEnabled() ? IndicesOptions.FailureStoreOptions.FAILURE_STORE : "name",
                 "verbose"
             )
         )

muted-tests.yml

@@ -345,8 +345,6 @@ tests:
 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT
   method: testPutE5Small_withPlatformAgnosticVariant
   issue: https://github.com/elastic/elasticsearch/issues/113983
-- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
-  issue: https://github.com/elastic/elasticsearch/issues/114013
 - class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT
   method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse}
   issue: https://github.com/elastic/elasticsearch/issues/114019

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -1629,6 +1629,27 @@ public static <T extends Enum<T>> Setting<T> enumSetting(
         return new Setting<>(key, defaultValue.toString(), e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), validator, properties);
     }
 
+    /**
+     * Creates a setting where the allowed values are defined as enum constants. All enum constants must be uppercase.
+     *
+     * @param <T>          the generics type parameter reflecting the actual type of the enum
+     * @param clazz        the enum class
+     * @param defaultValue a default value function that returns the default values string representation.
+     * @param key          the key for the setting
+     * @param validator    validator for this setting
+     * @param properties   properties for this setting like scope, filtering...
+     * @return the setting object
+     */
+    public static <T extends Enum<T>> Setting<T> enumSetting(
+        Class<T> clazz,
+        Function<Settings, String> defaultValue,
+        String key,
+        Validator<T> validator,
+        Property... properties
+    ) {
+        return new Setting<>(key, defaultValue, e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), validator, properties);
+    }
+
     /**
      * Creates a setting where the allowed values are defined as enum constants. All enum constants must be uppercase.
      *

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -14,6 +14,8 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.StringLiteralDeduplicator;
 import org.elasticsearch.features.NodeFeature;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.xcontent.ToXContentFragment;
@@ -83,11 +85,18 @@ public void toXContent(XContentBuilder builder) throws IOException {
     // Setting to SourceKeepMode.ALL is equivalent to disabling synthetic source, so this is not allowed.
     public static final Setting<SourceKeepMode> SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting(
         SourceKeepMode.class,
+        settings -> {
+            var indexMode = IndexSettings.MODE.get(settings);
+            if (indexMode == IndexMode.LOGSDB) {
+                return SourceKeepMode.ARRAYS.toString();
+            } else {
+                return SourceKeepMode.NONE.toString();
+            }
+        },
         "index.mapping.synthetic_source_keep",
-        SourceKeepMode.NONE,
         value -> {
             if (value == SourceKeepMode.ALL) {
-                throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value.toString() + "]");
+                throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value + "]");
             }
         },
         Setting.Property.IndexScope,

server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java

@@ -16,7 +16,6 @@
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.RestApiVersion;
@@ -105,8 +104,6 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl
         // prepare the request for execution; has the side effect of touching the request parameters
         try (var action = prepareRequest(request, client)) {
 
-            assert assertConsumesSupportedParams(supported, request);
-
             // validate unconsumed params, but we must exclude params used to format the response
             // use a sorted set so the unconsumed parameters appear in a reliable sorted order
             final SortedSet<String> unconsumedParams = request.unconsumedParams()
@@ -151,20 +148,6 @@ public void close() {
         }
     }
 
-    private boolean assertConsumesSupportedParams(@Nullable Set<String> supported, RestRequest request) {
-        if (supported != null) {
-            final var supportedAndCommon = new TreeSet<>(supported);
-            supportedAndCommon.add("error_trace");
-            supportedAndCommon.addAll(ALWAYS_SUPPORTED);
-            supportedAndCommon.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS);
-            final var consumed = new TreeSet<>(request.consumedParams());
-            consumed.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS);
-            assert supportedAndCommon.equals(consumed)
-                : getName() + ": consumed params " + consumed + " while supporting " + supportedAndCommon;
-        }
-        return true;
-    }
-
     protected static String unrecognized(RestRequest request, Set<String> invalids, Set<String> candidates, String detail) {
         StringBuilder message = new StringBuilder().append("request [")
             .append(request.path())

x-pack/plugin/logsdb/build.gradle

@@ -11,6 +11,7 @@ evaluationDependsOn(xpackModule('core'))
 
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
 esplugin {
   name 'logsdb'
@@ -30,3 +31,7 @@ dependencies {
 tasks.named("javaRestTest").configure {
   usesDefaultDistribution()
 }
+
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}

x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+
+public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.license.self_generated.type", "trial")
+        .build();
+
+    public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+}

x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/10_setting.yml

@@ -0,0 +1,39 @@
+---
+synthetic_source_keep defaults:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: PUT
+          path: /{index}
+          capabilities: [ logsdb_index_mode ]
+      reason: "Support for 'logsdb' index mode capability required"
+
+  - do:
+      indices.create:
+        index: test1
+        body:
+          settings:
+            index:
+              mode: logsdb
+
+  - do:
+      indices.create:
+        index: test2
+
+  - do:
+      indices.get_settings:
+        index: test1
+        include_defaults: true
+
+  - is_true: test1
+  - match: { test1.settings.index.mode: "logsdb" }
+  - match: { test1.defaults.index.mapping.synthetic_source_keep: "arrays" }
+
+  - do:
+      indices.get_settings:
+        index: test2
+        include_defaults: true
+
+  - is_true: test2
+  - is_false: test2.settings.index.mode
+  - match: { test2.defaults.index.mapping.synthetic_source_keep: "none" }

x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml

@@ -48,9 +48,9 @@ setup:
         body:
           fields: ["*"]
   - length: { hits.hits: 1 }
-  - match: { hits.hits.0.fields.resource\.attributes\.host\.ip: ["0.0.0.0", "127.0.0.1"] }
-  - match: { hits.hits.0.fields.attributes\.foo: [1, 2, 3] }
-  - match: { hits.hits.0.fields.attributes\.bar: [a, b, c] }
+  - match: { hits.hits.0.fields.resource\.attributes\.host\.ip: ["127.0.0.1", "0.0.0.0"] }
+  - match: { hits.hits.0.fields.attributes\.foo: [3, 2, 1] }
+  - match: { hits.hits.0.fields.attributes\.bar: [b, c, a] }
 ---
 "Exception aliases":
   - do: