Merge remote-tracking branch 'upstream/main' into inference_metadata_…

…fields

docs/changelog/117748.yaml

@@ -0,0 +1,6 @@
+pr: 117748
+summary: Add IMDSv2 support to `repository-s3`
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 105135

docs/reference/modules/discovery/voting.asciidoc

@@ -63,8 +63,7 @@ departed nodes from the voting configuration manually. Use the
 of resilience.
 
 No matter how it is configured, Elasticsearch will not suffer from a 
-"{wikipedia}/Split-brain_(computing)[split-brain]" inconsistency. 
-The `cluster.auto_shrink_voting_configuration`
+"split-brain" inconsistency. The `cluster.auto_shrink_voting_configuration`
 setting affects only its availability in the event of the failure of some of its
 nodes and the administrative tasks that must be performed as nodes join and
 leave the cluster.

docs/reference/snapshot-restore/repository-s3.asciidoc

@@ -38,7 +38,8 @@ PUT _snapshot/my_s3_repository
 The client that you use to connect to S3 has a number of settings available.
 The settings have the form `s3.client.CLIENT_NAME.SETTING_NAME`. By default,
 `s3` repositories use a client named `default`, but this can be modified using
-the <<repository-s3-repository,repository setting>> `client`. For example:
+the <<repository-s3-repository,repository setting>> `client`. For example, to
+use a client named `my-alternate-client`, register the repository as follows:
 
 [source,console]
 ----
@@ -69,10 +70,19 @@ bin/elasticsearch-keystore add s3.client.default.secret_key
 bin/elasticsearch-keystore add s3.client.default.session_token
 ----
 
-If instead you want to use the instance role or container role to access S3
-then you should leave these settings unset. You can switch from using specific
-credentials back to the default of using the instance role or container role by
-removing these settings from the keystore as follows:
+If you do not configure these settings then {es} will attempt to automatically
+obtain credentials from the environment in which it is running:
+
+* Nodes running on an instance in AWS EC2 will attempt to use the EC2 Instance
+  Metadata Service (IMDS) to obtain instance role credentials. {es} supports
+  both IMDS version 1 and IMDS version 2.
+
+* Nodes running in a container in AWS ECS and AWS EKS will attempt to obtain
+  container role credentials similarly.
+
+You can switch from using specific credentials back to the default of using the
+instance role or container role by removing these settings from the keystore as
+follows:
 
 [source,sh]
 ----
@@ -82,20 +92,14 @@ bin/elasticsearch-keystore remove s3.client.default.secret_key
 bin/elasticsearch-keystore remove s3.client.default.session_token
 ----
 
-*All* client secure settings of this repository type are
-{ref}/secure-settings.html#reloadable-secure-settings[reloadable].
-You can define these settings before the node is started,
-or call the <<cluster-nodes-reload-secure-settings,Nodes reload secure settings API>>
-after the settings are defined to apply them to a running node.
-
-After you reload the settings, the internal `s3` clients, used to transfer the snapshot
-contents, will utilize the latest settings from the keystore. Any existing `s3`
-repositories, as well as any newly created ones, will pick up the new values
-stored in the keystore.
-
-NOTE: In-progress snapshot/restore tasks will not be preempted by a *reload* of
-the client's secure settings. The task will complete using the client as it was
-built when the operation started.
+Define the relevant secure settings in each node's keystore before starting the
+node. The secure settings described here are all
+{ref}/secure-settings.html#reloadable-secure-settings[reloadable] so you may
+update the keystore contents on each node while the node is running and then
+call the <<cluster-nodes-reload-secure-settings,Nodes reload secure settings
+API>> to apply the updated settings to the nodes in the cluster. After this API
+completes, {es} will use the updated setting values for all future snapshot
+operations, but ongoing operations may continue to use older setting values.
 
 The following list contains the available client settings. Those that must be
 stored in the keystore are marked as "secure" and are *reloadable*; the other

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsVersion;
 import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
@@ -36,6 +37,7 @@ public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTe
     private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        Ec2ImdsVersion.V1,
         dynamicS3Credentials::addValidCredentials,
         Set.of("/ecs_credentials_endpoint")
     );

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsVersion;
 import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
 
@@ -36,6 +37,7 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res
     private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        Ec2ImdsVersion.V1,
         dynamicS3Credentials::addValidCredentials,
         Set.of()
     );

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsVersion;
+import fixture.s3.DynamicS3Credentials;
+import fixture.s3.S3HttpFixture;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.util.Set;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
+public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3RestTestCase {
+
+    private static final String PREFIX = getIdentifierPrefix("RepositoryS3ImdsV2CredentialsRestIT");
+    private static final String BUCKET = PREFIX + "bucket";
+    private static final String BASE_PATH = PREFIX + "base_path";
+    private static final String CLIENT = "imdsv2_credentials_client";
+
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        Ec2ImdsVersion.V2,
+        dynamicS3Credentials::addValidCredentials,
+        Set.of()
+    );
+
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
+        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String getBucketName() {
+        return BUCKET;
+    }
+
+    @Override
+    protected String getBasePath() {
+        return BASE_PATH;
+    }
+
+    @Override
+    protected String getClientName() {
+        return CLIENT;
+    }
+}

muted-tests.yml

@@ -234,6 +234,9 @@ tests:
 - class: org.elasticsearch.search.ccs.CrossClusterIT
   method: testCancel
   issue: https://github.com/elastic/elasticsearch/issues/108061
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
+  issue: https://github.com/elastic/elasticsearch/issues/117815
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/aliases/IndexAliasesIT.java

@@ -65,6 +65,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
@@ -262,27 +263,16 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
                 .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
         ).actionGet();
 
-        logger.info("--> checking single filtering alias search");
-        assertResponse(
+        assertResponses(
+            searchResponse -> assertHits(searchResponse.getHits(), "1"),
             prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1")
-        );
-
-        logger.info("--> checking single filtering alias wildcard search");
-        assertResponse(
-            prepareSearch("fo*").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1")
+            prepareSearch("fo*").setQuery(QueryBuilders.matchAllQuery())
         );
 
-        assertResponse(
+        assertResponses(
+            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3"),
             prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3")
-        );
-
-        logger.info("--> checking single filtering alias search with sort");
-        assertResponse(
-            prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).addSort("_index", SortOrder.ASC),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3")
+            prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).addSort("_index", SortOrder.ASC)
         );
 
         logger.info("--> checking single filtering alias search with global facets");
@@ -323,28 +313,12 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception {
             searchResponse -> assertHits(searchResponse.getHits(), "1", "2")
         );
 
-        logger.info("--> checking single non-filtering alias search");
-        assertResponse(
+        assertResponses(
+            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3", "4"),
             prepareSearch("alias1").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3", "4")
-        );
-
-        logger.info("--> checking non-filtering alias and filtering alias search");
-        assertResponse(
             prepareSearch("alias1", "foos").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3", "4")
-        );
-
-        logger.info("--> checking index and filtering alias search");
-        assertResponse(
             prepareSearch("test", "foos").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3", "4")
-        );
-
-        logger.info("--> checking index and alias wildcard search");
-        assertResponse(
-            prepareSearch("te*", "fo*").setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3", "4")
+            prepareSearch("te*", "fo*").setQuery(QueryBuilders.matchAllQuery())
         );
     }
 
@@ -506,11 +480,11 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
             prepareSearch("filter23", "filter13").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "21", "31", "13", "33")
         );
-        assertResponse(
+        assertResponses(
+            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(4L)),
             prepareSearch("filter23", "filter13").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(4L))
+            prepareSearch("filter13", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery())
         );
-
         assertResponse(
             prepareSearch("filter23", "filter1").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "21", "31", "11", "12", "13")
@@ -519,16 +493,10 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
             prepareSearch("filter23", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(5L))
         );
-
         assertResponse(
             prepareSearch("filter13", "filter1").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "11", "12", "13", "33")
         );
-        assertResponse(
-            prepareSearch("filter13", "filter1").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(4L))
-        );
-
         assertResponse(
             prepareSearch("filter13", "filter1", "filter23").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "11", "12", "13", "21", "31", "33")
@@ -537,7 +505,6 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
             prepareSearch("filter13", "filter1", "filter23").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(6L))
         );
-
         assertResponse(
             prepareSearch("filter23", "filter13", "test2").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "21", "22", "23", "31", "13", "33")
@@ -546,7 +513,6 @@ public void testSearchingFilteringAliasesMultipleIndices() throws Exception {
             prepareSearch("filter23", "filter13", "test2").setSize(0).setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(6L))
         );
-
         assertResponse(
             prepareSearch("filter23", "filter13", "test1", "test2").setQuery(QueryBuilders.matchAllQuery()),
             searchResponse -> assertHits(searchResponse.getHits(), "11", "12", "13", "21", "22", "23", "31", "33")
@@ -1325,17 +1291,13 @@ public void testIndexingAndQueryingHiddenAliases() throws Exception {
             searchResponse -> assertHits(searchResponse.getHits(), "2", "3")
         );
 
-        // Ensure that all docs can be gotten through the alias
-        assertResponse(
+        assertResponses(
+            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3"),
+            // Ensure that all docs can be gotten through the alias
             prepareSearch(alias).setQuery(QueryBuilders.matchAllQuery()),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3")
-        );
-
-        // And querying using a wildcard with indices options set to expand hidden
-        assertResponse(
+            // And querying using a wildcard with indices options set to expand hidden
             prepareSearch("alias*").setQuery(QueryBuilders.matchAllQuery())
-                .setIndicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, true, false, false)),
-            searchResponse -> assertHits(searchResponse.getHits(), "1", "2", "3")
+                .setIndicesOptions(IndicesOptions.fromOptions(false, false, true, false, true, true, true, false, false))
         );
 
         // And that querying the alias with a wildcard and no expand options fails