Merge remote-tracking branch 'upstream/main' into search_pit_resolved…

…_indices

docs/build.gradle

@@ -111,6 +111,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+  requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 
   extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json")
   extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")

docs/changelog/102078.yaml

@@ -0,0 +1,5 @@
+pr: 102078
+summary: Derive expected replica size from primary
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/103035.yaml

@@ -0,0 +1,5 @@
+pr: 103035
+summary: "x-pack/plugin/core: add `match_mapping_type` to `ecs@mappings` dynamic templates"
+area: Data streams
+type: bug
+issues: []

docs/changelog/103176.yaml

@@ -0,0 +1,5 @@
+pr: 103176
+summary: Validate settings in `ReloadSecureSettings` API
+area: Client
+type: bug
+issues: []

docs/changelog/103251.yaml

@@ -0,0 +1,5 @@
+pr: 103251
+summary: Wait for reroute before acking put-shutdown
+area: Infra/Node Lifecycle
+type: bug
+issues: []

docs/reference/rest-api/common-parms.asciidoc

@@ -611,9 +611,10 @@ The `similarity` parameter is the direct vector similarity calculation.
 
 * `l2_norm`: also known as Euclidean, will include documents where the vector is within
 the `dims` dimensional hypersphere with radius `similarity` with origin at `query_vector`.
-* `cosine` & `dot_product`: Only return vectors where the cosine similarity or dot-product are at least the provided
+* `cosine`, `dot_product`, and `max_inner_product`: Only return vectors where the cosine similarity or dot-product are at least the provided
 `similarity`.
 --
+Read more here: <<knn-similarity-search, knn similarity search>>
 end::knn-similarity[]
 
 tag::lenient[]

docs/reference/search/search-your-data/knn-search.asciidoc

@@ -547,6 +547,7 @@ score = 0.9 * match_score + 0.1 * knn_score_image-vector + 0.5 * knn_score_title
 ```
 
 [discrete]
+[[knn-similarity-search]]
 ==== Search kNN with expected similarity
 
 While kNN is a powerful tool, it always tries to return `k` nearest neighbors. Consequently, when using `knn` with
@@ -563,6 +564,18 @@ minimum similarity for a vector to be considered a match. The `knn` search flow
 * Do not return any vectors that are further away than the configured `similarity`
 --
 
+NOTE: `similarity` is the true <<dense-vector-similarity, similarity>> before it has been transformed into `_score` and boost applied.
+
+For each configured <<dense-vector-similarity, similarity>>, here is the corresponding inverted `_score` function. This is so if you are wanting to filter from a `_score` perspective, you can do this minor transformation to correctly reject irrelevant results.
+--
+ - `l2_norm`: `sqrt((1 / _score) - 1)`
+ - `cosine`: `(2 * _score) - 1`
+ - `dot_product`: `(2 * _score) - 1`
+ - `max_inner_product`:
+    - `_score < 1`: `1 - (1 / _score)`
+    - `_score >= 1`: `_score - 1`
+--
+
 Here is an example. In this example we search for the given `query_vector` for `k` nearest neighbors. However, with
 `filter` applied and requiring that the found vectors have at least the provided `similarity` between them.
 [source,console]

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
@@ -25,6 +26,7 @@ public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
+        .feature(FeatureFlag.FAILURE_STORE_ENABLED)
         .setting("xpack.security.enabled", "false")
         .setting("xpack.watcher.enabled", "false")
         .build();

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecyclePermissionsRestIT.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.util.resource.Resource;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -38,6 +39,7 @@ public class DataStreamLifecyclePermissionsRestIT extends ESRestTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
+        .feature(FeatureFlag.FAILURE_STORE_ENABLED)
         .setting("xpack.watcher.enabled", "false")
         .setting("xpack.ml.enabled", "false")
         .setting("xpack.security.enabled", "true")

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -15,8 +15,8 @@
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ResultDeduplicator;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
@@ -719,7 +719,7 @@ private void deleteIndexOnce(String indexName, String reason) {
         transportActionsDeduplicator.executeOnce(
             deleteIndexRequest,
             new ErrorRecordingActionListener(
-                DeleteIndexAction.NAME,
+                TransportDeleteIndexAction.TYPE.name(),
                 indexName,
                 errorStore,
                 Strings.format("Data stream lifecycle encountered an error trying to delete index [%s]", indexName),

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java

@@ -26,9 +26,8 @@
  */
 public class DeleteDataStreamLifecycleAction {
 
-    public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>(
-        "indices:admin/data_stream/lifecycle/delete",
-        AcknowledgedResponse::readFrom
+    public static final ActionType<AcknowledgedResponse> INSTANCE = ActionType.acknowledgedResponse(
+        "indices:admin/data_stream/lifecycle/delete"
     );
 
     private DeleteDataStreamLifecycleAction() {/* no instances */}

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/PutDataStreamLifecycleAction.java

@@ -40,9 +40,8 @@
  */
 public class PutDataStreamLifecycleAction {
 
-    public static final ActionType<AcknowledgedResponse> INSTANCE = new ActionType<>(
-        "indices:admin/data_stream/lifecycle/put",
-        AcknowledgedResponse::readFrom
+    public static final ActionType<AcknowledgedResponse> INSTANCE = ActionType.acknowledgedResponse(
+        "indices:admin/data_stream/lifecycle/put"
     );
 
     private PutDataStreamLifecycleAction() {/* no instances */}

modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java

@@ -19,6 +19,8 @@
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
+import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
+
 public class DataStreamsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
     public DataStreamsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) {
@@ -43,6 +45,7 @@ protected Settings restClientSettings() {
     private static ElasticsearchCluster createCluster() {
         LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
+            .feature(FAILURE_STORE_ENABLED)
             .setting("xpack.security.enabled", "true")
             .keystore("bootstrap.password", "x-pack-test-password")
             .user("x_pack_rest_user", "x-pack-test-password");

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -210,10 +210,8 @@ setup:
 ---
 "Create data stream with failure store":
   - skip:
-      version: all
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/102873"
-#      version: " - 8.10.99"
-#      reason: "data stream failure stores only creatable in 8.11+"
+      version: " - 8.10.99"
+      reason: "data stream failure stores only creatable in 8.11+"
 
   - do:
       allowed_warnings:

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml

@@ -50,10 +50,8 @@
 ---
 "Put index template with failure store":
   - skip:
-      version: all
-      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/102873"
-#      version: " - 8.10.99"
-#      reason: "data stream failure stores only creatable in 8.11+"
+      version: " - 8.10.99"
+      reason: "data stream failure stores only creatable in 8.11+"
       features: allowed_warnings
 
   - do:

server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java

@@ -14,8 +14,8 @@
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.TransportShardFlushAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
@@ -502,14 +502,14 @@ public void testCloseIndex() {
     }
 
     public void testDeleteIndex() {
-        interceptTransportActions(DeleteIndexAction.NAME);
+        interceptTransportActions(TransportDeleteIndexAction.TYPE.name());
 
         String[] randomIndicesOrAliases = randomUniqueIndices();
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(randomIndicesOrAliases);
         assertAcked(internalCluster().coordOnlyNodeClient().admin().indices().delete(deleteIndexRequest).actionGet());
 
         clearInterceptedActions();
-        assertSameIndices(deleteIndexRequest, DeleteIndexAction.NAME);
+        assertSameIndices(deleteIndexRequest, TransportDeleteIndexAction.TYPE.name());
     }
 
     public void testGetMappings() {

server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java

@@ -11,10 +11,13 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
@@ -43,6 +46,8 @@
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class ReloadSecureSettingsIT extends ESIntegTestCase {
 
+    private static final String VALID_SECURE_SETTING_NAME = "some.setting.that.exists";
+
     @BeforeClass
     public static void disableInFips() {
         // Reload secure settings with a password protected keystore is tested in ReloadSecureSettingsWithPasswordProtectedKeystoreRestIT
@@ -350,9 +355,46 @@ public void testReloadWhileKeystoreChanged() throws Exception {
         }
     }
 
+    public void testInvalidKeyInSettings() throws Exception {
+        final Environment environment = internalCluster().getInstance(Environment.class);
+
+        try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
+            keyStoreWrapper.setString(VALID_SECURE_SETTING_NAME, new char[0]);
+            keyStoreWrapper.save(environment.configFile(), new char[0], false);
+        }
+
+        PlainActionFuture<NodesReloadSecureSettingsResponse> actionFuture = new PlainActionFuture<>();
+        clusterAdmin().prepareReloadSecureSettings()
+            .setSecureStorePassword(new SecureString(new char[0]))
+            .setNodesIds(Strings.EMPTY_ARRAY)
+            .execute(actionFuture);
+
+        actionFuture.get().getNodes().forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), nullValue()));
+
+        try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
+            assertThat(keyStoreWrapper, notNullValue());
+            keyStoreWrapper.setString("some.setting.that.does.not.exist", new char[0]);
+            keyStoreWrapper.save(environment.configFile(), new char[0], false);
+        }
+
+        actionFuture = new PlainActionFuture<>();
+        clusterAdmin().prepareReloadSecureSettings()
+            .setSecureStorePassword(new SecureString(new char[0]))
+            .setNodesIds(Strings.EMPTY_ARRAY)
+            .execute(actionFuture);
+
+        actionFuture.get()
+            .getNodes()
+            .forEach(nodeResponse -> assertThat(nodeResponse.reloadException(), instanceOf(IllegalArgumentException.class)));
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        final List<Class<? extends Plugin>> plugins = Arrays.asList(MockReloadablePlugin.class, MisbehavingReloadablePlugin.class);
+        final List<Class<? extends Plugin>> plugins = Arrays.asList(
+            MockWithSecureSettingPlugin.class,
+            MockReloadablePlugin.class,
+            MisbehavingReloadablePlugin.class
+        );
         // shuffle as reload is called in order
         Collections.shuffle(plugins, random());
         return plugins;
@@ -455,4 +497,10 @@ public synchronized void setShouldThrow(boolean shouldThrow) {
         }
     }
 
+    public static class MockWithSecureSettingPlugin extends Plugin {
+        public List<Setting<?>> getSettings() {
+            return List.of(SecureSetting.secureString(VALID_SECURE_SETTING_NAME, null));
+        }
+    };
+
 }

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetDesiredBalanceActionIT.java

@@ -40,7 +40,7 @@ public void testDesiredBalanceOnMultiNodeCluster() throws Exception {
         var clusterHealthResponse = clusterAdmin().health(new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)).get();
         assertEquals(RestStatus.OK, clusterHealthResponse.status());
 
-        DesiredBalanceResponse desiredBalanceResponse = client().execute(GetDesiredBalanceAction.INSTANCE, new DesiredBalanceRequest())
+        DesiredBalanceResponse desiredBalanceResponse = client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest())
             .get();
 
         assertEquals(1, desiredBalanceResponse.getRoutingTable().size());
@@ -75,7 +75,7 @@ public void testDesiredBalanceWithUnassignedShards() throws Exception {
         var clusterHealthResponse = clusterAdmin().health(new ClusterHealthRequest(index).waitForStatus(ClusterHealthStatus.YELLOW)).get();
         assertEquals(RestStatus.OK, clusterHealthResponse.status());
 
-        DesiredBalanceResponse desiredBalanceResponse = client().execute(GetDesiredBalanceAction.INSTANCE, new DesiredBalanceRequest())
+        DesiredBalanceResponse desiredBalanceResponse = client().execute(TransportGetDesiredBalanceAction.TYPE, new DesiredBalanceRequest())
             .get();
 
         assertEquals(1, desiredBalanceResponse.getRoutingTable().size());

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportDesiredNodesActionsIT.java

@@ -271,7 +271,9 @@ public void testDeleteDesiredNodesTasksAreBatchedCorrectly() throws Exception {
 
         final List<ActionFuture<ActionResponse.Empty>> deleteDesiredNodesFutures = new ArrayList<>(15);
         for (int i = 0; i < 15; i++) {
-            deleteDesiredNodesFutures.add(client().execute(DeleteDesiredNodesAction.INSTANCE, new DeleteDesiredNodesAction.Request()));
+            deleteDesiredNodesFutures.add(
+                client().execute(TransportDeleteDesiredNodesAction.TYPE, new TransportDeleteDesiredNodesAction.Request())
+            );
         }
 
         for (ActionFuture<ActionResponse.Empty> future : deleteDesiredNodesFutures) {
@@ -347,8 +349,8 @@ private UpdateDesiredNodesRequest randomDryRunUpdateDesiredNodesRequest(Version
     }
 
     private void deleteDesiredNodes() {
-        final DeleteDesiredNodesAction.Request request = new DeleteDesiredNodesAction.Request();
-        client().execute(DeleteDesiredNodesAction.INSTANCE, request).actionGet();
+        final TransportDeleteDesiredNodesAction.Request request = new TransportDeleteDesiredNodesAction.Request();
+        client().execute(TransportDeleteDesiredNodesAction.TYPE, request).actionGet();
     }
 
     private DesiredNodes getLatestDesiredNodes() {

server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java

@@ -142,10 +142,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(TestPlugin.class);
     }
 
-    private static final ActionType<ActionResponse.Empty> TEST_ACTION = new ActionType<>(
-        TestTransportAction.NAME,
-        in -> ActionResponse.Empty.INSTANCE
-    );
+    private static final ActionType<ActionResponse.Empty> TEST_ACTION = ActionType.emptyResponse(TestTransportAction.NAME);
 
     public static class TestPlugin extends Plugin implements ActionPlugin {
         volatile CyclicBarrier barrier;

server/src/internalClusterTest/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java

@@ -8,10 +8,10 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
+import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
+import org.elasticsearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
@@ -111,7 +111,7 @@ public void testTwoNodesNoMasterBlock() throws Exception {
         String masterNode = internalCluster().getMasterName();
         String otherNode = node1Name.equals(masterNode) ? node2Name : node1Name;
         logger.info("--> add voting config exclusion for non-master node, to be sure it's not elected");
-        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(otherNode)).get();
+        client().execute(TransportAddVotingConfigExclusionsAction.TYPE, new AddVotingConfigExclusionsRequest(otherNode)).get();
         logger.info("--> stop master node, no master block should appear");
         Settings masterDataPathSettings = internalCluster().dataPathSettings(masterNode);
         internalCluster().stopNode(masterNode);
@@ -156,12 +156,12 @@ public void testTwoNodesNoMasterBlock() throws Exception {
         logger.info("--> clearing voting config exclusions");
         ClearVotingConfigExclusionsRequest clearRequest = new ClearVotingConfigExclusionsRequest();
         clearRequest.setWaitForRemoval(false);
-        client().execute(ClearVotingConfigExclusionsAction.INSTANCE, clearRequest).get();
+        client().execute(TransportClearVotingConfigExclusionsAction.TYPE, clearRequest).get();
 
         masterNode = internalCluster().getMasterName();
         otherNode = node1Name.equals(masterNode) ? node2Name : node1Name;
         logger.info("--> add voting config exclusion for master node, to be sure it's not elected");
-        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(masterNode)).get();
+        client().execute(TransportAddVotingConfigExclusionsAction.TYPE, new AddVotingConfigExclusionsRequest(masterNode)).get();
         logger.info("--> stop non-master node, no master block should appear");
         Settings otherNodeDataPathSettings = internalCluster().dataPathSettings(otherNode);
         internalCluster().stopNode(otherNode);