Merge branch '8.x' into backport/8.x/pr-120223

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -180,8 +180,8 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
             throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
         }
         // We instrument classes in these modules to call the bridge. Because the bridge gets patched
-        // into java.base, we must export the bridge from java.base to these modules.
-        String modulesContainingEntitlementInstrumentation = "java.logging";
+        // into java.base, we must export the bridge from java.base to these modules, as a comma-separated list
+        String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming";
         return Stream.of(
             "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",

libs/entitlement/bridge/src/main/java/module-info.java

@@ -10,5 +10,7 @@
 // This module-info is used just to satisfy your IDE.
 // At build and run time, the bridge is patched into the java.base module.
 module org.elasticsearch.entitlement.bridge {
+    requires java.net.http;
+
     exports org.elasticsearch.entitlement.bridge;
 }

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -30,6 +30,10 @@
 import java.net.URL;
 import java.net.URLStreamHandler;
 import java.net.URLStreamHandlerFactory;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.security.cert.CertStoreParameters;
 import java.util.List;
 
 import javax.net.ssl.HostnameVerifier;
@@ -254,4 +258,37 @@ public interface EntitlementChecker {
     void check$java_net_Socket$connect(Class<?> callerClass, Socket that, SocketAddress endpoint);
 
     void check$java_net_Socket$connect(Class<?> callerClass, Socket that, SocketAddress endpoint, int backlog);
+
+    // Network miscellanea
+    void check$java_net_URL$openConnection(Class<?> callerClass, java.net.URL that, Proxy proxy);
+
+    // HttpClient.Builder is an interface, so we instrument its only (internal) implementation
+    void check$jdk_internal_net_http_HttpClientBuilderImpl$build(Class<?> callerClass, HttpClient.Builder that);
+
+    // HttpClient#send and sendAsync are abstract, so we instrument their internal implementation
+    void check$jdk_internal_net_http_HttpClientImpl$send(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest request,
+        HttpResponse.BodyHandler<?> responseBodyHandler
+    );
+
+    void check$jdk_internal_net_http_HttpClientImpl$sendAsync(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest userRequest,
+        HttpResponse.BodyHandler<?> responseHandler
+    );
+
+    void check$jdk_internal_net_http_HttpClientImpl$sendAsync(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest userRequest,
+        HttpResponse.BodyHandler<?> responseHandler,
+        HttpResponse.PushPromiseHandler<?> pushPromiseHandler
+    );
+
+    // We need to check the LDAPCertStore, as this will connect, but this is internal/created via SPI,
+    // so we instrument the general factory instead and then filter in the check method implementation
+    void check$java_security_cert_CertStore$$getInstance(Class<?> callerClass, String type, CertStoreParameters params);
 }

libs/entitlement/qa/common/src/main/java/module-info.java

@@ -14,6 +14,7 @@
 
     // Modules we'll attempt to use in order to exercise entitlements
     requires java.logging;
+    requires java.net.http;
 
     exports org.elasticsearch.entitlement.qa.common;
 }

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/NetworkAccessCheckActions.java

@@ -17,6 +17,12 @@
 import java.net.Proxy;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertStore;
+import java.util.Arrays;
 
 class NetworkAccessCheckActions {
 
@@ -59,4 +65,21 @@ static void socketConnect() throws IOException {
             socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
         }
     }
+
+    @SuppressForbidden(reason = "Testing entitlement check on forbidden action")
+    static void urlOpenConnectionWithProxy() throws URISyntaxException, IOException {
+        var url = new URI("http://localhost").toURL();
+        var urlConnection = url.openConnection(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(0)));
+        assert urlConnection != null;
+    }
+
+    static void createLDAPCertStore() throws NoSuchAlgorithmException {
+        try {
+            // We pass down null params to provoke a InvalidAlgorithmParameterException
+            CertStore.getInstance("LDAP", null);
+        } catch (InvalidAlgorithmParameterException ex) {
+            // Assert we actually hit the class we care about, LDAPCertStore (or its impl)
+            assert Arrays.stream(ex.getStackTrace()).anyMatch(e -> e.getClassName().endsWith("LDAPCertStore"));
+        }
+    }
 }

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java

@@ -157,7 +157,13 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
         entry("socket_bind", forPlugins(NetworkAccessCheckActions::socketBind)),
         entry("socket_connect", forPlugins(NetworkAccessCheckActions::socketConnect)),
         entry("server_socket_bind", forPlugins(NetworkAccessCheckActions::serverSocketBind)),
-        entry("server_socket_accept", forPlugins(NetworkAccessCheckActions::serverSocketAccept))
+        entry("server_socket_accept", forPlugins(NetworkAccessCheckActions::serverSocketAccept)),
+
+        entry("url_open_connection_proxy", forPlugins(NetworkAccessCheckActions::urlOpenConnectionWithProxy)),
+        entry("http_client_builder_build", forPlugins(VersionSpecificNetworkChecks::httpClientBuilderBuild)),
+        entry("http_client_send", forPlugins(VersionSpecificNetworkChecks::httpClientSend)),
+        entry("http_client_send_async", forPlugins(VersionSpecificNetworkChecks::httpClientSendAsync)),
+        entry("create_ldap_cert_store", forPlugins(NetworkAccessCheckActions::createLDAPCertStore))
     )
         .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion())
         .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/VersionSpecificNetworkChecks.java

@@ -9,6 +9,30 @@
 
 package org.elasticsearch.entitlement.qa.common;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+
 class VersionSpecificNetworkChecks {
     static void createInetAddressResolverProvider() {}
+
+    static void httpClientBuilderBuild() {
+        HttpClient.newBuilder().build();
+    }
+
+    static void httpClientSend() throws InterruptedException {
+        HttpClient httpClient = HttpClient.newBuilder().build();
+        try {
+            httpClient.send(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+        } catch (IOException e) {
+            // Expected, the send action may fail with these parameters (but after it run the entitlement check in the prologue)
+        }
+    }
+
+    static void httpClientSendAsync() {
+        HttpClient httpClient = HttpClient.newBuilder().build();
+        httpClient.sendAsync(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+    }
 }

libs/entitlement/qa/common/src/main18/java/org/elasticsearch/entitlement/qa/common/VersionSpecificNetworkChecks.java

@@ -9,6 +9,11 @@
 
 package org.elasticsearch.entitlement.qa.common;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
 import java.net.spi.InetAddressResolver;
 import java.net.spi.InetAddressResolverProvider;
 
@@ -26,4 +31,22 @@ public String name() {
             }
         };
     }
+
+    static void httpClientBuilderBuild() {
+        HttpClient.newBuilder().build();
+    }
+
+    static void httpClientSend() throws InterruptedException {
+        HttpClient httpClient = HttpClient.newBuilder().build();
+        try {
+            httpClient.send(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+        } catch (IOException e) {
+            // Expected, the send action may fail with these parameters (but after it run the entitlement check in the prologue)
+        }
+    }
+
+    static void httpClientSendAsync() {
+        HttpClient httpClient = HttpClient.newBuilder().build();
+        httpClient.sendAsync(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+    }
 }

libs/entitlement/qa/common/src/main21/java/org/elasticsearch/entitlement/qa/common/VersionSpecificNetworkChecks.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.common;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.net.spi.InetAddressResolver;
+import java.net.spi.InetAddressResolverProvider;
+
+class VersionSpecificNetworkChecks {
+    static void createInetAddressResolverProvider() {
+        var x = new InetAddressResolverProvider() {
+            @Override
+            public InetAddressResolver get(Configuration configuration) {
+                return null;
+            }
+
+            @Override
+            public String name() {
+                return "TEST";
+            }
+        };
+    }
+
+    static void httpClientBuilderBuild() {
+        try (HttpClient httpClient = HttpClient.newBuilder().build()) {
+            assert httpClient != null;
+        }
+    }
+
+    static void httpClientSend() throws InterruptedException {
+        try (HttpClient httpClient = HttpClient.newBuilder().build()) {
+            // Shutdown the client, so the send action will shortcut before actually executing any network operation
+            // (but after it run our check in the prologue)
+            httpClient.shutdown();
+            try {
+                httpClient.send(HttpRequest.newBuilder(URI.create("http://localhost")).build(), HttpResponse.BodyHandlers.discarding());
+            } catch (IOException e) {
+                // Expected, since we shut down the client
+            }
+        }
+    }
+
+    static void httpClientSendAsync() {
+        try (HttpClient httpClient = HttpClient.newBuilder().build()) {
+            // Shutdown the client, so the send action will return before actually executing any network operation
+            // (but after it run our check in the prologue)
+            httpClient.shutdown();
+            var future = httpClient.sendAsync(
+                HttpRequest.newBuilder(URI.create("http://localhost")).build(),
+                HttpResponse.BodyHandlers.discarding()
+            );
+            assert future.isCompletedExceptionally();
+            future.exceptionally(ex -> {
+                assert ex instanceof IOException;
+                return null;
+            });
+        }
+    }
+}

libs/entitlement/src/main/java/module-info.java

@@ -13,6 +13,7 @@
     requires java.instrument;
     requires org.elasticsearch.base;
     requires jdk.attach;
+    requires java.net.http;
 
     requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -34,6 +34,10 @@
 import java.net.URL;
 import java.net.URLStreamHandler;
 import java.net.URLStreamHandlerFactory;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.security.cert.CertStoreParameters;
 import java.util.List;
 
 import javax.net.ssl.HostnameVerifier;
@@ -504,4 +508,57 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     public void check$java_net_Socket$connect(Class<?> callerClass, Socket that, SocketAddress endpoint, int backlog) {
         policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
     }
+
+    @Override
+    public void check$java_net_URL$openConnection(Class<?> callerClass, URL that, Proxy proxy) {
+        if (proxy.type() != Proxy.Type.DIRECT) {
+            policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
+        }
+    }
+
+    @Override
+    public void check$jdk_internal_net_http_HttpClientBuilderImpl$build(Class<?> callerClass, HttpClient.Builder that) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.LISTEN_ACTION);
+    }
+
+    @Override
+    public void check$jdk_internal_net_http_HttpClientImpl$send(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest request,
+        HttpResponse.BodyHandler<?> responseBodyHandler
+    ) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
+    }
+
+    @Override
+    public void check$jdk_internal_net_http_HttpClientImpl$sendAsync(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest userRequest,
+        HttpResponse.BodyHandler<?> responseHandler
+    ) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
+    }
+
+    @Override
+    public void check$jdk_internal_net_http_HttpClientImpl$sendAsync(
+        Class<?> callerClass,
+        HttpClient that,
+        HttpRequest userRequest,
+        HttpResponse.BodyHandler<?> responseHandler,
+        HttpResponse.PushPromiseHandler<?> pushPromiseHandler
+    ) {
+        policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
+    }
+
+    @Override
+    public void check$java_security_cert_CertStore$$getInstance(Class<?> callerClass, String type, CertStoreParameters params) {
+        // We need to check "just" the LDAPCertStore instantiation: this is the CertStore that will try to perform a network operation
+        // (connect to an LDAP server). But LDAPCertStore is internal (created via SPI), so we instrument the general factory instead and
+        // then do the check only for the path that leads to sensitive code (by looking at the `type` parameter).
+        if ("LDAP".equals(type)) {
+            policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION);
+        }
+    }
 }

muted-tests.yml

@@ -379,9 +379,6 @@ tests:
 - class: org.elasticsearch.discovery.ec2.DiscoveryEc2AvailabilityZoneAttributeNoImdsIT
   method: testAvailabilityZoneAttribute
   issue: https://github.com/elastic/elasticsearch/issues/118564
-- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
-  method: testRetryPointInTime
-  issue: https://github.com/elastic/elasticsearch/issues/118514
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/20_metrics_ingest/Test metrics-apm.app-* setting event.ingested via ingest pipeline}
   issue: https://github.com/elastic/elasticsearch/issues/118875

x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamTransportActionIT.java

@@ -28,7 +28,6 @@
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.migrate.MigratePlugin;
 import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamRequest;
-import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.ReindexDataStreamResponse;
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamEnrichedStatus;
 import org.elasticsearch.xpack.migrate.task.ReindexDataStreamTask;
 
@@ -61,8 +60,7 @@ public void testNonExistentDataStream() {
         );
         assertThrows(
             ResourceNotFoundException.class,
-            () -> client().execute(new ActionType<ReindexDataStreamResponse>(ReindexDataStreamAction.NAME), reindexDataStreamRequest)
-                .actionGet()
+            () -> client().execute(new ActionType<AcknowledgedResponse>(ReindexDataStreamAction.NAME), reindexDataStreamRequest).actionGet()
         );
     }
 
@@ -74,12 +72,11 @@ public void testAlreadyUpToDateDataStream() throws Exception {
             dataStreamName
         );
         final int backingIndexCount = createDataStream(dataStreamName);
-        ReindexDataStreamResponse response = client().execute(
-            new ActionType<ReindexDataStreamResponse>(ReindexDataStreamAction.NAME),
+        AcknowledgedResponse response = client().execute(
+            new ActionType<AcknowledgedResponse>(ReindexDataStreamAction.NAME),
             reindexDataStreamRequest
         ).actionGet();
-        String persistentTaskId = response.getTaskId();
-        assertThat(persistentTaskId, equalTo("reindex-data-stream-" + dataStreamName));
+        String persistentTaskId = "reindex-data-stream-" + dataStreamName;
         AtomicReference<ReindexDataStreamTask> runningTask = new AtomicReference<>();
         for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
             TaskManager taskManager = transportService.getTaskManager();