From 65e849f1b210aa44afcabc54e14908f32be931ae Mon Sep 17 00:00:00 2001 From: SylvainJuge <763082+SylvainJuge@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:42:09 +0100 Subject: [PATCH] Set context CL for reliable weblogic connection (#3870) * set context CL for weblogic --- CHANGELOG.asciidoc | 1 + .../report/AbstractIntakeApiHandler.java | 109 +++++++++--------- .../apm/agent/report/ApmServerClient.java | 12 ++ .../report/PartialTransactionReporter.java | 42 ++++--- .../apm/agent/util/UrlConnectionUtils.java | 46 ++++++++ .../agent/util/UrlConnectionUtilsTest.java | 80 +++++++++++++ docs/troubleshooting.asciidoc | 4 + 7 files changed, 223 insertions(+), 71 deletions(-) create mode 100644 apm-agent-core/src/test/java/co/elastic/apm/agent/util/UrlConnectionUtilsTest.java diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index efbc03a7cb..13b3bc07aa 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -34,6 +34,7 @@ Use subheadings with the "=====" level for adding notes for unreleased changes: [float] ===== Bug fixes * Prevent NPE in OpenTelemetry metrics bridge in case of asynchronous agent start - {pull}3880[#3880] +* Fix random Weblogic ClassNotFoundException related to thread context classloader - {pull}3870[#3870] [[release-notes-1.x]] === Java Agent version 1.x diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/AbstractIntakeApiHandler.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/AbstractIntakeApiHandler.java index 5fea031483..f10f8dc0d4 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/AbstractIntakeApiHandler.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/AbstractIntakeApiHandler.java @@ -22,6 +22,7 @@ import co.elastic.apm.agent.report.serialize.SerializationConstants; import co.elastic.apm.agent.sdk.logging.Logger; import co.elastic.apm.agent.sdk.logging.LoggerFactory; +import co.elastic.apm.agent.util.UrlConnectionUtils; import org.stagemonitor.util.IOUtils; import javax.annotation.Nullable; @@ -93,63 +94,64 @@ protected boolean shouldEndRequest() { protected HttpURLConnection startRequest(String endpoint) throws Exception { payloadSerializer.blockUntilReady(); final HttpURLConnection connection = apmServerClient.startRequest(endpoint); - if (connection != null) { + if (connection == null) { + return null; + } + try (UrlConnectionUtils.ContextClassloaderScope clScope = UrlConnectionUtils.withContextClassloaderOf(connection)){ + if (logger.isDebugEnabled()) { + logger.debug("Starting new request to {}", connection.getURL()); + } boolean useCompression = !isLocalhost(connection); + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setChunkedStreamingMode(SerializationConstants.BUFFER_SIZE); + if (useCompression) { + connection.setRequestProperty("Content-Encoding", "deflate"); + } + connection.setRequestProperty("Content-Type", "application/x-ndjson"); + connection.setUseCaches(false); + connection.connect(); + countingOs = new CountingOutputStream(connection.getOutputStream()); // TODO : here + if (useCompression) { + os = new DeflaterOutputStream(countingOs, deflater, true); + } else { + os = countingOs; + } + payloadSerializer.setOutputStream(os); + payloadSerializer.appendMetaDataNdJsonToStream(); + payloadSerializer.flushToOutputStream(); + requestStartedNanos = System.nanoTime(); + } catch (IOException e) { try { - if (logger.isDebugEnabled()) { - logger.debug("Starting new request to {}", connection.getURL()); - } - connection.setRequestMethod("POST"); - connection.setDoOutput(true); - connection.setChunkedStreamingMode(SerializationConstants.BUFFER_SIZE); - if (useCompression) { - connection.setRequestProperty("Content-Encoding", "deflate"); - } - connection.setRequestProperty("Content-Type", "application/x-ndjson"); - connection.setUseCaches(false); - connection.connect(); - countingOs = new CountingOutputStream(connection.getOutputStream()); - if (useCompression) { - os = new DeflaterOutputStream(countingOs, deflater, true); - } else { - os = countingOs; - } - payloadSerializer.setOutputStream(os); - payloadSerializer.appendMetaDataNdJsonToStream(); - payloadSerializer.flushToOutputStream(); - requestStartedNanos = System.nanoTime(); - } catch (IOException e) { - try { - logger.error("Error trying to connect to APM Server at {}. Although not necessarily related to SSL, some related SSL " + - "configurations corresponding the current connection are logged at INFO level.", connection.getURL()); - if (logger.isInfoEnabled() && connection instanceof HttpsURLConnection) { - HttpsURLConnection httpsURLConnection = (HttpsURLConnection) connection; - try { - logger.info("Cipher suite used for this connection: {}", httpsURLConnection.getCipherSuite()); - } catch (Exception e1) { - SSLSocketFactory sslSocketFactory = httpsURLConnection.getSSLSocketFactory(); - logger.info("Default cipher suites: {}", Arrays.toString(sslSocketFactory.getDefaultCipherSuites())); - logger.info("Supported cipher suites: {}", Arrays.toString(sslSocketFactory.getSupportedCipherSuites())); - } - try { - logger.info("APM Server certificates: {}", Arrays.toString(httpsURLConnection.getServerCertificates())); - } catch (Exception e1) { - // ignore - invalid - } - try { - logger.info("Local certificates: {}", Arrays.toString(httpsURLConnection.getLocalCertificates())); - } catch (Exception e1) { - // ignore - invalid - } + logger.error("Error trying to connect to APM Server at {}. Although not necessarily related to SSL, some related SSL " + + "configurations corresponding the current connection are logged at INFO level.", connection.getURL()); + if (logger.isInfoEnabled() && connection instanceof HttpsURLConnection) { + HttpsURLConnection httpsURLConnection = (HttpsURLConnection) connection; + try { + logger.info("Cipher suite used for this connection: {}", httpsURLConnection.getCipherSuite()); + } catch (Exception e1) { + SSLSocketFactory sslSocketFactory = httpsURLConnection.getSSLSocketFactory(); + logger.info("Default cipher suites: {}", Arrays.toString(sslSocketFactory.getDefaultCipherSuites())); + logger.info("Supported cipher suites: {}", Arrays.toString(sslSocketFactory.getSupportedCipherSuites())); + } + try { + logger.info("APM Server certificates: {}", Arrays.toString(httpsURLConnection.getServerCertificates())); + } catch (Exception e1) { + // ignore - invalid + } + try { + logger.info("Local certificates: {}", Arrays.toString(httpsURLConnection.getLocalCertificates())); + } catch (Exception e1) { + // ignore - invalid } - } finally { - closeAndSuppressErrors(connection); } - throw e; - } catch (Throwable t) { + } finally { closeAndSuppressErrors(connection); - throw t; } + throw e; + } catch (Throwable t) { + closeAndSuppressErrors(connection); + throw t; } return connection; } @@ -188,7 +190,10 @@ protected void endRequestExceptionally() { } private void endRequest(boolean isFailed) { - if (connection != null) { + if (connection == null) { + return; + } + try (UrlConnectionUtils.ContextClassloaderScope clScope = UrlConnectionUtils.withContextClassloaderOf(connection)) { long writtenBytes = countingOs != null ? countingOs.getCount() : 0L; try { payloadSerializer.fullFlush(); diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java index 9c9d198c0c..c2e8234fc5 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/ApmServerClient.java @@ -265,8 +265,10 @@ public V execute(String path, ConnectionHandler connectionHandler) throws Exception previousException = null; for (URL serverUrl : prioritizedUrlList) { HttpURLConnection connection = null; + UrlConnectionUtils.ContextClassloaderScope clScope = null; try { connection = startRequestToUrl(appendPath(serverUrl, path)); + clScope = UrlConnectionUtils.withContextClassloaderOf(connection); return connectionHandler.withConnection(connection); } catch (Exception e) { expectedErrorCount = incrementAndGetErrorCount(expectedErrorCount); @@ -277,6 +279,9 @@ public V execute(String path, ConnectionHandler connectionHandler) throws previousException = e; } finally { HttpUtils.consumeAndClose(connection); + if (clScope != null) { + clScope.close(); + } } } if (previousException == null) { @@ -290,13 +295,19 @@ public List executeForAllUrls(String path, ConnectionHandler connectio List results = new ArrayList<>(serverUrls.size()); for (URL serverUrl : serverUrls) { HttpURLConnection connection = null; + UrlConnectionUtils.ContextClassloaderScope clScope = null; try { connection = startRequestToUrl(appendPath(serverUrl, path)); + clScope = UrlConnectionUtils.withContextClassloaderOf(connection); results.add(connectionHandler.withConnection(connection)); } catch (Exception e) { logger.debug("Exception while interacting with APM Server", e); } finally { HttpUtils.consumeAndClose(connection); + if (clScope != null) { + clScope.close(); + } + } } return results; @@ -436,6 +447,7 @@ private static String getUserAgent(CoreConfigurationImpl coreConfiguration) { /** * Escapes the provided string from characters that are disallowed within HTTP header comments. * See spec- https://httpwg.org/specs/rfc7230.html#field.components + * * @param headerFieldComment HTTP header comment value to be escaped * @return the escaped header comment */ diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/PartialTransactionReporter.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/PartialTransactionReporter.java index e88e5d603b..fa2635b50c 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/report/PartialTransactionReporter.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/report/PartialTransactionReporter.java @@ -26,6 +26,7 @@ import co.elastic.apm.agent.sdk.logging.Logger; import co.elastic.apm.agent.sdk.logging.LoggerFactory; import co.elastic.apm.agent.tracer.pooling.Allocator; +import co.elastic.apm.agent.util.UrlConnectionUtils; import java.net.HttpURLConnection; @@ -67,27 +68,30 @@ public void reportPartialTransaction(TransactionImpl transaction) { logger.debug("Cannot report partial transaction because server url is not configured"); return; } - connection.setRequestMethod("POST"); - connection.setDoOutput(true); - connection.setChunkedStreamingMode(SerializationConstants.BUFFER_SIZE); - connection.setRequestProperty("Content-Type", "application/vnd.elastic.apm.transaction+ndjson"); - connection.setRequestProperty("x-elastic-aws-request-id", requestId); - connection.setUseCaches(false); - connection.connect(); - DslJsonSerializer.Writer writer = writerPool.createInstance(); - try { - writer.setOutputStream(connection.getOutputStream()); - writer.blockUntilReady(); //should actually not block on AWS Lambda, as metadata is available immediately - writer.appendMetaDataNdJsonToStream(); - writer.serializeTransactionNdJson(transaction); - writer.fullFlush(); - } finally { - writerPool.recycle(writer); - } + try (UrlConnectionUtils.ContextClassloaderScope clScope = UrlConnectionUtils.withContextClassloaderOf(connection)) { + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.setChunkedStreamingMode(SerializationConstants.BUFFER_SIZE); + connection.setRequestProperty("Content-Type", "application/vnd.elastic.apm.transaction+ndjson"); + connection.setRequestProperty("x-elastic-aws-request-id", requestId); + connection.setUseCaches(false); + connection.connect(); + + DslJsonSerializer.Writer writer = writerPool.createInstance(); + try { + writer.setOutputStream(connection.getOutputStream()); + writer.blockUntilReady(); //should actually not block on AWS Lambda, as metadata is available immediately + writer.appendMetaDataNdJsonToStream(); + writer.serializeTransactionNdJson(transaction); + writer.fullFlush(); + } finally { + writerPool.recycle(writer); + } - handleResponse(connection); - connection.disconnect(); + handleResponse(connection); + connection.disconnect(); + } } catch (Exception e) { logger.error("Failed to report partial transaction {}", transaction, e); diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/util/UrlConnectionUtils.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/util/UrlConnectionUtils.java index 631e1f18d1..67f6195b5d 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/util/UrlConnectionUtils.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/util/UrlConnectionUtils.java @@ -23,7 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.Proxy; import java.net.ProxySelector; import java.net.URISyntaxException; @@ -87,4 +89,48 @@ private static void debugPrintProxySettings(URL url, boolean allowProxy) { } } } + + public static ContextClassloaderScope withContextClassloaderOf(HttpURLConnection connection) { + Class type = connection.getClass(); + // so far, only weblogic is known to use context CL to load parts of the SSL/TLS communication + // see https://github.com/elastic/apm-agent-java/issues/2409 for background on why this is needed + boolean override = type.getCanonicalName().startsWith("weblogic."); + return withContextClassloader(type.getClassLoader(), override); + } + + // package private for testing + static ContextClassloaderScope withContextClassloader(@Nullable ClassLoader cl, boolean override) { + if (!override) { + return ContextClassloaderScope.NOOP; + } + return new ContextClassloaderScope(cl, false); + } + + public static class ContextClassloaderScope implements AutoCloseable { + + private static final ContextClassloaderScope NOOP = new ContextClassloaderScope(null, true); + + private final boolean noop; + @Nullable + private final ClassLoader previous; + + private ContextClassloaderScope(@Nullable ClassLoader classLoader, boolean noop) { + this.noop = noop; + if (noop) { + this.previous = null; + return; + } + this.previous = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(classLoader); + } + + @Override + public void close() { + if (noop) { + return; + } + Thread.currentThread().setContextClassLoader(previous); + + } + } } diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/util/UrlConnectionUtilsTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/util/UrlConnectionUtilsTest.java new file mode 100644 index 0000000000..3231165124 --- /dev/null +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/util/UrlConnectionUtilsTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.apm.agent.util; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import static org.assertj.core.api.Assertions.assertThat; + +class UrlConnectionUtilsTest { + + @BeforeEach + void beforeEach() { + Thread.currentThread().setContextClassLoader(null); + } + + @AfterEach + void afterEach() { + Thread.currentThread().setContextClassLoader(null); + } + + @Test + void threadContextClassLoader_noop() { + ClassLoader cl = UrlConnectionUtils.class.getClassLoader(); + testContextClassLoader(null, null, false); + testContextClassLoader(cl, cl, false); + } + + @Test + void threadContextClassLoader_override() { + ClassLoader cl = UrlConnectionUtils.class.getClassLoader(); + testContextClassLoader(null, cl, true); + testContextClassLoader(cl, cl, true); + + DummyClassLoader overrideCl = new DummyClassLoader(); + testContextClassLoader(null, overrideCl, true); + testContextClassLoader(cl, overrideCl, true); + } + + private void testContextClassLoader(@Nullable ClassLoader initialClassLoader, @Nullable ClassLoader scopeClassLoader, boolean override) { + Thread.currentThread().setContextClassLoader(initialClassLoader); + checkContextClassLoader(initialClassLoader); + try (UrlConnectionUtils.ContextClassloaderScope scope = UrlConnectionUtils.withContextClassloader(scopeClassLoader, override)) { + checkContextClassLoader(scopeClassLoader); + } + checkContextClassLoader(initialClassLoader); + } + + private void checkContextClassLoader(@Nullable ClassLoader expected) { + ClassLoader threadContextCl = Thread.currentThread().getContextClassLoader(); + if (expected == null) { + assertThat(threadContextCl).isNull(); + } else { + assertThat(threadContextCl).isSameAs(expected); + } + } + + private static class DummyClassLoader extends ClassLoader { + + } +} diff --git a/docs/troubleshooting.asciidoc b/docs/troubleshooting.asciidoc index fad18447c4..353b48809e 100644 --- a/docs/troubleshooting.asciidoc +++ b/docs/troubleshooting.asciidoc @@ -288,6 +288,10 @@ further information about how to download this file can be found <>. +With Oracle Weblogic application server wildcard TLS certificates are not allowed by default, when this +happens a message error like this is visible in agent logs: `Hostname verification failed: HostnameVerifier=weblogic.security.utils.SSLWLSHostnameVerifier`. +Disabling this extra check can be done with `-Dweblogic.security.SSL.ignoreHostnameVerification=true`. + For other SSL/TLS related problems, - check out https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#Troubleshooting[the JSSE troubleshooting section].