Merge pull request #12 from michaelhyatt/distributed-http-tracing-3.0

Distributed http tracing 3.0

examples/tracing-test/.gitignore

@@ -0,0 +1,48 @@
+# ------------------------------------------------------------------------------ #
+# Java defaults (https://github.com/github/gitignore/blob/master/Java.gitignore) #
+# ------------------------------------------------------------------------------ #
+*.class
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# ------------------------------------------------------------------------------------------- #
+# Eclipse-specific (https://github.com/github/gitignore/blob/master/Global/Eclipse.gitignore) #
+# ------------------------------------------------------------------------------------------- #
+*.pydevproject
+.metadata
+bin/**
+tmp/**
+tmp/**/*
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.project
+.classpath
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# CDT-specific
+.cproject
+
+# PDT-specific
+.buildpath
+
+# --------------- #
+# Studio-specific #
+# --------------- #
+target/
+.mule/**
+.mule/**/*
+.DS_Store
+velocity.log

examples/tracing-test/README.md

@@ -0,0 +1,13 @@
+# Tracing test
+
+## Intro
+This Mule project runs as a Mule engine instrumented with the Mule APM agent and it serves a HTML page with Elastic RUM agent that both participate in a distributed trace. The RUM agent in the browser starts a transaction and sends a `traceparent` header containing the `trace.id` back to the Mule runtime that records its own transaction using the sent `trace.id` which results in a complete trace populated in Kibana APM UI:
+![kibana-apm](images/kibana.png)
+
+## How to run
+Start the Mule project with the following additional parameters to initialise the APM agent:
+```
+-M-Delastic.apm.server_urls=https://XXXXX.elastic-cloud.com -M-Delastic.apm.secret_token=YYYYYY -M-Delastic.apm.service_name=component1 -M-Delastic.apm.service_version=v1.0.0 -M-Delastic.apm.log_level=INFO
+```
+
+Open the browser at http://localhost:8081 and press the `Press me` button a couple of times.

examples/tracing-test/mule-artifact.json

@@ -0,0 +1,3 @@
+{
+  "minMuleVersion": "4.3.0"
+}

examples/tracing-test/pom.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>com.mycompany</groupId>
+	<artifactId>tracing-test</artifactId>
+	<version>1.0.0-SNAPSHOT</version>
+	<packaging>mule-application</packaging>
+
+	<name>tracing-test</name>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+		<app.runtime>4.3.0-20201013</app.runtime>
+		<mule.maven.plugin.version>3.3.5</mule.maven.plugin.version>
+	</properties>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-clean-plugin</artifactId>
+				<version>3.0.0</version>
+			</plugin>
+			<plugin>
+				<groupId>org.mule.tools.maven</groupId>
+				<artifactId>mule-maven-plugin</artifactId>
+				<version>${mule.maven.plugin.version}</version>
+				<extensions>true</extensions>
+				<configuration>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.mule.connectors</groupId>
+			<artifactId>mule-http-connector</artifactId>
+			<version>1.5.22</version>
+			<classifier>mule-plugin</classifier>
+		</dependency>
+		<dependency>
+			<groupId>org.mule.connectors</groupId>
+			<artifactId>mule-sockets-connector</artifactId>
+			<version>1.2.0</version>
+			<classifier>mule-plugin</classifier>
+		</dependency>
+		<dependency>
+			<groupId>org.mule.modules</groupId>
+			<artifactId>mule-scripting-module</artifactId>
+			<version>1.1.7</version>
+			<classifier>mule-plugin</classifier>
+		</dependency>
+		<dependency>
+			<groupId>co.elastic.apm</groupId>
+			<artifactId>mule4-agent</artifactId>
+			<version>0.4.0</version>
+		</dependency>
+	</dependencies>
+
+	<repositories>
+		<repository>
+			<id>anypoint-exchange-v2</id>
+			<name>Anypoint Exchange</name>
+			<url>https://maven.anypoint.mulesoft.com/api/v2/maven</url>
+			<layout>default</layout>
+		</repository>
+		<repository>
+			<id>mulesoft-releases</id>
+			<name>MuleSoft Releases Repository</name>
+			<url>https://repository.mulesoft.org/releases/</url>
+			<layout>default</layout>
+		</repository>
+	</repositories>
+	<pluginRepositories>
+		<pluginRepository>
+			<id>mulesoft-releases</id>
+			<name>mulesoft release repository</name>
+			<layout>default</layout>
+			<url>https://repository.mulesoft.org/releases/</url>
+			<snapshots>
+				<enabled>false</enabled>
+			</snapshots>
+		</pluginRepository>
+	</pluginRepositories>
+
+</project>

examples/tracing-test/src/main/resources/log4j2.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Configuration>
+
+    <!--These are some of the loggers you can enable. 
+        There are several more you can find in the documentation. 
+        Besides this log4j configuration, you can also use Java VM environment variables
+        to enable other logs like network (-Djavax.net.debug=ssl or all) and 
+        Garbage Collector (-XX:+PrintGC). These will be append to the console, so you will 
+        see them in the mule_ee.log file. -->
+
+    <Appenders>
+        <RollingFile name="file" fileName="${sys:mule.home}${sys:file.separator}logs${sys:file.separator}tracing-test.log" 
+                 filePattern="${sys:mule.home}${sys:file.separator}logs${sys:file.separator}tracing-test-%i.log">
+            <PatternLayout pattern="%-5p %d [%t] [processor: %X{processorPath}; event: %X{correlationId}] %c: %m%n" />
+            <SizeBasedTriggeringPolicy size="10 MB" />
+            <DefaultRolloverStrategy max="10"/>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+
+        <!-- Http Logger shows wire traffic on DEBUG. -->
+        <!--AsyncLogger name="org.mule.service.http.impl.service.HttpMessageLogger" level="DEBUG" /-->
+        <AsyncLogger name="org.mule.service.http" level="WARN"/>
+        <AsyncLogger name="org.mule.extension.http" level="WARN"/>
+
+		<!-- Mule logger -->        
+        <AsyncLogger name="org.mule.runtime.core.internal.processor.LoggerMessageProcessor" level="INFO"/>
+
+        <AsyncRoot level="INFO">
+            <AppenderRef ref="file" />
+        </AsyncRoot>
+    </Loggers>
+</Configuration>

examples/tracing-test/src/test/resources/log4j2-test.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%-5p %d [%t] %c: %m%n"/>
+        </Console>
+    </Appenders>
+
+    <Loggers>
+
+        <!-- Http Logger shows wire traffic on DEBUG. -->
+        <!--AsyncLogger name="org.mule.service.http.impl.service.HttpMessageLogger" level="DEBUG" /-->
+        <AsyncLogger name="org.mule.service.http" level="WARN"/>
+        <AsyncLogger name="org.mule.extension.http" level="WARN"/>
+
+         <!-- Reduce startup noise -->         
+        <AsyncLogger name="com.mulesoft.mule.runtime.plugin" level="WARN"/>               
+        <AsyncLogger name="org.mule.maven.client" level="WARN"/>
+        <AsyncLogger name="org.mule.runtime.core.internal.util" level="WARN"/>
+        <AsyncLogger name="org.quartz" level="WARN"/>
+        <AsyncLogger name="org.mule.munit.plugins.coverage.server" level="WARN"/>        
+
+        <!-- Mule logger -->        
+        <AsyncLogger name="org.mule.runtime.core.internal.processor.LoggerMessageProcessor" level="INFO"/>  
+
+        <AsyncRoot level="INFO">
+            <AppenderRef ref="Console"/>
+        </AsyncRoot>
+    </Loggers>
+
+</Configuration>

mule4-agent/pom.xml

@@ -4,7 +4,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>co.elastic.apm</groupId>
 	<artifactId>mule4-agent</artifactId>
-	<version>0.3.0</version>
+	<version>0.4.0</version>
 	<name>Elastic Mule 4 APM agent</name>
 	<description>Elastic Mule 4 APM agent</description>
 	<packaging>jar</packaging>

mule4-agent/src/main/java/co/elastic/apm/mule4/agent/tracing/HttpTracingUtils.java

@@ -13,7 +13,8 @@ public class HttpTracingUtils {
 
 	private static final String HTTP_REQUEST_NAMESPACE = "http://www.mulesoft.org/schema/mule/http";
 	private static final String HTTP_REQUEST_NAME = "request";
-	public static final String ELASTIC_APM_TRACEPARENT = "elastic-apm-traceparent";
+	public static final String ELASTIC_APM_TRACEPARENT_HEADER = "elastic-apm-traceparent";
+	public static final String TRACEPARENT_HEADER = "traceparent";
 
 	public static boolean isHttpEvent(PipelineMessageNotification notification) {
 		return extractAttributes(notification).getDataType().getType() == HttpRequestAttributes.class;
@@ -24,17 +25,22 @@ public static boolean hasRemoteParent(PipelineMessageNotification notification)
 		if (!isHttpEvent(notification))
 			return false;
 
-		return getHttpAttributes(notification).containsKey(ELASTIC_APM_TRACEPARENT);
+		return getHttpAttributes(notification).containsKey(ELASTIC_APM_TRACEPARENT_HEADER)
+				|| getHttpAttributes(notification).containsKey(TRACEPARENT_HEADER);
 	}
 
 	public static String getTracingHeaderValue(String x, PipelineMessageNotification notification) {
-		return getHttpAttributes(notification).get(ELASTIC_APM_TRACEPARENT);
+
+		if (getHttpAttributes(notification).containsKey(ELASTIC_APM_TRACEPARENT_HEADER))
+			return getHttpAttributes(notification).get(ELASTIC_APM_TRACEPARENT_HEADER);
+		else
+			return getHttpAttributes(notification).get(TRACEPARENT_HEADER);
 	}
 
 	private static MultiMap<String, String> getHttpAttributes(PipelineMessageNotification notification) {
-		
+
 		HttpRequestAttributes attributes = (HttpRequestAttributes) extractAttributes(notification).getValue();
-		
+
 		return attributes.getHeaders();
 	}
 
@@ -43,17 +49,17 @@ private static TypedValue<Object> extractAttributes(PipelineMessageNotification
 	}
 
 	public static boolean isHttpRequester(MessageProcessorNotification notification) {
-		
+
 		ComponentIdentifier identifier = notification.getInfo().getComponent().getIdentifier();
 		String name = identifier.getName();
 		String namespace = identifier.getNamespaceUri();
-		
+
 		return HTTP_REQUEST_NAME.equals(name) && HTTP_REQUEST_NAMESPACE.equals(namespace);
 	}
 
 	public static void propagateTraceIdHeader(Span span, MessageProcessorNotification notification) {
 		// TODO Auto-generated method stub
-		
+
 	}
 
 }

mule4-agent/src/test/java/co/elastic/apm/mule4/agent/DistributedTracingTest.java

@@ -24,7 +24,7 @@ public class DistributedTracingTest extends BaseAbstractApmMuleTestCase {
 	public void testTraceIdPropagation() throws Exception {
 
 		HttpGet getRequest = new HttpGet("http://localhost:8998/request");
-		getRequest.addHeader(HttpTracingUtils.ELASTIC_APM_TRACEPARENT, TRACE_PARENT1);
+		getRequest.addHeader(HttpTracingUtils.ELASTIC_APM_TRACEPARENT_HEADER, TRACE_PARENT1);
 
 		HttpClient httpClient = HttpClientBuilder.create().build();
 
@@ -63,14 +63,21 @@ public void testNoTraceIdPropagation() throws Exception {
 	@Test
 	public void testTraceIdPropagationThroughHttp() throws Exception {
 
-		HttpGet getRequest = new HttpGet("http://localhost:8998");
-		getRequest.addHeader(HttpTracingUtils.ELASTIC_APM_TRACEPARENT, TRACE_PARENT1);
+		HttpGet getRequest = new HttpGet("http://localhost:8998/traceparentrequest");
+		getRequest.addHeader(HttpTracingUtils.TRACEPARENT_HEADER, TRACE_PARENT1);
 
 		HttpClient httpClient = HttpClientBuilder.create().build();
 
 		HttpResponse response = httpClient.execute(getRequest);
+
+		Thread.sleep(1000);
 
-
+		assertEquals(200, response.getStatusLine().getStatusCode());
+		assertEquals(TRACE_PARENT1, EntityUtils.toString(response.getEntity()));
+		List<Transaction> transactions = getTransactions();
+		assertEquals(1, transactions.size());
+		assertEquals(1, getSpans().size());
+		assertEquals(TRACE_ID1, getTransaction().getTraceContext().getTraceId().toString());
 	}
 
 	@Override

mule4-agent/src/test/resources/DistributedTracingTest.xml

@@ -21,6 +21,12 @@ http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/htt
 			<scripting:code ><![CDATA[result = attributes.headers['elastic-apm-traceparent']]]></scripting:code>
 		</scripting:execute>
 	</flow>
+	<flow name="testRequestFlowz" doc:id="de80bbbe-140a-4e18-905d-021bfeabeb80" >
+		<http:listener doc:name="Listener" doc:id="7184f118-3882-4c7d-8422-4e58094ce5bf" config-ref="HTTP_Listener_config" path="/traceparentrequest"/>
+		<scripting:execute engine="groovy" doc:name="Execute" doc:id="3b6a7ee7-2c85-407b-81e7-b83d9653a87d" >
+			<scripting:code ><![CDATA[result = attributes.headers['traceparent']]]></scripting:code>
+		</scripting:execute>
+	</flow>
 	<flow name="testFlow" doc:id="aacb4a95-ad0f-4a74-b1ee-995aa4048a75" >
 		<http:listener doc:name="Listener" doc:id="aa4582d9-f2bd-4c0f-b0ca-7db7410f58fe" config-ref="HTTP_Listener_config" path="/"/>
 		<logger level="INFO" doc:name="Logger" doc:id="e3fc1e1c-1247-4323-98f3-c6a9274f0b91" message="1a"/>