Skip to content

Commit

Permalink
Updates to latest Zipkin supporting Elasticsearch 8.x
Browse files Browse the repository at this point in the history
This drops support of Elasticsearch 6.x, including writing to old index
templates, as it is no longer possible with the driver that supports 7.x
and 8.x.

See elastic/elasticsearch-hadoop#2188

This also updates to the latest Spark 3.x version possible, currently
3.3.x. Note: we can't update to Scala 2.13 due to a conflict between
connectors: (DataStax) Cassandra requires Spark 3.4 on Scala 2.13, but
Elastic requires Spark 3.3.

See elastic/elasticsearch-hadoop#2187

Signed-off-by: Adrian Cole <[email protected]>
  • Loading branch information
Adrian Cole committed Dec 30, 2023
1 parent 7b24605 commit ad20993
Show file tree
Hide file tree
Showing 13 changed files with 70 additions and 84 deletions.
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ are supported, including Cassandra, MySQL and Elasticsearch.

## Versions

* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 3.11
* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 5.6
* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 5+; tested against last minor release of 6.x and 7.x
* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 4.0
* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 10.11
* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+; tested against last minor release of 7.x and 8.x

## Quick-start

Due to [SPARK-26134](https://issues.apache.org/jira/browse/SPARK-26134), Zipkin Dependencies currently requires Java 1.8 or 9 to run.
Zipkin Dependencies requires Java 11+ to run.

The quickest way to get started is to fetch the [latest released job](https://search.maven.org/remote_content?g=io.zipkin.dependencies&a=zipkin-dependencies&v=LATEST) as a self-contained jar. For example:

Expand Down Expand Up @@ -144,7 +144,7 @@ You can use the following command to display what this project is built against:
```bash
$ SPARK_VERSION=$(./mvnw help:evaluate -Dexpression=spark.version -q -DforceStdout)
$ echo $SPARK_VERSION
2.4.0
3.3.4
```

Once you've verified your setup is on the correct version, set the `SPARK_MASTER` variable:
Expand Down
2 changes: 1 addition & 1 deletion cassandra3/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>io.zipkin.dependencies</groupId>
<artifactId>zipkin-dependencies-parent</artifactId>
<version>2.7.4-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
</parent>

<artifactId>zipkin-dependencies-cassandra3</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ static boolean poolInFlight(CqlSession session) {
// mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
static final class CassandraContainer extends GenericContainer<CassandraContainer> {
CassandraContainer() {
super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.25.2"));
super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.26.0"));
if ("true".equals(System.getProperty("docker.skip"))) {
throw new TestAbortedException("${docker.skip} == true");
}
Expand Down
8 changes: 5 additions & 3 deletions elasticsearch/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@
the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>io.zipkin.dependencies</groupId>
<artifactId>zipkin-dependencies-parent</artifactId>
<version>2.7.4-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
</parent>

<artifactId>zipkin-dependencies-elasticsearch</artifactId>
Expand All @@ -35,7 +37,7 @@
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-30_${scala.binary.version}</artifactId>
<version>7.13.4</version>
<version>${elasticsearch-spark.version}</version>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,12 +168,7 @@ private static String getSystemPropertyAsFileResource(String key) {
}

public void run() {
run( // single-type index
index + ":span-" + dateStamp + "/span",
index + ":dependency-" + dateStamp + "/dependency",
SpanBytesDecoder.JSON_V2);

run( // single-type index with ES 7+
run(
index + "-span-" + dateStamp,
index + "-dependency-" + dateStamp,
SpanBytesDecoder.JSON_V2);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2016-2020 The OpenZipkin Authors
* Copyright 2016-2023 The OpenZipkin Authors
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
Expand All @@ -14,6 +14,7 @@
package zipkin2.dependencies.elasticsearch;

import java.io.IOException;
import java.util.Base64;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import org.elasticsearch.hadoop.EsHadoopException;
Expand All @@ -24,7 +25,6 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static okhttp3.mockwebserver.SocketPolicy.DISCONNECT_AT_START;
import static okhttp3.tls.internal.TlsUtil.localhost;
import static org.apache.commons.net.util.Base64.encodeBase64String;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

Expand Down Expand Up @@ -65,8 +65,9 @@ class ElasticsearchDependenciesJobTest {
assertThatThrownBy(job::run)
.isInstanceOf(EsHadoopException.class);

String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8));
assertThat(es.takeRequest().getHeader("Authorization"))
.isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes(UTF_8)).trim());
.isEqualTo("Basic " + encoded.trim());
}

@Test void authWorksWithSsl() throws Exception {
Expand All @@ -88,8 +89,9 @@ class ElasticsearchDependenciesJobTest {
assertThatThrownBy(job::run)
.isInstanceOf(EsHadoopException.class);

String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8));
assertThat(es.takeRequest().getHeader("Authorization"))
.isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes("UTF-8")).trim());
.isEqualTo("Basic " + encoded.trim());
}

@Test void parseHosts_default() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ String hostPort() {
// mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
static final class ElasticsearchContainer extends GenericContainer<ElasticsearchContainer> {
ElasticsearchContainer(int majorVersion) {
super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.25.2"));
super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.26.0"));
if ("true".equals(System.getProperty("docker.skip"))) {
throw new TestAbortedException("${docker.skip} == true");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2016-2020 The OpenZipkin Authors
* Copyright 2016-2023 The OpenZipkin Authors
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
Expand All @@ -17,8 +17,8 @@
import org.junit.jupiter.api.extension.RegisterExtension;

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class ITElasticsearchV6Dependencies extends ITElasticsearchDependencies {
@RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(6);
class ITElasticsearchV8Dependencies extends ITElasticsearchDependencies {
@RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(8);

@Override ElasticsearchExtension elasticsearch() {
return elasticsearch;
Expand Down
2 changes: 1 addition & 1 deletion main/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<parent>
<groupId>io.zipkin.dependencies</groupId>
<artifactId>zipkin-dependencies-parent</artifactId>
<version>2.7.4-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
</parent>

<artifactId>zipkin-dependencies</artifactId>
Expand Down
8 changes: 1 addition & 7 deletions mysql/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,14 @@
<parent>
<groupId>io.zipkin.dependencies</groupId>
<artifactId>zipkin-dependencies-parent</artifactId>
<version>2.7.4-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
</parent>

<artifactId>zipkin-dependencies-mysql</artifactId>
<name>Zipkin Dependencies: MySQL</name>

<properties>
<main.basedir>${project.basedir}/..</main.basedir>
<!-- MySQL connector is GPL, even if it has an OSS exception.
https://www.mysql.com/about/legal/licensing/foss-exception/
MariaDB has a friendlier license, LGPL, which is less scary in audits.
-->
<mariadb-java-client.version>2.7.11</mariadb-java-client.version>
</properties>

<dependencies>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2016-2020 The OpenZipkin Authors
* Copyright 2016-2023 The OpenZipkin Authors
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
Expand Down Expand Up @@ -159,7 +159,8 @@ public MySQLDependenciesJob build() {
this.url = new StringBuilder("jdbc:mysql://")
.append(builder.host).append(":").append(builder.port)
.append("/").append(builder.db)
.append("?autoReconnect=true")
.append("?permitMysqlScheme") // spark doesn't understand "mariadb"
.append("&autoReconnect=true")
.append("&useSSL=").append(builder.useSsl).toString();
this.user = builder.user;
this.password = builder.password;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,11 @@ MySQLStorage.Builder computeStorageBuilder() {
final MariaDbDataSource dataSource;

try {
dataSource = new MariaDbDataSource(host(), port(), "zipkin");
dataSource = new MariaDbDataSource(String.format(
"jdbc:mysql://%s:%s/zipkin?permitMysqlScheme&autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8",
host(), port()));
dataSource.setUser("zipkin");
dataSource.setPassword("zipkin");
dataSource.setProperties("autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8");
} catch (SQLException e) {
throw new AssertionError(e);
}
Expand All @@ -78,7 +79,7 @@ int port() {
// mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
static final class MySQLContainer extends GenericContainer<MySQLContainer> {
MySQLContainer() {
super(parse("ghcr.io/openzipkin/zipkin-mysql:2.25.2"));
super(parse("ghcr.io/openzipkin/zipkin-mysql:2.26.0"));
if ("true".equals(System.getProperty("docker.skip"))) {
throw new TestAbortedException("${docker.skip} == true");
}
Expand Down
85 changes: 38 additions & 47 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

<groupId>io.zipkin.dependencies</groupId>
<artifactId>zipkin-dependencies-parent</artifactId>
<version>2.7.4-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
<packaging>pom</packaging>

<modules>
Expand All @@ -36,35 +36,53 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

<main.basedir>${project.basedir}</main.basedir>
<main.java.version>1.8</main.java.version>
<main.signature.artifact>java18</main.signature.artifact>

<!-- All source is Java 11+ -->
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<!-- We don't use animal-sniffer anymore as release obviates it.
See https://github.com/mojohaus/animal-sniffer/issues/62 -->
<maven.compiler.release>11</maven.compiler.release>

<!-- override to set exclusions per-project -->
<errorprone.args />
<errorprone.version>2.23.0</errorprone.version>
<errorprone.version>2.24.0</errorprone.version>

<!-- TODO: can't update to 2.13 without updating ES driver -->
<!-- Use a common scala binary version, and the highest spark patch version.
Scala 2.13 is held back due to spark version conflicts until Elastic
updates their version. -->
<scala.binary.version>2.12</scala.binary.version>
<!-- TODO: can't update further without updating ES driver -->
<spark.version>3.0.3</spark.version>
<!-- Latest patch of highest common minor. 3.3, not 3.4 until ES updates.
See https://github.com/elastic/elasticsearch-hadoop/issues/2187 -->
<spark.version>3.3.4</spark.version>

<!-- This dependency is in elasticsearch-hadoop -->
<elasticsearch-spark.version>8.11.3</elasticsearch-spark.version>

<!-- Use latest Cassandra connector matching minor versioned of spark. -->
<spark-cassandra-connector.version>3.3.0</spark-cassandra-connector.version>
<!-- Note: Zipkin uses the Apache java-driver. Watch for the connector to
change from DataStax to this. -->
<java-driver.version>4.17.0</java-driver.version>

<spark-cassandra-connector.version>2.5.1</spark-cassandra-connector.version>
<java-driver.version>4.9.0</java-driver.version>
<!-- Use zipkin's version of MariaDB client -->
<mariadb-java-client.version>3.3.2</mariadb-java-client.version>

<junit-jupiter.version>5.10.1</junit-jupiter.version>
<testcontainers.version>1.19.3</testcontainers.version>

<!-- These need to match zipkin, so that tests don't have classpath
conflicts. When updating, also update IT*.java -->
<zipkin.version>2.25.2</zipkin.version>
<zipkin.version>2.26.0</zipkin.version>
<armeria.version>1.26.4</armeria.version>
<jackson.version>2.16.0</jackson.version>
<!-- Match Armeria version to avoid conflicts including running tests in the IDE -->
<netty.version>4.1.100.Final</netty.version>
<jackson.version>2.16.1</jackson.version>
<assertj.version>3.24.2</assertj.version>
<slf4j.version>1.7.36</slf4j.version>

<license.skip>${skipTests}</license.skip>

<animal-sniffer-maven-plugin.version>1.23</animal-sniffer-maven-plugin.version>
<go-offline-maven-plugin.version>1.2.8</go-offline-maven-plugin.version>
<!-- TODO: cleanup any redundant ignores now also in the 4.0 release (once final) -->
<license-maven-plugin.version>4.3</license-maven-plugin.version>
Expand Down Expand Up @@ -137,6 +155,14 @@

<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>${netty.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
Expand Down Expand Up @@ -213,13 +239,6 @@
<configuration>
<!-- Add dependencies indirectly referenced by build plugins -->
<dynamicDependencies>
<DynamicDependency>
<groupId>org.codehaus.mojo.signature</groupId>
<artifactId>${main.signature.artifact}</artifactId>
<version>1.0</version>
<type>signature</type>
<repositoryType>MAIN</repositoryType>
</DynamicDependency>
<DynamicDependency>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin-git</artifactId>
Expand Down Expand Up @@ -299,10 +318,7 @@
<version>${maven-compiler-plugin.version}</version>
<inherited>true</inherited>
<configuration>
<source>${main.java.version}</source>
<target>${main.java.version}</target>
<fork>true</fork>
<release>8</release>
<showWarnings>true</showWarnings>
</configuration>
</plugin>
Expand Down Expand Up @@ -335,29 +351,6 @@
</configuration>
</plugin>

<!-- Ensure no Java 9+ api usage -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>animal-sniffer-maven-plugin</artifactId>
<version>${animal-sniffer-maven-plugin.version}</version>
<configuration>
<checkTestClasses>false</checkTestClasses>
<skip>${skipTests}</skip>
<signature>
<groupId>org.codehaus.mojo.signature</groupId>
<artifactId>${main.signature.artifact}</artifactId>
<version>1.0</version>
</signature>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>${maven-enforcer-plugin.version}</version>
Expand Down Expand Up @@ -481,8 +474,6 @@
<version>${maven-compiler-plugin.version}</version>
<inherited>true</inherited>
<configuration>
<source>${main.java.version}</source>
<target>${main.java.version}</target>
<fork>true</fork>
<showWarnings>true</showWarnings>
</configuration>
Expand Down

0 comments on commit ad20993

Please sign in to comment.