From ad209932c324912939eda72e570c3788429768ff Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 30 Dec 2023 09:10:53 +0800 Subject: [PATCH] Updates to latest Zipkin supporting Elasticsearch 8.x This drops support of Elasticsearch 6.x, including writing to old index templates, as it is no longer possible with the driver that supports 7.x and 8.x. See https://github.com/elastic/elasticsearch-hadoop/issues/2188 This also updates to the latest Spark 3.x version possible, currently 3.3.x. Note: we can't update to Scala 2.13 due to a conflict between connectors: (DataStax) Cassandra requires Spark 3.4 on Scala 2.13, but Elastic requires Spark 3.3. See https://github.com/elastic/elasticsearch-hadoop/issues/2187 Signed-off-by: Adrian Cole --- README.md | 10 +-- cassandra3/pom.xml | 2 +- .../storage/cassandra/CassandraExtension.java | 2 +- elasticsearch/pom.xml | 8 +- .../ElasticsearchDependenciesJob.java | 7 +- .../ElasticsearchDependenciesJobTest.java | 10 ++- .../elasticsearch/ElasticsearchExtension.java | 2 +- ...ava => ITElasticsearchV8Dependencies.java} | 6 +- main/pom.xml | 2 +- mysql/pom.xml | 8 +- .../mysql/MySQLDependenciesJob.java | 5 +- .../storage/mysql/v1/MySQLExtension.java | 7 +- pom.xml | 85 +++++++++---------- 13 files changed, 70 insertions(+), 84 deletions(-) rename elasticsearch/src/test/java/zipkin2/elasticsearch/{ITElasticsearchV6Dependencies.java => ITElasticsearchV8Dependencies.java} (86%) diff --git a/README.md b/README.md index 03f8e646..8ca24bb9 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ are supported, including Cassandra, MySQL and Elasticsearch. ## Versions -* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 3.11 -* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 5.6 -* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 5+; tested against last minor release of 6.x and 7.x +* `STORAGE_TYPE=cassandra3` : requires Cassandra 3.11.3+; tested against the latest patch of 4.0 +* `STORAGE_TYPE=mysql` : requires MySQL 5.6+; tested against MySQL 10.11 +* `STORAGE_TYPE=elasticsearch` : requires Elasticsearch 7+; tested against last minor release of 7.x and 8.x ## Quick-start -Due to [SPARK-26134](https://issues.apache.org/jira/browse/SPARK-26134), Zipkin Dependencies currently requires Java 1.8 or 9 to run. +Zipkin Dependencies requires Java 11+ to run. The quickest way to get started is to fetch the [latest released job](https://search.maven.org/remote_content?g=io.zipkin.dependencies&a=zipkin-dependencies&v=LATEST) as a self-contained jar. For example: @@ -144,7 +144,7 @@ You can use the following command to display what this project is built against: ```bash $ SPARK_VERSION=$(./mvnw help:evaluate -Dexpression=spark.version -q -DforceStdout) $ echo $SPARK_VERSION -2.4.0 +3.3.4 ``` Once you've verified your setup is on the correct version, set the `SPARK_MASTER` variable: diff --git a/cassandra3/pom.xml b/cassandra3/pom.xml index 6a2ef365..04fd1492 100644 --- a/cassandra3/pom.xml +++ b/cassandra3/pom.xml @@ -20,7 +20,7 @@ io.zipkin.dependencies zipkin-dependencies-parent - 2.7.4-SNAPSHOT + 3.0.0-SNAPSHOT zipkin-dependencies-cassandra3 diff --git a/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraExtension.java b/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraExtension.java index fe7b8022..d74469b0 100644 --- a/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraExtension.java +++ b/cassandra3/src/test/java/zipkin2/storage/cassandra/CassandraExtension.java @@ -169,7 +169,7 @@ static boolean poolInFlight(CqlSession session) { // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537 static final class CassandraContainer extends GenericContainer { CassandraContainer() { - super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.25.2")); + super(parse("ghcr.io/openzipkin/zipkin-cassandra:2.26.0")); if ("true".equals(System.getProperty("docker.skip"))) { throw new TestAbortedException("${docker.skip} == true"); } diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index 6902f379..9c88c7ab 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -14,13 +14,15 @@ the License. --> - + 4.0.0 io.zipkin.dependencies zipkin-dependencies-parent - 2.7.4-SNAPSHOT + 3.0.0-SNAPSHOT zipkin-dependencies-elasticsearch @@ -35,7 +37,7 @@ org.elasticsearch elasticsearch-spark-30_${scala.binary.version} - 7.13.4 + ${elasticsearch-spark.version} diff --git a/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java b/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java index f8156be1..10a9c18e 100644 --- a/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java +++ b/elasticsearch/src/main/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJob.java @@ -168,12 +168,7 @@ private static String getSystemPropertyAsFileResource(String key) { } public void run() { - run( // single-type index - index + ":span-" + dateStamp + "/span", - index + ":dependency-" + dateStamp + "/dependency", - SpanBytesDecoder.JSON_V2); - - run( // single-type index with ES 7+ + run( index + "-span-" + dateStamp, index + "-dependency-" + dateStamp, SpanBytesDecoder.JSON_V2); diff --git a/elasticsearch/src/test/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJobTest.java b/elasticsearch/src/test/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJobTest.java index e5e8d8c6..c87872db 100644 --- a/elasticsearch/src/test/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJobTest.java +++ b/elasticsearch/src/test/java/zipkin2/dependencies/elasticsearch/ElasticsearchDependenciesJobTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 The OpenZipkin Authors + * Copyright 2016-2023 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -14,6 +14,7 @@ package zipkin2.dependencies.elasticsearch; import java.io.IOException; +import java.util.Base64; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.elasticsearch.hadoop.EsHadoopException; @@ -24,7 +25,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static okhttp3.mockwebserver.SocketPolicy.DISCONNECT_AT_START; import static okhttp3.tls.internal.TlsUtil.localhost; -import static org.apache.commons.net.util.Base64.encodeBase64String; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -65,8 +65,9 @@ class ElasticsearchDependenciesJobTest { assertThatThrownBy(job::run) .isInstanceOf(EsHadoopException.class); + String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8)); assertThat(es.takeRequest().getHeader("Authorization")) - .isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes(UTF_8)).trim()); + .isEqualTo("Basic " + encoded.trim()); } @Test void authWorksWithSsl() throws Exception { @@ -88,8 +89,9 @@ class ElasticsearchDependenciesJobTest { assertThatThrownBy(job::run) .isInstanceOf(EsHadoopException.class); + String encoded = Base64.getEncoder().encodeToString("foo:bar".getBytes(UTF_8)); assertThat(es.takeRequest().getHeader("Authorization")) - .isEqualTo("Basic " + encodeBase64String("foo:bar".getBytes("UTF-8")).trim()); + .isEqualTo("Basic " + encoded.trim()); } @Test void parseHosts_default() { diff --git a/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchExtension.java b/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchExtension.java index b4647003..47ee5449 100644 --- a/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchExtension.java +++ b/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchExtension.java @@ -126,7 +126,7 @@ String hostPort() { // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537 static final class ElasticsearchContainer extends GenericContainer { ElasticsearchContainer(int majorVersion) { - super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.25.2")); + super(parse("ghcr.io/openzipkin/zipkin-elasticsearch" + majorVersion + ":2.26.0")); if ("true".equals(System.getProperty("docker.skip"))) { throw new TestAbortedException("${docker.skip} == true"); } diff --git a/elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV6Dependencies.java b/elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV8Dependencies.java similarity index 86% rename from elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV6Dependencies.java rename to elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV8Dependencies.java index 33fc746b..7131f026 100644 --- a/elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV6Dependencies.java +++ b/elasticsearch/src/test/java/zipkin2/elasticsearch/ITElasticsearchV8Dependencies.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 The OpenZipkin Authors + * Copyright 2016-2023 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -17,8 +17,8 @@ import org.junit.jupiter.api.extension.RegisterExtension; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class ITElasticsearchV6Dependencies extends ITElasticsearchDependencies { - @RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(6); +class ITElasticsearchV8Dependencies extends ITElasticsearchDependencies { + @RegisterExtension ElasticsearchExtension elasticsearch = new ElasticsearchExtension(8); @Override ElasticsearchExtension elasticsearch() { return elasticsearch; diff --git a/main/pom.xml b/main/pom.xml index 8659b44f..37bb0e6a 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -20,7 +20,7 @@ io.zipkin.dependencies zipkin-dependencies-parent - 2.7.4-SNAPSHOT + 3.0.0-SNAPSHOT zipkin-dependencies diff --git a/mysql/pom.xml b/mysql/pom.xml index 1afb307d..2d31198c 100644 --- a/mysql/pom.xml +++ b/mysql/pom.xml @@ -20,7 +20,7 @@ io.zipkin.dependencies zipkin-dependencies-parent - 2.7.4-SNAPSHOT + 3.0.0-SNAPSHOT zipkin-dependencies-mysql @@ -28,12 +28,6 @@ ${project.basedir}/.. - - 2.7.11 diff --git a/mysql/src/main/java/zipkin2/dependencies/mysql/MySQLDependenciesJob.java b/mysql/src/main/java/zipkin2/dependencies/mysql/MySQLDependenciesJob.java index 3f020cea..0b3f4071 100644 --- a/mysql/src/main/java/zipkin2/dependencies/mysql/MySQLDependenciesJob.java +++ b/mysql/src/main/java/zipkin2/dependencies/mysql/MySQLDependenciesJob.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 The OpenZipkin Authors + * Copyright 2016-2023 The OpenZipkin Authors * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -159,7 +159,8 @@ public MySQLDependenciesJob build() { this.url = new StringBuilder("jdbc:mysql://") .append(builder.host).append(":").append(builder.port) .append("/").append(builder.db) - .append("?autoReconnect=true") + .append("?permitMysqlScheme") // spark doesn't understand "mariadb" + .append("&autoReconnect=true") .append("&useSSL=").append(builder.useSsl).toString(); this.user = builder.user; this.password = builder.password; diff --git a/mysql/src/test/java/zipkin2/storage/mysql/v1/MySQLExtension.java b/mysql/src/test/java/zipkin2/storage/mysql/v1/MySQLExtension.java index ed44659c..982a1128 100644 --- a/mysql/src/test/java/zipkin2/storage/mysql/v1/MySQLExtension.java +++ b/mysql/src/test/java/zipkin2/storage/mysql/v1/MySQLExtension.java @@ -54,10 +54,11 @@ MySQLStorage.Builder computeStorageBuilder() { final MariaDbDataSource dataSource; try { - dataSource = new MariaDbDataSource(host(), port(), "zipkin"); + dataSource = new MariaDbDataSource(String.format( + "jdbc:mysql://%s:%s/zipkin?permitMysqlScheme&autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8", + host(), port())); dataSource.setUser("zipkin"); dataSource.setPassword("zipkin"); - dataSource.setProperties("autoReconnect=true&useUnicode=yes&characterEncoding=UTF-8"); } catch (SQLException e) { throw new AssertionError(e); } @@ -78,7 +79,7 @@ int port() { // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537 static final class MySQLContainer extends GenericContainer { MySQLContainer() { - super(parse("ghcr.io/openzipkin/zipkin-mysql:2.25.2")); + super(parse("ghcr.io/openzipkin/zipkin-mysql:2.26.0")); if ("true".equals(System.getProperty("docker.skip"))) { throw new TestAbortedException("${docker.skip} == true"); } diff --git a/pom.xml b/pom.xml index f2f3ead5..55c091f6 100755 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ io.zipkin.dependencies zipkin-dependencies-parent - 2.7.4-SNAPSHOT + 3.0.0-SNAPSHOT pom @@ -36,35 +36,53 @@ UTF-8 ${project.basedir} - 1.8 - java18 + + + 11 + 11 + + 11 - 2.23.0 + 2.24.0 - + 2.12 - - 3.0.3 + + 3.3.4 + + + 8.11.3 + + + 3.3.0 + + 4.17.0 - 2.5.1 - 4.9.0 + + 3.3.2 5.10.1 1.19.3 - 2.25.2 + 2.26.0 1.26.4 - 2.16.0 + + 4.1.100.Final + 2.16.1 3.24.2 1.7.36 ${skipTests} - 1.23 1.2.8 4.3 @@ -137,6 +155,14 @@ + + io.netty + netty-bom + ${netty.version} + pom + import + + org.apache.spark spark-core_${scala.binary.version} @@ -213,13 +239,6 @@ - - org.codehaus.mojo.signature - ${main.signature.artifact} - 1.0 - signature - MAIN - com.mycila license-maven-plugin-git @@ -299,10 +318,7 @@ ${maven-compiler-plugin.version} true - ${main.java.version} - ${main.java.version} true - 8 true @@ -335,29 +351,6 @@ - - - org.codehaus.mojo - animal-sniffer-maven-plugin - ${animal-sniffer-maven-plugin.version} - - false - ${skipTests} - - org.codehaus.mojo.signature - ${main.signature.artifact} - 1.0 - - - - - - check - - - - - maven-enforcer-plugin ${maven-enforcer-plugin.version} @@ -481,8 +474,6 @@ ${maven-compiler-plugin.version} true - ${main.java.version} - ${main.java.version} true true