deps: updates to latest versions and testcontainers jupiter

This updates to latest dependency versions and migrates tests to use testcontainers jupiter. This also removes use of nested tests as it is simpler to run and diagnose top-level tests. Signed-off-by: Adrian Cole <[email protected]>
openzipkin · Jan 24, 2024 · c1c5fd8 · c1c5fd8
1 parent 48b0de4
commit c1c5fd8
Show file tree

Hide file tree

Showing 36 changed files with 612 additions and 542 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -35,26 +35,31 @@ jobs:
           distribution: 'zulu'  # zulu as it supports a wide version range
           java-version: '11'  # Most recent LTS that passes tests
       - name: Test without Docker
-        run: build-bin/maven/maven_go_offline && build-bin/test -Ddocker.skip=true
+        run: build-bin/maven/maven_go_offline && build-bin/test -DexcludedGroups=docker
   test_docker:
+    name: test_docker (${{ matrix.name }})
     runs-on: ubuntu-22.04 # newest available distribution, aka jellyfish
     if: "!contains(github.event.head_commit.message, 'maven-release-plugin')"
     strategy:
       matrix:
         include:
           - name: zipkin-dependencies-cassandra3
-          - name: zipkin-dependencies-elasticsearch
+            module: zipkin-dependencies-cassandra3
+            groups: docker
+          - name: zipkin-dependencies-elasticsearch-v7
+            module: zipkin-dependencies-elasticsearch
+            groups: docker,elasticsearch7
+          - name: zipkin-dependencies-elasticsearch-v8
+            module: zipkin-dependencies-elasticsearch
+            groups: docker,elasticsearch8
           - name: zipkin-dependencies-mysql
+            module: zipkin-dependencies-mysql
+            groups: docker
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
-          fetch-depth: 1  # -Dlicense.skip=true so we don't need a full clone
-      # Remove apt repos that are known to break from time to time.
-      # See https://github.com/actions/virtual-environments/issues/323
-      - name: Remove broken apt repos
-        run: |
-          for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done
+          fetch-depth: 1  # -Dlicense.skip=true, so we don't need a full clone
       - name: Cache local Maven repository
         uses: actions/cache@v3
         with:
@@ -71,5 +76,10 @@ jobs:
           java-version: '11'  # Most recent LTS that passes tests
       - name: Test with Docker
         run: |
-          build-bin/configure_test &&
-          build-bin/test -pl :${{ matrix.name }} --am -Dlicense.skip=true
+          build-bin/docker/configure_docker &&
+          build-bin/maven/maven_go_offline &&
+          build-bin/maven/maven_build -pl :${{ matrix.module }} --am &&
+          build-bin/test -Dgroups=${{ matrix.groups }} -pl :${{ matrix.module }}
+        env:
+          MAVEN_GOAL: install  # docker build needs dependencies in mavenLocal
+          MAVEN_CONFIG: '-Dlicense.skip=true'  # license check already run
diff --git a/build-bin/docker/docker_args b/build-bin/docker/docker_args
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -46,7 +46,7 @@ if [ -n "${DOCKER_TARGET}" ]; then
 fi
 
 # When non-empty, becomes the base layer including tag appropriate for the image being built.
-# e.g. ghcr.io/openzipkin/java:21.0.1_p12-jre
+# e.g. ghcr.io/openzipkin/java:21.0.2_p13-jre
 #
 # This is not required to be a base (FROM scratch) image like ghcr.io/openzipkin/alpine:3.12.3
 # See https://docs.docker.com/glossary/#parent-image

diff --git a/build-bin/maven/maven_build b/build-bin/maven/maven_build
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -16,9 +16,10 @@
 set -ue
 
 export MAVEN_OPTS="$($(dirname "$0")/maven_opts)"
+maven_goal=${MAVEN_GOAL:-package}
 if [ -x ./mvnw ]; then alias mvn=${PWD}/mvnw; fi
 
 (
   if [ "${MAVEN_PROJECT_BASEDIR:-.}" != "." ]; then cd ${MAVEN_PROJECT_BASEDIR}; fi
-  mvn -T1C -q --batch-mode -DskipTests package "$@"
+  mvn -T1C -q --batch-mode -DskipTests "${maven_goal}" "$@"
 )
diff --git a/build-bin/maven/maven_build_or_unjar b/build-bin/maven/maven_build_or_unjar
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/build-bin/maven/maven_deploy b/build-bin/maven/maven_deploy
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/build-bin/maven/maven_go_offline b/build-bin/maven/maven_go_offline
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/build-bin/maven/maven_opts b/build-bin/maven/maven_opts
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/build-bin/maven/maven_release b/build-bin/maven/maven_release
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/build-bin/maven/maven_unjar b/build-bin/maven/maven_unjar
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2016-2023 The OpenZipkin Authors
+# Copyright 2016-2024 The OpenZipkin Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at

diff --git a/cassandra3/pom.xml b/cassandra3/pom.xml
@@ -47,7 +47,7 @@
 
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>testcontainers</artifactId>
+      <artifactId>junit-jupiter</artifactId>
       <version>${testcontainers.version}</version>
       <scope>test</scope>
     </dependency>

diff --git a/...storage/cassandra/CassandraExtension.java → ...storage/cassandra/CassandraContainer.java b/...storage/cassandra/CassandraExtension.java → ...storage/cassandra/CassandraContainer.java
@@ -19,26 +19,27 @@
 import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric;
 import com.datastax.oss.driver.api.core.metrics.Metrics;
 import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException;
+import com.google.common.collect.Lists;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
-import org.junit.jupiter.api.extension.AfterAllCallback;
-import org.junit.jupiter.api.extension.BeforeAllCallback;
-import org.junit.jupiter.api.extension.ExtensionContext;
-import org.opentest4j.TestAbortedException;
+import java.util.Set;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.containers.wait.strategy.Wait;
+import zipkin2.Span;
+import zipkin2.dependencies.cassandra3.CassandraDependenciesJob;
 import zipkin2.storage.cassandra.CassandraStorage.SessionFactory;
 
 import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.testcontainers.utility.DockerImageName.parse;
 import static zipkin2.Call.propagateIfFatal;
+import static zipkin2.storage.ITDependencies.aggregateLinks;
 import static zipkin2.storage.cassandra.Schema.TABLE_AUTOCOMPLETE_TAGS;
 import static zipkin2.storage.cassandra.Schema.TABLE_DEPENDENCY;
 import static zipkin2.storage.cassandra.Schema.TABLE_SERVICE_REMOTE_SERVICES;
@@ -47,8 +48,8 @@
 import static zipkin2.storage.cassandra.Schema.TABLE_TRACE_BY_SERVICE_REMOTE_SERVICE;
 import static zipkin2.storage.cassandra.Schema.TABLE_TRACE_BY_SERVICE_SPAN;
 
-public class CassandraExtension implements BeforeAllCallback, AfterAllCallback {
-  static final Logger LOGGER = LoggerFactory.getLogger(CassandraExtension.class);
+final class CassandraContainer extends GenericContainer<CassandraContainer> {
+  static final Logger LOGGER = LoggerFactory.getLogger(CassandraContainer.class);
   static final List<String> SEARCH_TABLES = asList(
     TABLE_AUTOCOMPLETE_TAGS,
     TABLE_SERVICE_REMOTE_SERVICES,
@@ -57,16 +58,17 @@ public class CassandraExtension implements BeforeAllCallback, AfterAllCallback {
     TABLE_TRACE_BY_SERVICE_SPAN
   );
 
-  final CassandraContainer container = new CassandraContainer();
-  CqlSession globalSession;
+  CassandraContainer() {
+    super(parse("ghcr.io/openzipkin/zipkin-cassandra:3.0.5"));
+    addExposedPort(9042);
+    waitStrategy = Wait.forHealthcheck();
+    withLogConsumer(new Slf4jLogConsumer(LOGGER));
+  }
 
-  @Override public void beforeAll(ExtensionContext context) {
-    if (context.getRequiredTestClass().getEnclosingClass() != null) {
-      // Only run once in outermost scope.
-      return;
-    }
+  CqlSession globalSession;
 
-    container.start();
+  @Override public void start() {
+    super.start();
     LOGGER.info("Using contactPoint " + contactPoint());
     globalSession = tryToInitializeSession(contactPoint());
   }
@@ -80,7 +82,9 @@ static CqlSession tryToInitializeSession(String contactPoint) {
       session.execute("SELECT now() FROM system.local");
     } catch (Throwable e) {
       propagateIfFatal(e);
-      if (session != null) session.close();
+      if (session != null) {
+        session.close();
+      }
       assumeTrue(false, e.getMessage());
     }
     return session;
@@ -95,16 +99,20 @@ static CassandraStorage.Builder newStorageBuilder(String contactPoint) {
   }
 
   String contactPoint() {
-    return container.getHost() + ":" + container.getMappedPort(9042);
+    return getHost() + ":" + getMappedPort(9042);
   }
 
   void clear(CassandraStorage storage) {
     // Clear any key cache
     CassandraSpanConsumer spanConsumer = storage.spanConsumer;
-    if (spanConsumer != null) spanConsumer.clear();
+    if (spanConsumer != null) {
+      spanConsumer.clear();
+    }
 
     CqlSession session = storage.session.session;
-    if (session == null) session = globalSession;
+    if (session == null) {
+      session = globalSession;
+    }
 
     List<String> toTruncate = new ArrayList<>(SEARCH_TABLES);
     toTruncate.add(TABLE_DEPENDENCY);
@@ -121,12 +129,12 @@ void clear(CassandraStorage storage) {
     blockWhileInFlight(storage);
   }
 
-  @Override public void afterAll(ExtensionContext context) {
-    if (context.getRequiredTestClass().getEnclosingClass() != null) {
-      // Only run once in outermost scope.
-      return;
+  @Override
+  public void stop() {
+    if (globalSession != null) {
+      globalSession.close();
     }
-    if (globalSession != null) globalSession.close();
+    super.stop();
   }
 
   static void blockWhileInFlight(CassandraStorage storage) {
@@ -135,7 +143,9 @@ static void blockWhileInFlight(CassandraStorage storage) {
     boolean wasInFlight = false;
     while (true) {
       if (!poolInFlight(session)) {
-        if (wasInFlight) sleep(100); // give a little more to avoid flakey tests
+        if (wasInFlight) {
+          sleep(100); // give a little more to avoid flakey tests
+        }
         return;
       }
       wasInFlight = true;
@@ -161,21 +171,39 @@ static boolean poolInFlight(CqlSession session) {
       int inFlight = metrics.flatMap(m -> m.getNodeMetric(node, DefaultNodeMetric.IN_FLIGHT))
         .map(m -> ((Gauge<Integer>) m).getValue())
         .orElse(0);
-      if (inFlight > 0) return true;
+      if (inFlight > 0) {
+        return true;
+      }
     }
     return false;
   }
 
-  // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537
-  static final class CassandraContainer extends GenericContainer<CassandraContainer> {
-    CassandraContainer() {
-      super(parse("ghcr.io/openzipkin/zipkin-cassandra:3.0.1"));
-      if ("true".equals(System.getProperty("docker.skip"))) {
-        throw new TestAbortedException("${docker.skip} == true");
-      }
-      addExposedPort(9042);
-      waitStrategy = Wait.forHealthcheck();
-      withLogConsumer(new Slf4jLogConsumer(LOGGER));
+  /**
+   * This processes the job as if it were a batch. For each day we had traces, run the job again.
+   */
+  void processDependencies(CassandraStorage storage, List<Span> spans) throws Exception {
+    // TODO: this avoids overrunning the cluster with BusyPoolException
+    for (List<Span> nextChunk : Lists.partition(spans, 100)) {
+      storage.spanConsumer().accept(nextChunk).execute();
+      // Now, block until writes complete, notably so we can read them.
+      blockWhileInFlight(storage);
+    }
+
+    // aggregate links in memory to determine which days they are in
+    Set<Long> days = aggregateLinks(spans).keySet();
+
+    // process the job for each day of links.
+    for (long day : days) {
+      CassandraDependenciesJob.builder()
+        .keyspace(storage.keyspace)
+        .localDc(storage.localDc)
+        .contactPoints(storage.contactPoints)
+        .strictTraceId(false)
+        .day(day)
+        .build()
+        .run();
     }
+
+    blockWhileInFlight(storage);
   }
 }