diff --git a/.dockerignore b/.dockerignore index edd1df83c8..498d972aec 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,6 +24,9 @@ !docker/test-images/zipkin-elasticsearch8/config/ !docker/test-images/zipkin-elasticsearch8/start-elasticsearch +!docker/test-images/zipkin-opensearch2/config/ +!docker/test-images/zipkin-opensearch2/start-opensearch + !docker/test-images/zipkin-eureka/src/ !docker/test-images/zipkin-eureka/pom.xml !docker/test-images/zipkin-eureka/start-eureka diff --git a/docker/test-images/zipkin-opensearch2/Dockerfile b/docker/test-images/zipkin-opensearch2/Dockerfile new file mode 100644 index 0000000000..99d958996d --- /dev/null +++ b/docker/test-images/zipkin-opensearch2/Dockerfile @@ -0,0 +1,73 @@ +# +# Copyright The OpenZipkin Authors +# SPDX-License-Identifier: Apache-2.0 +# + +# java_version is used for install and runtime layers of zipkin-opensearch2 +# +# Use latest version here: https://github.com/orgs/openzipkin/packages/container/package/java +# This is defined in many places because Docker has no "env" script functionality unless you use +# docker-compose: When updating, update everywhere. +ARG java_version=21.0.2_p13 + +# We copy files from the context into a scratch container first to avoid a problem where docker and +# docker-compose don't share layer hashes https://github.com/docker/compose/issues/883 normally. +# COPY --from= works around the issue. +FROM scratch as scratch + +COPY build-bin/docker/docker-healthcheck /docker-bin/ +COPY docker/test-images/zipkin-opensearch2/start-opensearch /docker-bin/ +COPY docker/test-images/zipkin-opensearch2/config/ /config/ + +FROM ghcr.io/openzipkin/java:${java_version} as install + +WORKDIR /install + +# Use latest 2.x version from https://opensearch.org/downloads.html +# This is defined in many places because Docker has no "env" script functionality unless you use +# docker-compose: When updating, update everywhere. +ARG opensearch2_version=2.13.0 + +# Download only the OSS distribution (lacks X-Pack) +RUN \ +# Connection resets are frequent in GitHub Actions workflows \ +wget --random-wait --tries=5 -qO- \ +# We don't download bin scripts as we customize for reasons including BusyBox problems +https://artifacts.opensearch.org/releases/bundle/opensearch/${opensearch2_version}/opensearch-${opensearch2_version}-linux-x64.tar.gz| tar xz \ + --wildcards --strip=1 --exclude=jdk +COPY --from=scratch /config/ ./config/ + +# Use a full Java distribution rather than adding test modules to the +# production -jre base layer used by zipkin and zipkin-slim. +FROM ghcr.io/openzipkin/java:${java_version} as zipkin-opensearch2 +LABEL org.opencontainers.image.description="OpenSearch distribution on OpenJDK and Alpine Linux" +ARG opensearch2_version=2.13.0 +LABEL opensearch-version=$opensearch2_version + +# Install bash +RUN apk update && apk add bash + +# The full license is also included in the image at /opensearch/LICENSE.txt. +LABEL org.opencontainers.image.licenses="Apache-2.0" + +# Add HEALTHCHECK and ENTRYPOINT scripts into the default search path +COPY --from=scratch /docker-bin/* /usr/local/bin/ +# We use start period of 30s to avoid marking the container unhealthy on slow or contended CI hosts +HEALTHCHECK --interval=1s --start-period=30s --timeout=5s CMD ["docker-healthcheck"] +ENTRYPOINT ["start-opensearch"] + +# All content including binaries and logs write under WORKDIR +ARG USER=opensearch +WORKDIR /${USER} + +# Ensure the process doesn't run as root +RUN adduser -g '' -h ${PWD} -D ${USER} +USER ${USER} + +# Copy binaries and config we installed earlier +COPY --from=install --chown=${USER} /install . + +# Use to set heap, trust store or other system properties. +ENV OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -XX:+ExitOnOutOfMemoryError" +ENV LIBFFI_TMPDIR=/tmp +EXPOSE 9200 diff --git a/docker/test-images/zipkin-opensearch2/README.md b/docker/test-images/zipkin-opensearch2/README.md new file mode 100644 index 0000000000..90f7879bad --- /dev/null +++ b/docker/test-images/zipkin-opensearch2/README.md @@ -0,0 +1,27 @@ +## zipkin-opensearch2 Docker image + +The `zipkin-opensearch2` testing image runs OpenSearch 2.x for [Elasticsearch storage](../../../zipkin-storage/elasticsearch) +integration. + +To build `openzipkin/zipkin-opensearch2:test`, from the top-level of the repository, run: +```bash +$ DOCKER_FILE=docker/test-images/zipkin-opensearch2/Dockerfile build-bin/docker/docker_build openzipkin/zipkin-opensearch2:test +``` + +You can use the env variable `OPENSEARCH_JAVA_OPTS` to change settings such as heap size for OpenSearch. + +#### Host setup + +OpenSearch is [strict](https://github.com/docker-library/docs/tree/master/elasticsearch#host-setup) +about virtual memory. You will need to adjust accordingly (especially if you notice OpenSearch crash!) + +```bash +# If docker is running on your host machine, adjust the kernel setting directly +$ sudo sysctl -w vm.max_map_count=262144 + +# If using docker-machine/Docker Toolbox/Boot2Docker, remotely adjust the same +$ docker-machine ssh default "sudo sysctl -w vm.max_map_count=262144" + +# If using colima, it is similar as well +$ colima ssh "sudo sysctl -w vm.max_map_count=262144" +``` diff --git a/docker/test-images/zipkin-opensearch2/config/log4j2.properties b/docker/test-images/zipkin-opensearch2/config/log4j2.properties new file mode 100644 index 0000000000..2bbd551db3 --- /dev/null +++ b/docker/test-images/zipkin-opensearch2/config/log4j2.properties @@ -0,0 +1,14 @@ +# +# Copyright The OpenZipkin Authors +# SPDX-License-Identifier: Apache-2.0 +# + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console diff --git a/docker/test-images/zipkin-opensearch2/config/opensearch.yml b/docker/test-images/zipkin-opensearch2/config/opensearch.yml new file mode 100644 index 0000000000..31670f207f --- /dev/null +++ b/docker/test-images/zipkin-opensearch2/config/opensearch.yml @@ -0,0 +1,21 @@ +# +# Copyright The OpenZipkin Authors +# SPDX-License-Identifier: Apache-2.0 +# + +cluster.name: "docker-cluster" +network.host: 0.0.0.0 +node.name: zipkin-opensearch + +# Enable development mode and disable bootstrap checks +# See https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html +discovery.type: single-node +# Avoid deprecation errors: as of 8.x the only accepted value is true. +cluster.routing.allocation.disk.watermark.enable_for_single_data_node: true + +# This is a test image, so we are not afraid to delete all indices. Avoids: +# Wildcard expressions or all indices are not allowed +action.destructive_requires_name: false + +# Disable security +plugins.security.disabled: true diff --git a/docker/test-images/zipkin-opensearch2/start-opensearch b/docker/test-images/zipkin-opensearch2/start-opensearch new file mode 100644 index 0000000000..0212120d7d --- /dev/null +++ b/docker/test-images/zipkin-opensearch2/start-opensearch @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Copyright The OpenZipkin Authors +# SPDX-License-Identifier: Apache-2.0 +# + +# ENTRYPOINT script that starts OpenSearch +# +# This intentionally locates config using the current working directory, in order to consolidate +# Dockerfile instructions to WORKDIR +set -eu + +# Configure the Docker HEALTHCHECK +export HEALTHCHECK_PORT=9200 +export HEALTHCHECK_PATH=/_cluster/health + +exec ./bin/opensearch diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/BaseVersion.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/BaseVersion.java new file mode 100644 index 0000000000..e11bfbbc2e --- /dev/null +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/BaseVersion.java @@ -0,0 +1,108 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package zipkin2.elasticsearch; + +import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; + +import java.io.IOException; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.linecorp.armeria.common.AggregatedHttpRequest; +import com.linecorp.armeria.common.HttpMethod; + +import zipkin2.elasticsearch.internal.client.HttpCall; + +/** + * Base version for both Elasticsearch and OpenSearch distributions. + */ +public abstract class BaseVersion { + final int major, minor; + + BaseVersion(int major, int minor) { + this.major = major; + this.minor = minor; + } + + /** + * Gets the version for particular distribution, returns either {@link ElasticsearchVersion} + * or {@link OpensearchVersion} instance. + * @param http the HTTP client + * @return either {@link ElasticsearchVersion} or {@link OpensearchVersion} instance + * @throws IOException in case of I/O errors + */ + static BaseVersion get(HttpCall.Factory http) throws IOException { + return Parser.INSTANCE.get(http); + } + + /** + * Does this version of Elasticsearch / OpenSearch still support mapping types? + * @return "true" if mapping types are supported, "false" otherwise + */ + public abstract boolean supportsTypes(); + + enum Parser implements HttpCall.BodyConverter { + INSTANCE; + + final Pattern REGEX = Pattern.compile("(\\d+)\\.(\\d+).*"); + + BaseVersion get(HttpCall.Factory callFactory) throws IOException { + AggregatedHttpRequest getNode = AggregatedHttpRequest.of(HttpMethod.GET, "/"); + BaseVersion version = callFactory.newCall(getNode, this, "get-node").execute(); + if (version == null) { + throw new IllegalArgumentException("No content reading Elasticsearch version"); + } + return version; + } + + @Override + public BaseVersion convert(JsonParser parser, Supplier contentString) { + String version = null; + String distribution = null; + try { + if (enterPath(parser, "version") != null) { + while (parser.nextToken() != null) { + if (parser.currentToken() == JsonToken.VALUE_STRING) { + if (parser.currentName() == "distribution") { + distribution = parser.getText(); + } else if (parser.currentName() == "number") { + version = parser.getText(); + } + } + } + } + } catch (RuntimeException | IOException possiblyParseException) { + // EmptyCatch ignored + } + + if (version == null) { + throw new IllegalArgumentException( + ".version.number not found in response: " + contentString.get()); + } + + Matcher matcher = REGEX.matcher(version); + if (!matcher.matches()) { + throw new IllegalArgumentException("Invalid .version.number: " + version); + } + + try { + int major = Integer.parseInt(matcher.group(1)); + int minor = Integer.parseInt(matcher.group(2)); + if ("opensearch".equalsIgnoreCase(distribution)) { + return new OpensearchVersion(major, minor); + } else { + return new ElasticsearchVersion(major, minor); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid .version.number: " + version + + ", for .version.distribution:" + distribution); + } + } + } +} diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpecificTemplates.java new file mode 100644 index 0000000000..cc535a4618 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpecificTemplates.java @@ -0,0 +1,107 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package zipkin2.elasticsearch; + +import static zipkin2.elasticsearch.ElasticsearchVersion.V5_0; +import static zipkin2.elasticsearch.ElasticsearchVersion.V6_0; +import static zipkin2.elasticsearch.ElasticsearchVersion.V6_7; +import static zipkin2.elasticsearch.ElasticsearchVersion.V7_0; +import static zipkin2.elasticsearch.ElasticsearchVersion.V7_8; +import static zipkin2.elasticsearch.ElasticsearchVersion.V9_0; + +import zipkin2.internal.Nullable; + +final class ElasticsearchSpecificTemplates extends VersionSpecificTemplates { + static class DistributionTemplate extends DistributionSpecificTemplates { + private final ElasticsearchVersion version; + + DistributionTemplate(ElasticsearchVersion version) { + this.version = version; + } + + @Override String indexTemplatesUrl(String indexPrefix, String type, @Nullable Integer templatePriority) { + if (version.compareTo(V7_8) >= 0 && templatePriority != null) { + return "/_index_template/" + indexPrefix + type + "_template"; + } + if (version.compareTo(V6_7) >= 0 && version.compareTo(V7_0) < 0) { + // because deprecation warning on 6 to prepare for 7: + // + // [types removal] The parameter include_type_name should be explicitly specified in get + // template requests to prepare for 7.0. In 7.0 include_type_name will default to 'false', + // which means responses will omit the type name in mapping definitions. + // + // The parameter include_type_name was added in 6.7. Using this with ES older than + // 6.7 will result in unrecognized parameter: [include_type_name]. + return "/_template/" + indexPrefix + type + "_template?include_type_name=true"; + } + + return "/_template/" + indexPrefix + type + "_template"; + } + + @Override char indexTypeDelimiter() { + return ElasticsearchSpecificTemplates.indexTypeDelimiter(version); + } + + @Override + IndexTemplates get(String indexPrefix, int indexReplicas, int indexShards, + boolean searchEnabled, boolean strictTraceId, Integer templatePriority) { + return new ElasticsearchSpecificTemplates(indexPrefix, indexReplicas, indexShards, + searchEnabled, strictTraceId, templatePriority).get(version); + } + } + + ElasticsearchSpecificTemplates(String indexPrefix, int indexReplicas, int indexShards, + boolean searchEnabled, boolean strictTraceId, Integer templatePriority) { + super(indexPrefix, indexReplicas,indexShards, searchEnabled, strictTraceId, templatePriority); + } + + @Override String indexPattern(String type, ElasticsearchVersion version) { + return '"' + + (version.compareTo(V6_0) < 0 ? "template" : "index_patterns") + + "\": \"" + + indexPrefix + + indexTypeDelimiter(version) + + type + + "-*" + + "\""; + } + + @Override boolean useComposableTemplate(ElasticsearchVersion version) { + return (version.compareTo(V7_8) >= 0 && templatePriority != null); + } + + /** + * This returns a delimiter based on what's supported by the Elasticsearch version. + * + *

Starting in Elasticsearch 7.x, colons are no longer allowed in index names. This logic will + * make sure the pattern in our index template doesn't use them either. + * + *

See https://github.com/openzipkin/zipkin/issues/2219 + */ + static char indexTypeDelimiter(ElasticsearchVersion version) { + return version.compareTo(V7_0) < 0 ? ':' : '-'; + } + + @Override String maybeWrap(String type, ElasticsearchVersion version, String json) { + // ES 7.x defaults include_type_name to false https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_literal_include_type_name_literal_now_defaults_to_literal_false_literal + if (version.compareTo(V7_0) >= 0) return json; + return " \"" + type + "\": {\n " + json.replace("\n", "\n ") + " }\n"; + } + + @Override IndexTemplates get(ElasticsearchVersion version) { + if (version.compareTo(V5_0) < 0 || version.compareTo(V9_0) >= 0) { + throw new IllegalArgumentException( + "Elasticsearch versions 5-8.x are supported, was: " + version); + } + return IndexTemplates.newBuilder() + .version(version) + .indexTypeDelimiter(indexTypeDelimiter(version)) + .span(spanIndexTemplate(version)) + .dependency(dependencyTemplate(version)) + .autocomplete(autocompleteTemplate(version)) + .build(); + } +} diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java index 0658681a95..067226ce49 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java @@ -36,9 +36,6 @@ import zipkin2.storage.Traces; import static com.linecorp.armeria.common.HttpMethod.GET; -import static zipkin2.elasticsearch.ElasticsearchVersion.V6_7; -import static zipkin2.elasticsearch.ElasticsearchVersion.V7_0; -import static zipkin2.elasticsearch.ElasticsearchVersion.V7_8; import static zipkin2.elasticsearch.EnsureIndexTemplate.ensureIndexTemplate; import static zipkin2.elasticsearch.VersionSpecificTemplates.TYPE_AUTOCOMPLETE; import static zipkin2.elasticsearch.VersionSpecificTemplates.TYPE_DEPENDENCY; @@ -240,17 +237,17 @@ public final Builder dateSeparator(char dateSeparator) { return new ElasticsearchSpanConsumer(this); } - /** Returns the Elasticsearch version of the connected cluster. Internal use only */ - @Memoized public ElasticsearchVersion version() { + /** Returns the Elasticsearch / OpenSearch version of the connected cluster. Internal use only */ + @Memoized public BaseVersion version() { try { - return ElasticsearchVersion.get(http()); + return BaseVersion.get(http()); } catch (IOException e) { throw new UncheckedIOException(e); } } char indexTypeDelimiter() { - return VersionSpecificTemplates.indexTypeDelimiter(version()); + return VersionSpecificTemplates.forVersion(version()).indexTypeDelimiter(); } /** This is an internal blocking call, only used in tests. */ @@ -337,35 +334,20 @@ IndexTemplates doEnsureIndexTemplates() { } } - IndexTemplates versionSpecificTemplates(ElasticsearchVersion version) { - return new VersionSpecificTemplates( + IndexTemplates versionSpecificTemplates(BaseVersion version) { + return VersionSpecificTemplates.forVersion(version).get( indexNameFormatter().index(), indexReplicas(), indexShards(), searchEnabled(), strictTraceId(), templatePriority() - ).get(version); + ); } String buildUrl(IndexTemplates templates, String type) { String indexPrefix = indexNameFormatter().index() + templates.indexTypeDelimiter(); - - if (version().compareTo(V7_8) >= 0 && templatePriority() != null) { - return "/_index_template/" + indexPrefix + type + "_template"; - } - if (version().compareTo(V6_7) >= 0 && version().compareTo(V7_0) < 0) { - // because deprecation warning on 6 to prepare for 7: - // - // [types removal] The parameter include_type_name should be explicitly specified in get - // template requests to prepare for 7.0. In 7.0 include_type_name will default to 'false', - // which means responses will omit the type name in mapping definitions. - // - // The parameter include_type_name was added in 6.7. Using this with ES older than - // 6.7 will result in unrecognized parameter: [include_type_name]. - return "/_template/" + indexPrefix + type + "_template?include_type_name=true"; - } - return "/_template/" + indexPrefix + type + "_template"; + return VersionSpecificTemplates.forVersion(version()).indexTemplatesUrl(indexPrefix, type, templatePriority()); } @Override public final String toString() { diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchVersion.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchVersion.java index e7d17ce04c..e7c821e0c9 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchVersion.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchVersion.java @@ -4,20 +4,10 @@ */ package zipkin2.elasticsearch; -import com.fasterxml.jackson.core.JsonParser; -import com.linecorp.armeria.common.AggregatedHttpRequest; -import com.linecorp.armeria.common.HttpMethod; -import java.io.IOException; import java.util.Objects; -import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import zipkin2.elasticsearch.internal.client.HttpCall; - -import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; /** Helps avoid problems comparing versions by number. Ex 7.10 should be > 7.9 */ -public final class ElasticsearchVersion implements Comparable { +public final class ElasticsearchVersion extends BaseVersion implements Comparable { public static final ElasticsearchVersion V5_0 = new ElasticsearchVersion(5, 0); public static final ElasticsearchVersion V6_0 = new ElasticsearchVersion(6, 0); public static final ElasticsearchVersion V6_7 = new ElasticsearchVersion(6, 7); @@ -25,15 +15,12 @@ public final class ElasticsearchVersion implements Comparable { - INSTANCE; - - final Pattern REGEX = Pattern.compile("(\\d+)\\.(\\d+).*"); - ElasticsearchVersion get(HttpCall.Factory callFactory) throws IOException { - AggregatedHttpRequest getNode = AggregatedHttpRequest.of(HttpMethod.GET, "/"); - ElasticsearchVersion version = callFactory.newCall(getNode, this, "get-node").execute(); - if (version == null) { - throw new IllegalArgumentException("No content reading Elasticsearch version"); - } - return version; - } - - @Override - public ElasticsearchVersion convert(JsonParser parser, Supplier contentString) { - String version = null; - try { - if (enterPath(parser, "version", "number") != null) version = parser.getText(); - } catch (RuntimeException | IOException possiblyParseException) { - // EmptyCatch ignored - } - if (version == null) { - throw new IllegalArgumentException( - ".version.number not found in response: " + contentString.get()); - } - - Matcher matcher = REGEX.matcher(version); - if (!matcher.matches()) { - throw new IllegalArgumentException("Invalid .version.number: " + version); - } - - try { - int major = Integer.parseInt(matcher.group(1)); - int minor = Integer.parseInt(matcher.group(2)); - return new ElasticsearchVersion(major, minor); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid .version.number: " + version); - } - } - } } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java index 13b056e095..eb61a49973 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java @@ -12,7 +12,7 @@ static Builder newBuilder() { return new AutoValue_IndexTemplates.Builder(); } - abstract ElasticsearchVersion version(); + abstract BaseVersion version(); abstract char indexTypeDelimiter(); @@ -24,7 +24,7 @@ static Builder newBuilder() { @AutoValue.Builder interface Builder { - Builder version(ElasticsearchVersion version); + Builder version(BaseVersion version); Builder indexTypeDelimiter(char indexTypeDelimiter); diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchSpecificTemplates.java new file mode 100644 index 0000000000..cb1c4b3632 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchSpecificTemplates.java @@ -0,0 +1,82 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package zipkin2.elasticsearch; + +import static zipkin2.elasticsearch.OpensearchVersion.V1_0; +import static zipkin2.elasticsearch.OpensearchVersion.V4_0; + +import zipkin2.internal.Nullable; + +final class OpensearchSpecificTemplates extends VersionSpecificTemplates { + static class DistributionTemplate extends DistributionSpecificTemplates { + private final OpensearchVersion version; + + DistributionTemplate(OpensearchVersion version) { + this.version = version; + } + + @Override String indexTemplatesUrl(String indexPrefix, String type, @Nullable Integer templatePriority) { + if (version.compareTo(V1_0) >= 0 && templatePriority != null) { + return "/_index_template/" + indexPrefix + type + "_template"; + } + + return "/_template/" + indexPrefix + type + "_template"; + } + + @Override char indexTypeDelimiter() { + return OpensearchSpecificTemplates.indexTypeDelimiter(version); + } + + @Override + IndexTemplates get(String indexPrefix, int indexReplicas, int indexShards, + boolean searchEnabled, boolean strictTraceId, Integer templatePriority) { + return new OpensearchSpecificTemplates(indexPrefix, indexReplicas, indexShards, + searchEnabled, strictTraceId, templatePriority).get(version); + } + } + + OpensearchSpecificTemplates(String indexPrefix, int indexReplicas, int indexShards, + boolean searchEnabled, boolean strictTraceId, Integer templatePriority) { + super(indexPrefix, indexReplicas,indexShards, searchEnabled, strictTraceId, templatePriority); + } + + @Override String indexPattern(String type, OpensearchVersion version) { + return '"' + + "index_patterns" + + "\": \"" + + indexPrefix + + indexTypeDelimiter(version) + + type + + "-*" + + "\""; + } + + static char indexTypeDelimiter(OpensearchVersion version) { + return '-'; + } + + @Override boolean useComposableTemplate(OpensearchVersion version) { + return (templatePriority != null); + } + + @Override String maybeWrap(String type, OpensearchVersion version, String json) { + return json; + } + + @Override IndexTemplates get(OpensearchVersion version) { + if (version.compareTo(V1_0) < 0 || version.compareTo(V4_0) >= 0) { + throw new IllegalArgumentException( + "OpenSearch versions 1-3.x are supported, was: " + version); + } + return IndexTemplates.newBuilder() + .version(version) + .indexTypeDelimiter(indexTypeDelimiter(version)) + .span(spanIndexTemplate(version)) + .dependency(dependencyTemplate(version)) + .autocomplete(autocompleteTemplate(version)) + .build(); + } +} diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchVersion.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchVersion.java new file mode 100644 index 0000000000..b7f1e1714d --- /dev/null +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/OpensearchVersion.java @@ -0,0 +1,44 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package zipkin2.elasticsearch; + +import java.util.Objects; + +/** Helps avoid problems comparing versions by number. Ex 2.10 should be > 2.9 */ +public final class OpensearchVersion extends BaseVersion implements Comparable { + public static final OpensearchVersion V1_0 = new OpensearchVersion(1, 0); + public static final OpensearchVersion V2_0 = new OpensearchVersion(2, 0); + public static final OpensearchVersion V3_0 = new OpensearchVersion(2, 0); + public static final OpensearchVersion V4_0 = new OpensearchVersion(4, 0); + + OpensearchVersion(int major, int minor) { + super(major, minor); + } + + @Override public boolean supportsTypes() { + return compareTo(V2_0) < 0; + } + + @Override public int compareTo(OpensearchVersion other) { + if (major < other.major) return -1; + if (major > other.major) return 1; + return Integer.compare(minor, other.minor); + } + + @Override public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof OpensearchVersion)) return false; + OpensearchVersion that = (OpensearchVersion) o; + return this.major == that.major && this.minor == that.minor; + } + + @Override public int hashCode() { + return Objects.hash(major, minor); + } + + @Override public String toString() { + return major + "." + minor; + } +} diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java index b40f66423b..1307cef465 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java @@ -4,17 +4,13 @@ */ package zipkin2.elasticsearch; -import static zipkin2.elasticsearch.ElasticsearchVersion.V5_0; -import static zipkin2.elasticsearch.ElasticsearchVersion.V6_0; -import static zipkin2.elasticsearch.ElasticsearchVersion.V7_0; -import static zipkin2.elasticsearch.ElasticsearchVersion.V7_8; -import static zipkin2.elasticsearch.ElasticsearchVersion.V9_0; +import zipkin2.internal.Nullable; /** Returns version-specific index templates */ // TODO: make a main class that spits out the index template using ENV variables for the server, // a parameter for the version, and a parameter for the index type. Ex. // java -cp zipkin-storage-elasticsearch.jar zipkin2.elasticsearch.VersionSpecificTemplates 6.7 span -final class VersionSpecificTemplates { +abstract class VersionSpecificTemplates { /** Maximum character length constraint of most names, IP literals and IDs. */ static final int SHORT_STRING_LENGTH = 256; static final String TYPE_AUTOCOMPLETE = "autocomplete"; @@ -42,18 +38,7 @@ final class VersionSpecificTemplates { this.templatePriority = templatePriority; } - String indexPattern(String type, ElasticsearchVersion version) { - return '"' - + (version.compareTo(V6_0) < 0 ? "template" : "index_patterns") - + "\": \"" - + indexPrefix - + indexTypeDelimiter(version) - + type - + "-*" - + "\""; - } - - String indexProperties(ElasticsearchVersion version) { + String indexProperties(V version) { // 6.x _all disabled https://www.elastic.co/guide/en/elasticsearch/reference/6.7/breaking-changes-6.0.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default // 7.x _default disallowed https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_the_literal__default__literal_mapping_is_no_longer_allowed String result = " \"index.number_of_shards\": " + indexShards + ",\n" @@ -62,7 +47,7 @@ String indexProperties(ElasticsearchVersion version) { return result + "\n"; } - String indexTemplate(ElasticsearchVersion version) { + String indexTemplate(V version) { if (useComposableTemplate(version)) { return "\"template\": {\n"; } @@ -70,7 +55,7 @@ String indexTemplate(ElasticsearchVersion version) { return ""; } - String indexTemplateClosing(ElasticsearchVersion version) { + String indexTemplateClosing(V version) { if (useComposableTemplate(version)) { return "},\n"; } @@ -78,7 +63,7 @@ String indexTemplateClosing(ElasticsearchVersion version) { return ""; } - String templatePriority(ElasticsearchVersion version) { + String templatePriority(V version) { if (useComposableTemplate(version)) { return "\"priority\": " + templatePriority + "\n"; } @@ -86,7 +71,7 @@ String templatePriority(ElasticsearchVersion version) { return ""; } - String beginTemplate(String type, ElasticsearchVersion version) { + String beginTemplate(String type, V version) { return "{\n" + " " + indexPattern(type, version) + ",\n" + indexTemplate(version) @@ -94,14 +79,14 @@ String beginTemplate(String type, ElasticsearchVersion version) { + indexProperties(version); } - String endTemplate(ElasticsearchVersion version) { + String endTemplate(V version) { return indexTemplateClosing(version) + templatePriority(version) + "}"; } /** Templatized due to version differences. Only fields used in search are declared */ - String spanIndexTemplate(ElasticsearchVersion version) { + String spanIndexTemplate(V version) { String result = beginTemplate(TYPE_SPAN, version); String traceIdMapping = KEYWORD; @@ -187,7 +172,7 @@ String spanIndexTemplate(ElasticsearchVersion version) { } /** Templatized due to version differences. Only fields used in search are declared */ - String dependencyTemplate(ElasticsearchVersion version) { + String dependencyTemplate(V version) { return beginTemplate(TYPE_DEPENDENCY, version) + " },\n" + " \"mappings\": {\n" @@ -198,7 +183,7 @@ String dependencyTemplate(ElasticsearchVersion version) { // The key filed of a autocompleteKeys is intentionally names as tagKey since it clashes with the // BodyConverters KEY - String autocompleteTemplate(ElasticsearchVersion version) { + String autocompleteTemplate(V version) { return beginTemplate(TYPE_AUTOCOMPLETE, version) + " },\n" + " \"mappings\": {\n" @@ -211,40 +196,84 @@ String autocompleteTemplate(ElasticsearchVersion version) { + endTemplate(version); } - IndexTemplates get(ElasticsearchVersion version) { - if (version.compareTo(V5_0) < 0 || version.compareTo(V9_0) >= 0) { - throw new IllegalArgumentException( - "Elasticsearch versions 5-8.x are supported, was: " + version); - } - return IndexTemplates.newBuilder() - .version(version) - .indexTypeDelimiter(indexTypeDelimiter(version)) - .span(spanIndexTemplate(version)) - .dependency(dependencyTemplate(version)) - .autocomplete(autocompleteTemplate(version)) - .build(); - } + /** + * Returns index pattern + * @param type type + * @param version distribution version + * @return index pattern + */ + abstract String indexPattern(String type, V version); - boolean useComposableTemplate(ElasticsearchVersion version) { - return (version.compareTo(V7_8) >= 0 && templatePriority != null); - } + /** + * Returns index templates + * @param version distribution version + * @return index templates + */ + abstract IndexTemplates get(V version); + + /** + * Should composable templates be used or not + * @param version distribution version + * @return {@code true} if composable templates should be used, + * {@code false} otherwise + */ + abstract boolean useComposableTemplate(V version); /** - * This returns a delimiter based on what's supported by the Elasticsearch version. - * - *

Starting in Elasticsearch 7.x, colons are no longer allowed in index names. This logic will - * make sure the pattern in our index template doesn't use them either. - * - *

See https://github.com/openzipkin/zipkin/issues/2219 + * Wraps the JSON payload if needed + * @param type type + * @param version distribution version + * @param json JSON payload + * @return wrapped JSON payload if needed */ - static char indexTypeDelimiter(ElasticsearchVersion version) { - return version.compareTo(V7_0) < 0 ? ':' : '-'; + abstract String maybeWrap(String type, V version, String json); + + /** + * Returns distribution specific templates (index templates URL, index + * type delimiter, {@link IndexTemplates}); + */ + abstract static class DistributionSpecificTemplates { + /** + * Returns distribution specific index templates URL + * @param indexPrefix index prefix + * @param type type + * @param templatePriority index template priority + * @return index templates URL + */ + abstract String indexTemplatesUrl(String indexPrefix, String type, @Nullable Integer templatePriority); + + /** + * Returns distribution specific index type delimiter + * @return index type delimiter + */ + abstract char indexTypeDelimiter(); + + /** + * Returns distribution specific index templates + * @param indexPrefix index prefix + * @param indexReplicas number of replicas + * @param indexShards number of shards + * @param searchEnabled search is enabled or disabled + * @param strictTraceId strict trace ID + * @param templatePriority index template priority + * @return index templates + */ + abstract IndexTemplates get(String indexPrefix, int indexReplicas, int indexShards, + boolean searchEnabled, boolean strictTraceId, Integer templatePriority); } - static String maybeWrap(String type, ElasticsearchVersion version, String json) { - // ES 7.x defaults include_type_name to false https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_literal_include_type_name_literal_now_defaults_to_literal_false_literal - if (version.compareTo(V7_0) >= 0) return json; - return " \"" + type + "\": {\n " + json.replace("\n", "\n ") + " }\n"; + /** + * Creates a new {@link DistributionSpecificTemplates} instance based on the distribution + * @param version distribution version + * @return {@link OpensearchSpecificTemplates} or {@link ElasticsearchSpecificTemplates} instance + */ + static DistributionSpecificTemplates forVersion(BaseVersion version) { + if (version instanceof ElasticsearchVersion) { + return new ElasticsearchSpecificTemplates.DistributionTemplate((ElasticsearchVersion) version); + } else if (version instanceof OpensearchVersion) { + return new OpensearchSpecificTemplates.DistributionTemplate((OpensearchVersion) version); + } else { + throw new IllegalArgumentException("The distribution version is not supported: " + version); + } } } - diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/BulkCallBuilder.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/BulkCallBuilder.java index 1c66a40eff..57236d2b4b 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/BulkCallBuilder.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/BulkCallBuilder.java @@ -27,13 +27,13 @@ import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.function.Supplier; + +import zipkin2.elasticsearch.BaseVersion; import zipkin2.elasticsearch.ElasticsearchStorage; -import zipkin2.elasticsearch.ElasticsearchVersion; import zipkin2.elasticsearch.internal.client.HttpCall; import zipkin2.elasticsearch.internal.client.HttpCall.BodyConverter; import static zipkin2.Call.propagateIfFatal; -import static zipkin2.elasticsearch.ElasticsearchVersion.V7_0; import static zipkin2.elasticsearch.internal.JsonSerializers.OBJECT_MAPPER; import static zipkin2.elasticsearch.internal.client.HttpCall.maybeRootCauseReason; @@ -79,9 +79,9 @@ public final class BulkCallBuilder { // Mutated for each call to index final List> entries = new ArrayList<>(); - public BulkCallBuilder(ElasticsearchStorage es, ElasticsearchVersion version, String tag) { + public BulkCallBuilder(ElasticsearchStorage es, BaseVersion version, String tag) { this.tag = tag; - shouldAddType = version.compareTo(V7_0) < 0; + shouldAddType = version.supportsTypes(); http = Internal.instance.http(es); pipeline = es.pipeline(); waitForRefresh = es.flushOnWrites(); diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchBaseExtension.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchBaseExtension.java new file mode 100644 index 0000000000..a8bb8fad3d --- /dev/null +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchBaseExtension.java @@ -0,0 +1,128 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package zipkin2.elasticsearch.integration; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientOptions; +import com.linecorp.armeria.client.ClientOptionsBuilder; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.client.WebClientBuilder; +import com.linecorp.armeria.client.logging.ContentPreviewingClient; +import com.linecorp.armeria.client.logging.LoggingClient; +import com.linecorp.armeria.client.logging.LoggingClientBuilder; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.logging.LogLevel; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import zipkin2.elasticsearch.ElasticsearchStorage; +import zipkin2.elasticsearch.ElasticsearchStorage.Builder; + +import static org.testcontainers.utility.DockerImageName.parse; +import static zipkin2.elasticsearch.integration.IgnoredDeprecationWarnings.IGNORE_THESE_WARNINGS; + +class ElasticsearchBaseExtension implements BeforeAllCallback, AfterAllCallback { + static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchBaseExtension.class); + + final GenericContainer container; + + ElasticsearchBaseExtension(GenericContainer container) { + this.container = container; + } + + @Override public void beforeAll(ExtensionContext context) { + if (context.getRequiredTestClass().getEnclosingClass() != null) { + // Only run once in outermost scope. + return; + } + + container.start(); + LOGGER.info("Using baseUrl {}", baseUrl()); + } + + @Override public void afterAll(ExtensionContext context) { + if (context.getRequiredTestClass().getEnclosingClass() != null) { + // Only run once in outermost scope. + return; + } + + container.stop(); + } + + Builder computeStorageBuilder() { + WebClientBuilder builder = WebClient.builder(baseUrl()) + // Elasticsearch 7 never returns a response when receiving an HTTP/2 preface instead of the + // more valid behavior of returning a bad request response, so we can't use the preface. + // + // TODO: find or raise a bug with Elastic + .factory(ClientFactory.builder().useHttp2Preface(false).build()); + builder.decorator((delegate, ctx, req) -> { + final HttpResponse response = delegate.execute(ctx, req); + return HttpResponse.from(response.aggregate().thenApply(r -> { + // ES will return a 'warning' response header when using deprecated api, detect this and + // fail early so we can do something about it. + // Example usage: https://github.com/elastic/elasticsearch/blob/3049e55f093487bb582a7e49ad624961415ba31c/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java#L559 + final String warningHeader = r.headers().get("warning"); + if (warningHeader != null) { + if (IGNORE_THESE_WARNINGS.stream().noneMatch(p -> p.matcher(warningHeader).find())) { + throw new IllegalArgumentException("Detected usage of deprecated API for request " + + req + ":\n" + warningHeader); + } + } + // Convert AggregatedHttpResponse back to HttpResponse. + return r.toHttpResponse(); + })); + }); + + // When ES_DEBUG=true log full headers, request and response body to the category + // com.linecorp.armeria.client.logging + if (Boolean.parseBoolean(System.getenv("ES_DEBUG"))) { + ClientOptionsBuilder options = ClientOptions.builder(); + LoggingClientBuilder loggingBuilder = LoggingClient.builder() + .requestLogLevel(LogLevel.INFO) + .successfulResponseLogLevel(LogLevel.INFO); + options.decorator(loggingBuilder.newDecorator()); + options.decorator(ContentPreviewingClient.newDecorator(Integer.MAX_VALUE)); + builder.options(options.build()); + } + + WebClient client = builder.build(); + return ElasticsearchStorage.newBuilder(new ElasticsearchStorage.LazyHttpClient() { + @Override public WebClient get() { + return client; + } + + @Override public void close() { + client.endpointGroup().close(); + } + + @Override public String toString() { + return client.uri().toString(); + } + }).index("zipkin-test").flushOnWrites(true); + } + + String baseUrl() { + return "http://" + container.getHost() + ":" + container.getMappedPort(9200); + } + + static String index(TestInfo testInfo) { + String result; + if (testInfo.getTestMethod().isPresent()) { + result = testInfo.getTestMethod().get().getName(); + } else { + assert testInfo.getTestClass().isPresent(); + result = testInfo.getTestClass().get().getSimpleName(); + } + result = result.toLowerCase(); + return result.length() <= 48 ? result : result.substring(result.length() - 48); + } +} diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchExtension.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchExtension.java index 15b405d81f..23f8c5d7c9 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchExtension.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchExtension.java @@ -4,114 +4,19 @@ */ package zipkin2.elasticsearch.integration; -import com.linecorp.armeria.client.ClientFactory; -import com.linecorp.armeria.client.ClientOptions; -import com.linecorp.armeria.client.ClientOptionsBuilder; -import com.linecorp.armeria.client.WebClient; -import com.linecorp.armeria.client.WebClientBuilder; -import com.linecorp.armeria.client.logging.ContentPreviewingClient; -import com.linecorp.armeria.client.logging.LoggingClient; -import com.linecorp.armeria.client.logging.LoggingClientBuilder; -import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.logging.LogLevel; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; -import zipkin2.elasticsearch.ElasticsearchStorage; -import zipkin2.elasticsearch.ElasticsearchStorage.Builder; import static org.testcontainers.utility.DockerImageName.parse; -import static zipkin2.elasticsearch.integration.IgnoredDeprecationWarnings.IGNORE_THESE_WARNINGS; -class ElasticsearchExtension implements BeforeAllCallback, AfterAllCallback { +class ElasticsearchExtension extends ElasticsearchBaseExtension { static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchExtension.class); - final ElasticsearchContainer container; - ElasticsearchExtension(int majorVersion) { - container = new ElasticsearchContainer(majorVersion); - } - - @Override public void beforeAll(ExtensionContext context) { - if (context.getRequiredTestClass().getEnclosingClass() != null) { - // Only run once in outermost scope. - return; - } - - container.start(); - LOGGER.info("Using baseUrl {}", baseUrl()); - } - - @Override public void afterAll(ExtensionContext context) { - if (context.getRequiredTestClass().getEnclosingClass() != null) { - // Only run once in outermost scope. - return; - } - - container.stop(); - } - - Builder computeStorageBuilder() { - WebClientBuilder builder = WebClient.builder(baseUrl()) - // Elasticsearch 7 never returns a response when receiving an HTTP/2 preface instead of the - // more valid behavior of returning a bad request response, so we can't use the preface. - // - // TODO: find or raise a bug with Elastic - .factory(ClientFactory.builder().useHttp2Preface(false).build()); - builder.decorator((delegate, ctx, req) -> { - final HttpResponse response = delegate.execute(ctx, req); - return HttpResponse.from(response.aggregate().thenApply(r -> { - // ES will return a 'warning' response header when using deprecated api, detect this and - // fail early so we can do something about it. - // Example usage: https://github.com/elastic/elasticsearch/blob/3049e55f093487bb582a7e49ad624961415ba31c/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java#L559 - final String warningHeader = r.headers().get("warning"); - if (warningHeader != null) { - if (IGNORE_THESE_WARNINGS.stream().noneMatch(p -> p.matcher(warningHeader).find())) { - throw new IllegalArgumentException("Detected usage of deprecated API for request " - + req + ":\n" + warningHeader); - } - } - // Convert AggregatedHttpResponse back to HttpResponse. - return r.toHttpResponse(); - })); - }); - - // When ES_DEBUG=true log full headers, request and response body to the category - // com.linecorp.armeria.client.logging - if (Boolean.parseBoolean(System.getenv("ES_DEBUG"))) { - ClientOptionsBuilder options = ClientOptions.builder(); - LoggingClientBuilder loggingBuilder = LoggingClient.builder() - .requestLogLevel(LogLevel.INFO) - .successfulResponseLogLevel(LogLevel.INFO); - options.decorator(loggingBuilder.newDecorator()); - options.decorator(ContentPreviewingClient.newDecorator(Integer.MAX_VALUE)); - builder.options(options.build()); - } - - WebClient client = builder.build(); - return ElasticsearchStorage.newBuilder(new ElasticsearchStorage.LazyHttpClient() { - @Override public WebClient get() { - return client; - } - - @Override public void close() { - client.endpointGroup().close(); - } - - @Override public String toString() { - return client.uri().toString(); - } - }).index("zipkin-test").flushOnWrites(true); - } - - String baseUrl() { - return "http://" + container.getHost() + ":" + container.getMappedPort(9200); + super(new ElasticsearchContainer(majorVersion)); } // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537 @@ -123,16 +28,4 @@ static final class ElasticsearchContainer extends GenericContainer { diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITOpenSearchStorageV2.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITOpenSearchStorageV2.java new file mode 100644 index 0000000000..c4265c9647 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITOpenSearchStorageV2.java @@ -0,0 +1,32 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package zipkin2.elasticsearch.integration; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.RegisterExtension; +import zipkin2.elasticsearch.ElasticsearchStorage; + +import static zipkin2.elasticsearch.integration.ElasticsearchExtension.index; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Tag("docker") +class ITOpenSearchStorageV2 extends ITElasticsearchStorage { + + @RegisterExtension static OpenSearchExtension opensearch = new OpenSearchExtension(2); + + @Override OpenSearchExtension elasticsearch() { + return opensearch; + } + + @Nested + class ITEnsureIndexTemplate extends zipkin2.elasticsearch.integration.ITEnsureIndexTemplate { + @Override protected ElasticsearchStorage.Builder newStorageBuilder(TestInfo testInfo) { + return elasticsearch().computeStorageBuilder().index(index(testInfo)); + } + } +} diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/OpenSearchExtension.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/OpenSearchExtension.java new file mode 100644 index 0000000000..6066ef695f --- /dev/null +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/OpenSearchExtension.java @@ -0,0 +1,31 @@ +/* + * Copyright The OpenZipkin Authors + * SPDX-License-Identifier: Apache-2.0 + */ +package zipkin2.elasticsearch.integration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; + +import static org.testcontainers.utility.DockerImageName.parse; + +class OpenSearchExtension extends ElasticsearchBaseExtension { + static final Logger LOGGER = LoggerFactory.getLogger(OpenSearchExtension.class); + + OpenSearchExtension(int majorVersion) { + super(new OpenSearchContainer(majorVersion)); + } + + // mostly waiting for https://github.com/testcontainers/testcontainers-java/issues/3537 + static final class OpenSearchContainer extends GenericContainer { + OpenSearchContainer(int majorVersion) { + super(parse("ghcr.io/openzipkin/zipkin-opensearch" + majorVersion + ":3.3.1")); + addExposedPort(9200); + waitStrategy = Wait.forHealthcheck(); + withLogConsumer(new Slf4jLogConsumer(LOGGER)); + } + } +}