From c4e2d717fb04d51802f1bc9c5b0615cf52c9c523 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 2 Nov 2021 20:28:06 -0500 Subject: [PATCH] Maintenance: refactoring existing e2e tests on trace data ingestion (#512) * REF: refactoring existing trace e2e tests Signed-off-by: qchea * MAINT: github workflow Signed-off-by: qchea * ADD: README Signed-off-by: qchea * FIX: spotless Signed-off-by: qchea * STY: spotless for markdown Signed-off-by: qchea * MAINT: address PR comments Signed-off-by: qchea * MAINT: directory name Signed-off-by: qchea * MAINT: build directory reference Signed-off-by: qchea --- ...ytics-raw-span-compatibility-e2e-tests.yml | 2 +- ...per-trace-analytics-raw-span-e2e-tests.yml | 2 +- ...-trace-analytics-service-map-e2e-tests.yml | 2 +- data-prepper-core/build.gradle | 2 - e2e-test/README.md | 3 + e2e-test/build.gradle | 40 ++++++++++ e2e-test/trace/README.md | 36 +++++++++ .../trace/build.gradle | 74 +++++-------------- .../trace}/EndToEndRawSpanTest.java | 6 +- .../trace}/EndToEndServiceMapTest.java | 32 ++++---- .../integration/trace}/EndToEndTestSpan.java | 2 +- .../resources/data_prepper.yml | 0 .../raw-span-e2e-pipeline-latest-release.yml | 0 .../resources/raw-span-e2e-pipeline.yml | 0 .../resources/service-map-e2e-pipeline.yml | 0 settings.gradle | 2 + 16 files changed, 123 insertions(+), 80 deletions(-) create mode 100644 e2e-test/README.md create mode 100644 e2e-test/build.gradle create mode 100644 e2e-test/trace/README.md rename data-prepper-core/integrationTest.gradle => e2e-test/trace/build.gradle (90%) rename {data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration => e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace}/EndToEndRawSpanTest.java (99%) rename {data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration => e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace}/EndToEndServiceMapTest.java (99%) rename {data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration => e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace}/EndToEndTestSpan.java (98%) rename {data-prepper-core => e2e-test/trace}/src/integrationTest/resources/data_prepper.yml (100%) rename {data-prepper-core => e2e-test/trace}/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml (100%) rename {data-prepper-core => e2e-test/trace}/src/integrationTest/resources/raw-span-e2e-pipeline.yml (100%) rename {data-prepper-core => e2e-test/trace}/src/integrationTest/resources/service-map-e2e-pipeline.yml (100%) diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index a73e1e7c08..97c13c01ed 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -27,4 +27,4 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Run raw-span compatibility end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest + run: ./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml index 02c631d7b7..c88f278541 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -27,4 +27,4 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Run raw-span end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:rawSpanEndToEndTest \ No newline at end of file + run: ./gradlew :e2e-test:trace:rawSpanEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml index 272f98cafa..3c8c665ecc 100644 --- a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -27,4 +27,4 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Run service-map end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:serviceMapEndToEndTest \ No newline at end of file + run: ./gradlew :e2e-test:trace:serviceMapEndToEndTest \ No newline at end of file diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 2b0c7ca4f8..3741f3a747 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -17,8 +17,6 @@ sourceSets { } } -apply from: "integrationTest.gradle" - dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins') diff --git a/e2e-test/README.md b/e2e-test/README.md new file mode 100644 index 0000000000..9e7646ff02 --- /dev/null +++ b/e2e-test/README.md @@ -0,0 +1,3 @@ +# Data Prepper End-to-end Tests + +This module includes all e2e tests for data-prepper. diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle new file mode 100644 index 0000000000..57da98961a --- /dev/null +++ b/e2e-test/build.gradle @@ -0,0 +1,40 @@ +subprojects { + buildscript { + repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + } + dependencies { + classpath 'com.bmuschko:gradle-docker-plugin:7.0.0' + } + } + + sourceSets { + integrationTest { + java { + compileClasspath += main.output + test.output + runtimeClasspath += main.output + test.output + srcDir file('src/integrationTest/java') + } + resources.srcDir file('src/integrationTest/resources') + } + } + + configurations { + integrationTestImplementation.extendsFrom testImplementation + integrationTestRuntime.extendsFrom testRuntime + } + + def DATA_PREPPER_CORE_JAR = project(':data-prepper-core').jar + + task copyDataPrepperJar(type: Copy) { + dependsOn DATA_PREPPER_CORE_JAR + from("${DATA_PREPPER_CORE_JAR.archivePath}") + into "${project.buildDir}/bin" + } + + ext { + dataPrepperJarFilepath = "${project.buildDir.name}/bin/${DATA_PREPPER_CORE_JAR.archiveName}" + } +} \ No newline at end of file diff --git a/e2e-test/trace/README.md b/e2e-test/trace/README.md new file mode 100644 index 0000000000..f8b30ba74f --- /dev/null +++ b/e2e-test/trace/README.md @@ -0,0 +1,36 @@ +# Trace Data Ingestion End-to-end Tests + +This module includes e2e tests for trace data ingestion supported by data-prepper. + +## Raw Span Ingestion Pipeline End-to-end test + +Run from current directory +``` +./gradlew :rawSpanEndToEndTest +``` +or from project root directory +``` +./gradlew :e2e-test:trace:rawSpanEndToEndTest +``` + +## Raw Span Ingestion Pipelines Compatibility End-to-end test + +Run from current directory +``` +./gradlew :rawSpanCompatibilityEndToEndTest +``` +or from project root directory +``` +./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest +``` + +## Service Map Ingestion Pipelines End-to-end test + +Run from current directory +``` +./gradlew :serviceMapEndToEndTest +``` +or from project root directory +``` +./gradlew :e2e-test:trace:serviceMapEndToEndTest +``` diff --git a/data-prepper-core/integrationTest.gradle b/e2e-test/trace/build.gradle similarity index 90% rename from data-prepper-core/integrationTest.gradle rename to e2e-test/trace/build.gradle index 47549a5cc7..eb7631c654 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/e2e-test/trace/build.gradle @@ -1,59 +1,11 @@ -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } - dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:7.0.0' - } -} +apply plugin: DockerRemoteApiPlugin -apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin - -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin import com.bmuschko.gradle.docker.tasks.container.* import com.bmuschko.gradle.docker.tasks.image.* import com.bmuschko.gradle.docker.tasks.network.* -sourceSets { - integrationTest { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/integrationTest/java') - } - resources.srcDir file('src/integrationTest/resources') - } -} - -configurations { - integrationTestImplementation.extendsFrom testImplementation - integrationTestRuntime.extendsFrom testRuntime -} - -dependencies { - integrationTestCompile project(':data-prepper-plugins:opensearch') - integrationTestCompile project(':data-prepper-plugins:otel-trace-group-prepper') - integrationTestImplementation "org.awaitility:awaitility:4.0.3" - integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}" - integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.13.0' - integrationTestImplementation "com.linecorp.armeria:armeria:1.0.0" - integrationTestImplementation "com.linecorp.armeria:armeria-grpc:1.0.0" - integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${versionMap.opensearchVersion}" - integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind' -} - /** * End-to-end test docker network */ @@ -74,13 +26,13 @@ def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" * DataPrepper Docker tasks */ task createDataPrepperDockerFile(type: Dockerfile) { - dependsOn jar + dependsOn copyDataPrepperJar destFile = project.file('build/docker/Dockerfile') from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine") exposePort(21890) exposePort(4900) workingDir("/app") - copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar") + copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") @@ -209,7 +161,7 @@ task rawSpanEndToEndTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" + includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" } finalizedBy stopOpenSearchDockerContainer @@ -246,7 +198,7 @@ task rawSpanCompatibilityEndToEndTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" + includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" } finalizedBy stopOpenSearchDockerContainer @@ -283,7 +235,7 @@ task serviceMapEndToEndTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching "com.amazon.dataprepper.integration.EndToEndServiceMapTest*" + includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*" } finalizedBy stopOpenSearchDockerContainer @@ -295,3 +247,15 @@ task serviceMapEndToEndTest(type: Test) { finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) finalizedBy removeDataPrepperNetwork } + +dependencies { + integrationTestCompile project(':data-prepper-plugins:opensearch') + integrationTestCompile project(':data-prepper-plugins:otel-trace-group-prepper') + integrationTestImplementation "org.awaitility:awaitility:4.0.3" + integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}" + integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.13.0' + integrationTestImplementation "com.linecorp.armeria:armeria:1.0.0" + integrationTestImplementation "com.linecorp.armeria:armeria-grpc:1.0.0" + integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${versionMap.opensearchVersion}" + integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind' +} \ No newline at end of file diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndRawSpanTest.java similarity index 99% rename from data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java rename to e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndRawSpanTest.java index ddde549aa6..8867e2d90c 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package com.amazon.dataprepper.integration; +package com.amazon.dataprepper.integration.trace; import com.amazon.dataprepper.plugins.prepper.oteltracegroup.model.TraceGroup; @@ -30,6 +30,8 @@ import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; +import org.junit.Assert; +import org.junit.Test; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -37,8 +39,6 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; -import org.junit.Assert; -import org.junit.Test; import java.io.IOException; import java.time.Instant; diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndServiceMapTest.java similarity index 99% rename from data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java rename to e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndServiceMapTest.java index 97a3c9b475..b545416ef6 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java +++ b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndServiceMapTest.java @@ -9,23 +9,11 @@ * GitHub history for details. */ -package com.amazon.dataprepper.integration; +package com.amazon.dataprepper.integration.trace; -import com.google.protobuf.ByteString; import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - +import com.google.protobuf.ByteString; import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.retry.RetryRule; import com.linecorp.armeria.client.retry.RetryingClient; @@ -37,6 +25,8 @@ import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; +import org.junit.Assert; +import org.junit.Test; import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -44,8 +34,18 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; -import org.junit.Assert; -import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.awaitility.Awaitility.await; diff --git a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndTestSpan.java similarity index 98% rename from data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java rename to e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndTestSpan.java index 644f9441c0..152b9ea00c 100644 --- a/data-prepper-core/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndTestSpan.java +++ b/e2e-test/trace/src/integrationTest/java/com/amazon/dataprepper/integration/trace/EndToEndTestSpan.java @@ -1,4 +1,4 @@ -package com.amazon.dataprepper.integration; +package com.amazon.dataprepper.integration.trace; import io.opentelemetry.proto.trace.v1.Span; diff --git a/data-prepper-core/src/integrationTest/resources/data_prepper.yml b/e2e-test/trace/src/integrationTest/resources/data_prepper.yml similarity index 100% rename from data-prepper-core/src/integrationTest/resources/data_prepper.yml rename to e2e-test/trace/src/integrationTest/resources/data_prepper.yml diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml similarity index 100% rename from data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml rename to e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml similarity index 100% rename from data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml rename to e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml diff --git a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml similarity index 100% rename from data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml rename to e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml diff --git a/settings.gradle b/settings.gradle index d697e75ac9..9aa3c2b7a2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -38,4 +38,6 @@ include 'data-prepper-plugins:blocking-buffer' include 'data-prepper-plugins:http-source' include 'data-prepper-plugins:grok-prepper' include 'data-prepper-logstash-configuration' +include 'e2e-test' +include 'e2e-test:trace'