Maintenance: refactoring existing e2e tests on trace data ingestion (o…

…pensearch-project#512) * REF: refactoring existing trace e2e tests Signed-off-by: qchea <[email protected]> * MAINT: github workflow Signed-off-by: qchea <[email protected]> * ADD: README Signed-off-by: qchea <[email protected]> * FIX: spotless Signed-off-by: qchea <[email protected]> * STY: spotless for markdown Signed-off-by: qchea <[email protected]> * MAINT: address PR comments Signed-off-by: qchea <[email protected]> * MAINT: directory name Signed-off-by: qchea <[email protected]> * MAINT: build directory reference Signed-off-by: qchea <[email protected]>
asifsmohammed · Nov 3, 2021 · c4e2d71 · c4e2d71
1 parent 3c7ceec
commit c4e2d71
Show file tree

Hide file tree

Showing 16 changed files with 123 additions and 80 deletions.
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml
@@ -27,4 +27,4 @@ jobs:
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
     - name: Run raw-span compatibility end-to-end tests with Gradle
-      run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest
+      run: ./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -27,4 +27,4 @@ jobs:
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Run raw-span end-to-end tests with Gradle
-        run: ./gradlew :data-prepper-core:rawSpanEndToEndTest
+        run: ./gradlew :e2e-test:trace:rawSpanEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -27,4 +27,4 @@ jobs:
       - name: Grant execute permission for gradlew
         run: chmod +x gradlew
       - name: Run service-map end-to-end tests with Gradle
-        run: ./gradlew :data-prepper-core:serviceMapEndToEndTest
+        run: ./gradlew :e2e-test:trace:serviceMapEndToEndTest
diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle
@@ -17,8 +17,6 @@ sourceSets {
     }
 }
 
-apply from: "integrationTest.gradle"
-
 dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins')

diff --git a/e2e-test/README.md b/e2e-test/README.md
@@ -0,0 +1,3 @@
+# Data Prepper End-to-end Tests
+
+This module includes all e2e tests for data-prepper.
diff --git a/e2e-test/build.gradle b/e2e-test/build.gradle
@@ -0,0 +1,40 @@
+subprojects {
+    buildscript {
+        repositories {
+            maven {
+                url "https://plugins.gradle.org/m2/"
+            }
+        }
+        dependencies {
+            classpath 'com.bmuschko:gradle-docker-plugin:7.0.0'
+        }
+    }
+
+    sourceSets {
+        integrationTest {
+            java {
+                compileClasspath += main.output + test.output
+                runtimeClasspath += main.output + test.output
+                srcDir file('src/integrationTest/java')
+            }
+            resources.srcDir file('src/integrationTest/resources')
+        }
+    }
+
+    configurations {
+        integrationTestImplementation.extendsFrom testImplementation
+        integrationTestRuntime.extendsFrom testRuntime
+    }
+
+    def DATA_PREPPER_CORE_JAR = project(':data-prepper-core').jar
+
+    task copyDataPrepperJar(type: Copy) {
+        dependsOn DATA_PREPPER_CORE_JAR
+        from("${DATA_PREPPER_CORE_JAR.archivePath}")
+        into "${project.buildDir}/bin"
+    }
+
+    ext {
+        dataPrepperJarFilepath = "${project.buildDir.name}/bin/${DATA_PREPPER_CORE_JAR.archiveName}"
+    }
+}
diff --git a/e2e-test/trace/README.md b/e2e-test/trace/README.md
@@ -0,0 +1,36 @@
+# Trace Data Ingestion End-to-end Tests
+
+This module includes e2e tests for trace data ingestion supported by data-prepper.
+
+## Raw Span Ingestion Pipeline End-to-end test
+
+Run from current directory
+```
+./gradlew :rawSpanEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:rawSpanEndToEndTest
+```
+
+## Raw Span Ingestion Pipelines Compatibility End-to-end test
+
+Run from current directory
+```
+./gradlew :rawSpanCompatibilityEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest
+```
+
+## Service Map Ingestion Pipelines End-to-end test
+
+Run from current directory
+```
+./gradlew :serviceMapEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:serviceMapEndToEndTest
+```
diff --git a/data-prepper-core/integrationTest.gradle → e2e-test/trace/build.gradle b/data-prepper-core/integrationTest.gradle → e2e-test/trace/build.gradle
@@ -1,59 +1,11 @@
-buildscript {
-    repositories {
-        maven {
-            url "https://plugins.gradle.org/m2/"
-        }
-    }
-    dependencies {
-        classpath 'com.bmuschko:gradle-docker-plugin:7.0.0'
-    }
-}
+apply plugin: DockerRemoteApiPlugin
 
-apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin
-
-/*
- *  SPDX-License-Identifier: Apache-2.0
- *
- *  The OpenSearch Contributors require contributions made to
- *  this file be licensed under the Apache-2.0 license or a
- *  compatible open source license.
- *
- *  Modifications Copyright OpenSearch Contributors. See
- *  GitHub history for details.
- */
 
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
 import com.bmuschko.gradle.docker.tasks.container.*
 import com.bmuschko.gradle.docker.tasks.image.*
 import com.bmuschko.gradle.docker.tasks.network.*
 
-sourceSets {
-    integrationTest {
-        java {
-            compileClasspath += main.output + test.output
-            runtimeClasspath += main.output + test.output
-            srcDir file('src/integrationTest/java')
-        }
-        resources.srcDir file('src/integrationTest/resources')
-    }
-}
-
-configurations {
-    integrationTestImplementation.extendsFrom testImplementation
-    integrationTestRuntime.extendsFrom testRuntime
-}
-
-dependencies {
-    integrationTestCompile project(':data-prepper-plugins:opensearch')
-    integrationTestCompile project(':data-prepper-plugins:otel-trace-group-prepper')
-    integrationTestImplementation "org.awaitility:awaitility:4.0.3"
-    integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}"
-    integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.13.0'
-    integrationTestImplementation "com.linecorp.armeria:armeria:1.0.0"
-    integrationTestImplementation "com.linecorp.armeria:armeria-grpc:1.0.0"
-    integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${versionMap.opensearchVersion}"
-    integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind'
-}
-
 /**
  * End-to-end test docker network
  */
@@ -74,13 +26,13 @@ def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml"
  * DataPrepper Docker tasks
  */
 task createDataPrepperDockerFile(type: Dockerfile) {
-    dependsOn jar
+    dependsOn copyDataPrepperJar
     destFile = project.file('build/docker/Dockerfile')
     from("adoptopenjdk/openjdk14:jre-14.0.1_7-alpine")
     exposePort(21890)
     exposePort(4900)
     workingDir("/app")
-    copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar")
+    copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar")
     copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}")
     copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
     copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}")
@@ -209,7 +161,7 @@ task rawSpanEndToEndTest(type: Test) {
     classpath = sourceSets.integrationTest.runtimeClasspath
 
     filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*"
+        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
     }
 
     finalizedBy stopOpenSearchDockerContainer
@@ -246,7 +198,7 @@ task rawSpanCompatibilityEndToEndTest(type: Test) {
     classpath = sourceSets.integrationTest.runtimeClasspath
 
     filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*"
+        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
     }
 
     finalizedBy stopOpenSearchDockerContainer
@@ -283,7 +235,7 @@ task serviceMapEndToEndTest(type: Test) {
     classpath = sourceSets.integrationTest.runtimeClasspath
 
     filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.EndToEndServiceMapTest*"
+        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*"
     }
 
     finalizedBy stopOpenSearchDockerContainer
@@ -295,3 +247,15 @@ task serviceMapEndToEndTest(type: Test) {
     finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
     finalizedBy removeDataPrepperNetwork
 }
+
+dependencies {
+    integrationTestCompile project(':data-prepper-plugins:opensearch')
+    integrationTestCompile project(':data-prepper-plugins:otel-trace-group-prepper')
+    integrationTestImplementation "org.awaitility:awaitility:4.0.3"
+    integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}"
+    integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.13.0'
+    integrationTestImplementation "com.linecorp.armeria:armeria:1.0.0"
+    integrationTestImplementation "com.linecorp.armeria:armeria-grpc:1.0.0"
+    integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${versionMap.opensearchVersion}"
+    integrationTestImplementation 'com.fasterxml.jackson.core:jackson-databind'
+}
diff --git a/...pper/integration/EndToEndRawSpanTest.java → ...ntegration/trace/EndToEndRawSpanTest.java b/...pper/integration/EndToEndRawSpanTest.java → ...ntegration/trace/EndToEndRawSpanTest.java
@@ -9,7 +9,7 @@
  *  GitHub history for details.
  */
 
-package com.amazon.dataprepper.integration;
+package com.amazon.dataprepper.integration.trace;
 
 
 import com.amazon.dataprepper.plugins.prepper.oteltracegroup.model.TraceGroup;
@@ -30,15 +30,15 @@
 import io.opentelemetry.proto.trace.v1.ResourceSpans;
 import io.opentelemetry.proto.trace.v1.Span;
 import io.opentelemetry.proto.trace.v1.Status;
+import org.junit.Assert;
+import org.junit.Test;
 import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.search.SearchHits;
 import org.opensearch.search.builder.SearchSourceBuilder;
-import org.junit.Assert;
-import org.junit.Test;
 
 import java.io.IOException;
 import java.time.Instant;

diff --git a/...r/integration/EndToEndServiceMapTest.java → ...gration/trace/EndToEndServiceMapTest.java b/...r/integration/EndToEndServiceMapTest.java → ...gration/trace/EndToEndServiceMapTest.java
@@ -9,23 +9,11 @@
  *  GitHub history for details.
  */
 
-package com.amazon.dataprepper.integration;
+package com.amazon.dataprepper.integration.trace;
 
 
-import com.google.protobuf.ByteString;
 import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
+import com.google.protobuf.ByteString;
 import com.linecorp.armeria.client.Clients;
 import com.linecorp.armeria.client.retry.RetryRule;
 import com.linecorp.armeria.client.retry.RetryingClient;
@@ -37,15 +25,27 @@
 import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans;
 import io.opentelemetry.proto.trace.v1.ResourceSpans;
 import io.opentelemetry.proto.trace.v1.Span;
+import org.junit.Assert;
+import org.junit.Test;
 import org.opensearch.action.admin.indices.refresh.RefreshRequest;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.search.SearchHits;
 import org.opensearch.search.builder.SearchSourceBuilder;
-import org.junit.Assert;
-import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.awaitility.Awaitility.await;
 

diff --git a/...prepper/integration/EndToEndTestSpan.java → ...r/integration/trace/EndToEndTestSpan.java b/...prepper/integration/EndToEndTestSpan.java → ...r/integration/trace/EndToEndTestSpan.java
@@ -1,4 +1,4 @@
-package com.amazon.dataprepper.integration;
+package com.amazon.dataprepper.integration.trace;
 
 import io.opentelemetry.proto.trace.v1.Span;
 

diff --git a/...ntegrationTest/resources/data_prepper.yml → ...ntegrationTest/resources/data_prepper.yml b/...ntegrationTest/resources/data_prepper.yml → ...ntegrationTest/resources/data_prepper.yml
diff --git a/.../raw-span-e2e-pipeline-latest-release.yml → .../raw-span-e2e-pipeline-latest-release.yml b/.../raw-span-e2e-pipeline-latest-release.yml → .../raw-span-e2e-pipeline-latest-release.yml
diff --git a/...nTest/resources/raw-span-e2e-pipeline.yml → ...nTest/resources/raw-span-e2e-pipeline.yml b/...nTest/resources/raw-span-e2e-pipeline.yml → ...nTest/resources/raw-span-e2e-pipeline.yml
diff --git a/...st/resources/service-map-e2e-pipeline.yml → ...st/resources/service-map-e2e-pipeline.yml b/...st/resources/service-map-e2e-pipeline.yml → ...st/resources/service-map-e2e-pipeline.yml
diff --git a/settings.gradle b/settings.gradle
@@ -38,4 +38,6 @@ include 'data-prepper-plugins:blocking-buffer'
 include 'data-prepper-plugins:http-source'
 include 'data-prepper-plugins:grok-prepper'
 include 'data-prepper-logstash-configuration'
+include 'e2e-test'
+include 'e2e-test:trace'
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Data Prepper End-to-end Tests

		This module includes all e2e tests for data-prepper.