TST: trace event migration backward compatibility e2e tests (opensear…

…ch-project#1264) * MAINT: additional 3 e2e tests Signed-off-by: Qi Chen <[email protected]> * REF: e2e tests task definition and README Signed-off-by: Qi Chen <[email protected]> * MAINT: github workflow Signed-off-by: Qi Chen <[email protected]> * STY: spotless Signed-off-by: Qi Chen <[email protected]> * FIX: window_duration Signed-off-by: Qi Chen <[email protected]> * CLN: PR comments Signed-off-by: Qi Chen <[email protected]>
asifsmohammed · Apr 7, 2022 · de15772 · de15772
1 parent 178901f
commit de15772
Show file tree

Hide file tree

Showing 13 changed files with 270 additions and 146 deletions.
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml
@@ -24,5 +24,8 @@ jobs:
         java-version: ${{ matrix.java }}
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
-    - name: Run raw-span compatibility end-to-end tests with Gradle
-      run: ./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest
+      # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+    - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle
+      run: ./gradlew :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest
+    - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle
+      run: ./gradlew :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -24,5 +24,8 @@ jobs:
           java-version: ${{ matrix.java }}
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
-      - name: Run raw-span end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:rawSpanEndToEndTest
+        # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+      - name: Run raw-span OTLP record type end-to-end tests with Gradle
+        run: ./gradlew :e2e-test:trace:rawSpanOTLPEndToEndTest
+      - name: Run raw-span OTLP and Event end-to-end tests with Gradle
+        run: ./gradlew :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -24,5 +24,8 @@ jobs:
           java-version: ${{ matrix.java }}
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
-      - name: Run service-map end-to-end tests with Gradle
-        run: ./gradlew :e2e-test:trace:serviceMapEndToEndTest
+        # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+      - name: Run service-map OTLP record type end-to-end tests with Gradle
+        run: ./gradlew :e2e-test:trace:serviceMapOTLPEndToEndTest
+      - name: Run service-map OTLP and Event end-to-end tests with Gradle
+        run: ./gradlew :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest
diff --git a/e2e-test/trace/README.md b/e2e-test/trace/README.md
@@ -4,33 +4,72 @@ This module includes e2e tests for trace data ingestion supported by data-preppe
 
 ## Raw Span Ingestion Pipeline End-to-end test
 
+### OTLP record type
+
+Run from current directory
+```
+./gradlew :rawSpanOTLPEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:rawSpanOTLPEndToEndTest
+```
+
+### Event record type compatibility with OTLP record type
+
 Run from current directory
 ```
-./gradlew :rawSpanEndToEndTest
+./gradlew :rawSpanOTLPAndEventEndToEndTest
 ```
 or from project root directory
 ```
-./gradlew :e2e-test:trace:rawSpanEndToEndTest
+./gradlew :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest
 ```
 
-## Raw Span Ingestion Pipelines Compatibility End-to-end test
+## Raw Span Ingestion Pipelines Latest Release Compatibility End-to-end test
+
+### OTLP record type compatibility with latest release
 
 Run from current directory
 ```
-./gradlew :rawSpanCompatibilityEndToEndTest
+./gradlew :rawSpanOTLPLatestReleaseCompatibilityEndToEndTest
 ```
 or from project root directory
 ```
-./gradlew :e2e-test:trace:rawSpanCompatibilityEndToEndTest
+./gradlew :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest
+```
+
+### Event record type compatibility with latest release
+
+Run from current directory
+```
+./gradlew :rawSpanEventLatestReleaseCompatibilityEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest
 ```
 
 ## Service Map Ingestion Pipelines End-to-end test
 
+### OTLP record type
+
+Run from current directory
+```
+./gradlew :serviceMapOTLPEndToEndTest
+```
+or from project root directory
+```
+./gradlew :e2e-test:trace:serviceMapOTLPEndToEndTest
+```
+
+### Event record type compatibility with OTLP record type
+
 Run from current directory
 ```
-./gradlew :serviceMapEndToEndTest
+./gradlew :serviceMapOTLPAndEventEndToEndTest
 ```
 or from project root directory
 ```
-./gradlew :e2e-test:trace:serviceMapEndToEndTest
+./gradlew :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest
 ```
diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle
@@ -30,8 +30,11 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) {
 }
 
 def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml"
+def RAW_SPAN_PIPELINE_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-event-type.yml"
+def RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-peer-forwarder-event-type.yml"
 def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml"
 def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml"
+def SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML = "service-map-e2e-pipeline-event-type.yml"
 
 /**
  * DataPrepper Docker tasks
@@ -45,8 +48,11 @@ task createDataPrepperDockerFile(type: Dockerfile) {
     workingDir("/app")
     copyFile("${dataPrepperJarFilepath}", "/app/data-prepper.jar")
     copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}")
+    copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}")
+    copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}")
     copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
     copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}")
+    copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}", "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}")
     copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml")
     defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml")
 }
@@ -148,120 +154,104 @@ task stopOpenSearchDockerContainer(type: DockerStopContainer) {
  * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test
  * Stops the docker containers when finished
  */
-task rawSpanEndToEndTest(type: Test) {
-    dependsOn build
-    dependsOn startOpenSearchDockerContainer
-    def createDataPrepper1Task = createDataPrepperDockerContainer(
-            "rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
-    def createDataPrepper2Task = createDataPrepperDockerContainer(
-            "rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
-    def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
-    def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
-    dependsOn startDataPrepper1Task
-    dependsOn startDataPrepper2Task
-    startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
-    startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
-    // wait for data-preppers to be ready
-    doFirst {
-        sleep(10*1000)
+def createEndToEndTest(final String testName, final String includeTestsMatchPattern,
+                       final DockerCreateContainer createDataPrepper1Task, final DockerCreateContainer createDataPrepper2Task) {
+    return tasks.create(testName, Test) {
+        dependsOn build
+        dependsOn startOpenSearchDockerContainer
+        def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
+        def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
+        dependsOn startDataPrepper1Task
+        dependsOn startDataPrepper2Task
+        startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
+        startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
+        // wait for data-preppers to be ready
+        doFirst {
+            sleep(10*1000)
+        }
+
+        description = 'Runs the raw span integration tests.'
+        group = 'verification'
+        testClassesDirs = sourceSets.integrationTest.output.classesDirs
+        classpath = sourceSets.integrationTest.runtimeClasspath
+
+        filter {
+            includeTestsMatching includeTestsMatchPattern
+        }
+
+        finalizedBy stopOpenSearchDockerContainer
+        def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
+        def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
+        finalizedBy stopDataPrepper1Task
+        finalizedBy stopDataPrepper2Task
+        finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
+        finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
+        finalizedBy removeDataPrepperNetwork
     }
+}
 
-    description = 'Runs the raw span integration tests.'
-    group = 'verification'
-    testClassesDirs = sourceSets.integrationTest.output.classesDirs
-    classpath = sourceSets.integrationTest.runtimeClasspath
+def includeRawSpanTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
 
-    filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
-    }
+def createRawSpanDataPrepperOTLP1Task = createDataPrepperDockerContainer(
+        "rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
+def createRawSpanDataPrepperOTLP2Task = createDataPrepperDockerContainer(
+        "rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
 
-    finalizedBy stopOpenSearchDockerContainer
-    def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
-    def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
-    finalizedBy stopDataPrepper1Task
-    finalizedBy stopDataPrepper2Task
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
-    finalizedBy removeDataPrepperNetwork
-}
+// TODO: replace with Event type in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+def rawSpanOTLPEndToEndTest = createEndToEndTest("rawSpanOTLPEndToEndTest", includeRawSpanTestsMatchPattern,
+        createRawSpanDataPrepperOTLP1Task, createRawSpanDataPrepperOTLP2Task)
 
-task rawSpanCompatibilityEndToEndTest(type: Test) {
-    dependsOn build
-    dependsOn startOpenSearchDockerContainer
-    def createDataPrepper1Task = createDataPrepperDockerContainer(
-            "rawSpanDataPrepperFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
-    def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage(
-            "rawSpanDataPrepperFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
-    def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
-    def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
-    dependsOn startDataPrepper1Task
-    dependsOn startDataPrepper2Task
-    startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
-    startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
-    // wait for data-preppers to be ready
-    doFirst {
-        sleep(10*1000)
-    }
+def createRawSpanDataPrepperOTLPTask = createDataPrepperDockerContainer(
+        "rawSpanDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
+def createRawSpanDataPrepperEventTask = createDataPrepperDockerContainer(
+        "rawSpanDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}")
 
-    description = 'Runs the raw span compatibility integration tests.'
-    group = 'verification'
-    testClassesDirs = sourceSets.integrationTest.output.classesDirs
-    classpath = sourceSets.integrationTest.runtimeClasspath
+// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+def rawSpanOTLPAndEventEndToEndTest = createEndToEndTest("rawSpanOTLPAndEventEndToEndTest", includeRawSpanTestsMatchPattern,
+        createRawSpanDataPrepperOTLPTask, createRawSpanDataPrepperEventTask)
 
-    filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*"
-    }
+def rawSpanDataPrepperOTLPFromBuild = createDataPrepperDockerContainer(
+        "rawSpanDataPrepperOTLPFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
+def rawSpanDataPrepperOTLPFromPull = createDataPrepperDockerContainerFromPullImage(
+        "rawSpanDataPrepperOTLPFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
 
-    finalizedBy stopOpenSearchDockerContainer
-    def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
-    def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
-    finalizedBy stopDataPrepper1Task
-    finalizedBy stopDataPrepper2Task
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
-    finalizedBy removeDataPrepperNetwork
-}
+// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+def rawSpanOTLPLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanOTLPLatestReleaseCompatibilityEndToEndTest",
+        includeRawSpanTestsMatchPattern,
+        rawSpanDataPrepperOTLPFromBuild, rawSpanDataPrepperOTLPFromPull)
 
-task serviceMapEndToEndTest(type: Test) {
-    dependsOn build
-    dependsOn startOpenSearchDockerContainer
-    def createDataPrepper1Task = createDataPrepperDockerContainer(
-            "serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
-    def createDataPrepper2Task = createDataPrepperDockerContainer(
-            "serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
-    def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
-    def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
-    dependsOn startDataPrepper1Task
-    dependsOn startDataPrepper2Task
-    startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
-    startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
-    // wait for data-preppers to be ready
-    doFirst {
-        sleep(10*1000)
-    }
+def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer(
+        "rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}")
+def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage(
+        "rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
 
-    description = 'Runs the service-map integration tests.'
-    group = 'verification'
-    testClassesDirs = sourceSets.integrationTest.output.classesDirs
-    classpath = sourceSets.integrationTest.runtimeClasspath
+def rawSpanEventLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanEventLatestReleaseCompatibilityEndToEndTest",
+        includeRawSpanTestsMatchPattern,
+        rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull)
 
-    filter {
-        includeTestsMatching "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*"
-    }
+def includeServiceMapTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*"
 
-    finalizedBy stopOpenSearchDockerContainer
-    def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
-    def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
-    finalizedBy stopDataPrepper1Task
-    finalizedBy stopDataPrepper2Task
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer)
-    finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer)
-    finalizedBy removeDataPrepperNetwork
-}
+def createServiceMapDataPrepperOTLP1Task = createDataPrepperDockerContainer(
+        "serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
+def createServiceMapDataPrepperOTLP2Task = createDataPrepperDockerContainer(
+        "serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
+
+// TODO: replace with Event type in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+def serviceMapOTLPEndToEndTest = createEndToEndTest("serviceMapOTLPEndToEndTest", includeServiceMapTestsMatchPattern,
+        createServiceMapDataPrepperOTLP1Task, createServiceMapDataPrepperOTLP2Task)
+
+def serviceMapDataPrepperOTLP = createDataPrepperDockerContainer(
+        "serviceMapDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
+def serviceMapDataPrepperEvent = createDataPrepperDockerContainer(
+        "serviceMapDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}")
+
+// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272)
+def serviceMapOTLPAndEventEndToEndTest = createEndToEndTest("serviceMapOTLPAndEventEndToEndTest", includeServiceMapTestsMatchPattern,
+        serviceMapDataPrepperOTLP, serviceMapDataPrepperEvent)
 
 dependencies {
     integrationTestImplementation project(':data-prepper-plugins:opensearch')
-    integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-prepper')
+    integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-processor')
     integrationTestImplementation "org.awaitility:awaitility:4.1.1"
     integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}"
     integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.19.4'