Migrating commits from ODFE repo

Signed-off-by: Jeff Wright <[email protected]>
asifsmohammed · Aug 11, 2021 · 2d4c7ea · 2d4c7ea
2 parents a09c1c4 + 88e0236
commit 2d4c7ea
Show file tree

Hide file tree

Showing 152 changed files with 4,962 additions and 1,019 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,54 +4,54 @@ updates:
 - package-ecosystem: gradle
   directory: "/data-prepper-api"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-core"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/blocking-buffer"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/common"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/opensearch"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/mapdb-prepper-state"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/otel-trace-raw-prepper"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/otel-trace-group-prepper"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/otel-trace-source"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/peer-forwarder"
   schedule:
-    interval: weekly
+    interval: monthly
 
 - package-ecosystem: gradle
   directory: "/data-prepper-plugins/service-map-stateful"
   schedule:
-    interval: weekly
+    interval: monthly
diff --git a/...ata-prepper-trace-analytics-e2e-tests.yml → ...tics-raw-span-compatibility-e2e-tests.yml b/...ata-prepper-trace-analytics-e2e-tests.yml → ...tics-raw-span-compatibility-e2e-tests.yml
@@ -1,13 +1,12 @@
 # This workflow will build a Java project with Gradle
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
 
-name: Data Prepper Trace Analytics End-to-end test with Gradle
+name: Data Prepper Trace Analytics Raw Span Compatibility End-to-end test with Gradle
 
 on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
   workflow_dispatch:
 
 jobs:
@@ -37,7 +36,5 @@ jobs:
       uses: actions/checkout@v2
     - name: Grant execute permission for gradlew
       run: chmod +x gradlew
-    - name: Run raw-span end-to-end tests with Gradle
-      run: ./gradlew :data-prepper-core:rawSpanEndToEndTest
-    - name: Run service-map end-to-end tests with Gradle
-      run: ./gradlew :data-prepper-core:serviceMapEndToEndTest
+    - name: Run raw-span compatibility end-to-end tests with Gradle
+      run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml
@@ -0,0 +1,29 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper Trace Analytics Raw Span End-to-end test with Gradle
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [14]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Run raw-span end-to-end tests with Gradle
+        run: ./gradlew :data-prepper-core:rawSpanEndToEndTest
diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml
@@ -0,0 +1,29 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper Trace Analytics Service Map End-to-end test with Gradle
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [14]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Run service-map end-to-end tests with Gradle
+        run: ./gradlew :data-prepper-core:serviceMapEndToEndTest
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -3,7 +3,11 @@
 
 name: Data Prepper Java CI with Gradle
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   build:

diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
@@ -3,7 +3,11 @@
 
 name: Data Prepper OpenSearchSink integration tests with ODFE < 1.13.0
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   integration_tests:
@@ -41,4 +45,4 @@ jobs:
       - name: Run ODFE tests
         run: |
           ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin
-          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
+          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -3,7 +3,11 @@
 
 name: Data Prepper OpenSearchSink integration tests with ODFE >= 1.13.0
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   integration_tests:
@@ -41,4 +45,4 @@ jobs:
       - name: Run ODFE tests
         run: |
           ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin
-          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
+          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ We envision Data Prepper as an open source data collector for observability data
   - [Code of Conduct](#Code-of-Conduct)
   - [Security Issue Notifications](#Security-Issue-Notifications)
   - [License](#License)
-  
+
 
 
 ## Contribute

diff --git a/build.gradle b/build.gradle
@@ -8,21 +8,46 @@
  *  Modifications Copyright OpenSearch Contributors. See
  *  GitHub history for details.
  */
+plugins {
+    id "com.diffplug.spotless" version "5.12.4"
+}
 
 apply from: file("${rootDir}/build-resources.gradle")
 allprojects {
+    apply plugin: 'com.diffplug.spotless'
+
     group = 'com.amazon'
-    version = '1.0.0.0-alpha2'
+
     repositories {
         mavenCentral()
         maven { url "https://jitpack.io" }
     }
+
+    spotless {
+        format 'markdown', {
+            target '*.md'
+            // TODO: enrich format rules
+            endWithNewline()
+        }
+        format 'misc', {
+            target '.gitignore', '*.yml', '*.yaml'
+            // TODO: enrich format rules
+            trimTrailingWhitespace()
+            endWithNewline()
+        }
+    }
 }
 
 subprojects {
     apply plugin: 'java'
     apply plugin: 'jacoco'
     sourceCompatibility = '1.8'
+    spotless {
+        java {
+            // TODO: enrich format rules
+            removeUnusedImports()
+        }
+    }
     dependencies {
         implementation "com.google.guava:guava:29.0-jre"
         implementation "org.apache.logging.log4j:log4j-core:2.14.0"

diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle
@@ -13,7 +13,7 @@ plugins {
 
 }
 dependencies {
-    implementation "io.micrometer:micrometer-core:1.6.6"
+    implementation "io.micrometer:micrometer-core:1.7.2"
 
     testImplementation "org.hamcrest:hamcrest:2.2"
 }

diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java
@@ -13,6 +13,13 @@
 
 public class MetricNames {
     private MetricNames() {}
+
+    /**
+     * Metric dimension representing service name.
+     * Applicable to all components
+     */
+    public static final String SERVICE_NAME = "serviceName";
+
     /**
      * Metric representing the ingress of records to a pipeline component.
      * Applicable to preppers and sinks
@@ -46,6 +53,11 @@ private MetricNames() {}
      */
     public static final String RECORDS_INFLIGHT = "recordsInFlight";
 
+    /**
+     * Metric representing the number of records currently in the buffer.
+     */
+    public static final String RECORDS_IN_BUFFER = "recordsInBuffer";
+
     /**
      * Metric representing the number of records read from a buffer and processed by the pipeline.
      */

diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java
@@ -49,6 +49,10 @@ public Counter counter(final String name) {
         return Metrics.counter(getMeterName(name));
     }
 
+    public Counter counterWithTags(final String name, final String... tags) {
+        return Metrics.counter(getMeterName(name), tags);
+    }
+
     public Counter counter(final String name, final String metricsPrefix) {
         return Metrics.counter(new StringJoiner(MetricNames.DELIMITER).add(metricsPrefix).add(name).toString());
     }
@@ -57,6 +61,10 @@ public Timer timer(final String name) {
         return Metrics.timer(getMeterName(name));
     }
 
+    public Timer timerWithTags(final String name, final String... tags) {
+        return Metrics.timer(getMeterName(name), tags);
+    }
+
     public DistributionSummary summary(final String name) {
         return Metrics.summary(getMeterName(name));
     }