diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eff458d1a3..268ec594ac 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,54 +4,54 @@ updates: - package-ecosystem: gradle directory: "/data-prepper-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-core" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/blocking-buffer" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/common" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/opensearch" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/mapdb-prepper-state" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-raw-prepper" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-group-prepper" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-source" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/peer-forwarder" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/service-map-stateful" schedule: - interval: weekly + interval: monthly diff --git a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml similarity index 76% rename from .github/workflows/data-prepper-trace-analytics-e2e-tests.yml rename to .github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index e314de9d20..b9cf08be89 100644 --- a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -1,13 +1,12 @@ # This workflow will build a Java project with Gradle # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle -name: Data Prepper Trace Analytics End-to-end test with Gradle +name: Data Prepper Trace Analytics Raw Span Compatibility End-to-end test with Gradle on: push: branches: [ main ] pull_request: - branches: [ main ] workflow_dispatch: jobs: @@ -37,7 +36,5 @@ jobs: uses: actions/checkout@v2 - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Run raw-span end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:rawSpanEndToEndTest - - name: Run service-map end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:serviceMapEndToEndTest + - name: Run raw-span compatibility end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml new file mode 100644 index 0000000000..1d80d7be23 --- /dev/null +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -0,0 +1,29 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper Trace Analytics Raw Span End-to-end test with Gradle + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run raw-span end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:rawSpanEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml new file mode 100644 index 0000000000..7868746e36 --- /dev/null +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -0,0 +1,29 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper Trace Analytics Service Map End-to-end test with Gradle + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run service-map end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:serviceMapEndToEndTest \ No newline at end of file diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 99d482e5da..a6e704a8a1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,7 +3,11 @@ name: Data Prepper Java CI with Gradle -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml index 87f06f70c0..9e9f2cedae 100644 --- a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml +++ b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml @@ -3,7 +3,11 @@ name: Data Prepper OpenSearchSink integration tests with ODFE < 1.13.0 -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: integration_tests: @@ -41,4 +45,4 @@ jobs: - name: Run ODFE tests run: | ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin - ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin \ No newline at end of file + ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml index ff9b61c6e8..fb7ed8bf6b 100644 --- a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml +++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml @@ -3,7 +3,11 @@ name: Data Prepper OpenSearchSink integration tests with ODFE >= 1.13.0 -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: integration_tests: @@ -41,4 +45,4 @@ jobs: - name: Run ODFE tests run: | ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin - ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin \ No newline at end of file + ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin diff --git a/README.md b/README.md index 9e685135f7..f736bc4ce2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ We envision Data Prepper as an open source data collector for observability data - [Code of Conduct](#Code-of-Conduct) - [Security Issue Notifications](#Security-Issue-Notifications) - [License](#License) - + ## Contribute diff --git a/build.gradle b/build.gradle index 5f786f749c..482d4c01e4 100644 --- a/build.gradle +++ b/build.gradle @@ -8,21 +8,46 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ +plugins { + id "com.diffplug.spotless" version "5.12.4" +} apply from: file("${rootDir}/build-resources.gradle") allprojects { + apply plugin: 'com.diffplug.spotless' + group = 'com.amazon' - version = '1.0.0.0-alpha2' + repositories { mavenCentral() maven { url "https://jitpack.io" } } + + spotless { + format 'markdown', { + target '*.md' + // TODO: enrich format rules + endWithNewline() + } + format 'misc', { + target '.gitignore', '*.yml', '*.yaml' + // TODO: enrich format rules + trimTrailingWhitespace() + endWithNewline() + } + } } subprojects { apply plugin: 'java' apply plugin: 'jacoco' sourceCompatibility = '1.8' + spotless { + java { + // TODO: enrich format rules + removeUnusedImports() + } + } dependencies { implementation "com.google.guava:guava:29.0-jre" implementation "org.apache.logging.log4j:log4j-core:2.14.0" diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index c97f83d7cc..892ff324bc 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -13,7 +13,7 @@ plugins { } dependencies { - implementation "io.micrometer:micrometer-core:1.6.6" + implementation "io.micrometer:micrometer-core:1.7.2" testImplementation "org.hamcrest:hamcrest:2.2" } diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java index 20da25ff26..5800ba4e49 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java @@ -13,6 +13,13 @@ public class MetricNames { private MetricNames() {} + + /** + * Metric dimension representing service name. + * Applicable to all components + */ + public static final String SERVICE_NAME = "serviceName"; + /** * Metric representing the ingress of records to a pipeline component. * Applicable to preppers and sinks @@ -46,6 +53,11 @@ private MetricNames() {} */ public static final String RECORDS_INFLIGHT = "recordsInFlight"; + /** + * Metric representing the number of records currently in the buffer. + */ + public static final String RECORDS_IN_BUFFER = "recordsInBuffer"; + /** * Metric representing the number of records read from a buffer and processed by the pipeline. */ diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java index 281bb2472a..f2d6a6fb1c 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java @@ -49,6 +49,10 @@ public Counter counter(final String name) { return Metrics.counter(getMeterName(name)); } + public Counter counterWithTags(final String name, final String... tags) { + return Metrics.counter(getMeterName(name), tags); + } + public Counter counter(final String name, final String metricsPrefix) { return Metrics.counter(new StringJoiner(MetricNames.DELIMITER).add(metricsPrefix).add(name).toString()); } @@ -57,6 +61,10 @@ public Timer timer(final String name) { return Metrics.timer(getMeterName(name)); } + public Timer timerWithTags(final String name, final String... tags) { + return Metrics.timer(getMeterName(name), tags); + } + public DistributionSummary summary(final String name) { return Metrics.summary(getMeterName(name)); } diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java index 21d91bbefd..9d3152a224 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java @@ -11,19 +11,20 @@ package com.amazon.dataprepper.model.buffer; -import com.amazon.dataprepper.model.CheckpointState; -import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.metrics.MetricNames; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.CheckpointState; +import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; + import java.util.Collection; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; - /** * Abstract implementation of the Buffer interface to record boilerplate metrics */ @@ -32,6 +33,7 @@ public abstract class AbstractBuffer> implements Buffer { private final Counter recordsWrittenCounter; private final Counter recordsReadCounter; private final AtomicLong recordsInFlight; + private final AtomicLong recordsInBuffer; private final Counter recordsProcessedCounter; private final Counter writeTimeoutCounter; private final Timer writeTimer; @@ -51,6 +53,7 @@ private AbstractBuffer(final PluginMetrics pluginMetrics, final String pipelineN this.recordsWrittenCounter = pluginMetrics.counter(MetricNames.RECORDS_WRITTEN); this.recordsReadCounter = pluginMetrics.counter(MetricNames.RECORDS_READ); this.recordsInFlight = pluginMetrics.gauge(MetricNames.RECORDS_INFLIGHT, new AtomicLong()); + this.recordsInBuffer = pluginMetrics.gauge(MetricNames.RECORDS_IN_BUFFER, new AtomicLong()); this.recordsProcessedCounter = pluginMetrics.counter(MetricNames.RECORDS_PROCESSED, pipelineName); this.writeTimeoutCounter = pluginMetrics.counter(MetricNames.WRITE_TIMEOUTS); this.writeTimer = pluginMetrics.timer(MetricNames.WRITE_TIME_ELAPSED); @@ -61,42 +64,40 @@ private AbstractBuffer(final PluginMetrics pluginMetrics, final String pipelineN /** * Records metrics for ingress, time elapsed, and timeouts, while calling the doWrite method * to perform the actual write - * @param record the Record to add + * + * @param record the Record to add * @param timeoutInMillis how long to wait before giving up * @throws TimeoutException */ @Override public void write(T record, int timeoutInMillis) throws TimeoutException { + long startTime = System.nanoTime(); + try { - writeTimer.record(() -> { - try { - doWrite(record, timeoutInMillis); - } catch (TimeoutException e) { - writeTimeoutCounter.increment(); - throw new RuntimeException(e); - } - }); + doWrite(record, timeoutInMillis); recordsWrittenCounter.increment(); - } catch (RuntimeException e) { - if(e.getCause() instanceof TimeoutException) { - throw (TimeoutException) e.getCause(); - } else { - throw e; - } + recordsInBuffer.incrementAndGet(); + } catch (TimeoutException e) { + writeTimeoutCounter.increment(); + throw e; + } finally { + writeTimer.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } } /** * Records egress and time elapsed metrics, while calling the doRead function to * do the actual read + * * @param timeoutInMillis how long to wait before giving up * @return Records collection and checkpoint state read from the buffer */ @Override public Map.Entry, CheckpointState> read(int timeoutInMillis) { final Map.Entry, CheckpointState> readResult = readTimer.record(() -> doRead(timeoutInMillis)); - recordsReadCounter.increment(readResult.getKey().size()*1.0); + recordsReadCounter.increment(readResult.getKey().size() * 1.0); recordsInFlight.addAndGet(readResult.getValue().getNumRecordsToBeChecked()); + recordsInBuffer.addAndGet(-1 * readResult.getValue().getNumRecordsToBeChecked()); return readResult; } @@ -114,7 +115,8 @@ protected int getRecordsInFlight() { /** * This method should implement the logic for writing to the buffer - * @param record Record to write to buffer + * + * @param record Record to write to buffer * @param timeoutInMillis Timeout for write operation in millis * @throws TimeoutException */ @@ -122,6 +124,7 @@ protected int getRecordsInFlight() { /** * This method should implement the logic for reading from the buffer + * * @param timeoutInMillis Timeout in millis * @return Records collection and checkpoint state read from the buffer */ diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java index 8b8e0f9b19..4f5ee6096a 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java @@ -68,8 +68,14 @@ public interface PrepperState { /** * @return Size of the prepper state data stored in file, in bytes. */ + // TODO: Potentially remove, this is file-specific public long sizeInBytes(); + /** + * Clear internal state + */ + void clear(); + /** * Any cleanup code goes here */ diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java index c447fddc19..bc8b80e50c 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java @@ -12,9 +12,6 @@ package com.amazon.dataprepper.metrics; import com.amazon.dataprepper.model.configuration.PluginSetting; -import java.util.Collections; -import java.util.StringJoiner; -import java.util.concurrent.atomic.AtomicInteger; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Metrics; @@ -22,9 +19,15 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Collections; +import java.util.StringJoiner; +import java.util.concurrent.atomic.AtomicInteger; + public class PluginMetricsTest { private static final String PLUGIN_NAME = "testPlugin"; private static final String PIPELINE_NAME = "pipelineName"; + private static final String TAG_KEY = "tagKey"; + private static final String TAG_VALUE = "tagValue"; private static final PluginSetting PLUGIN_SETTING = new PluginSetting(PLUGIN_NAME, Collections.emptyMap()) {{ setPipelineName(PIPELINE_NAME); }}; @@ -40,6 +43,18 @@ public void testCounter() { counter.getId().getName()); } + @Test + public void testCounterWithTags() { + final Counter counter = PLUGIN_METRICS.counterWithTags("counter", TAG_KEY, TAG_VALUE); + Assert.assertEquals( + new StringJoiner(MetricNames.DELIMITER) + .add(PIPELINE_NAME).add(PLUGIN_NAME) + .add("counter").toString(), + counter.getId().getName()); + + Assert.assertEquals(TAG_VALUE, counter.getId().getTag(TAG_KEY)); + } + @Test public void testCustomMetricsPrefixCounter() { final Counter counter = PLUGIN_METRICS.counter("counter", PIPELINE_NAME); @@ -51,12 +66,24 @@ public void testCustomMetricsPrefixCounter() { @Test public void testTimer() { - final Timer counter = PLUGIN_METRICS.timer("timer"); + final Timer timer = PLUGIN_METRICS.timer("timer"); Assert.assertEquals( new StringJoiner(MetricNames.DELIMITER) .add(PIPELINE_NAME).add(PLUGIN_NAME) .add("timer").toString(), - counter.getId().getName()); + timer.getId().getName()); + } + + @Test + public void testTimerWithTags() { + final Timer timer = PLUGIN_METRICS.timerWithTags("timer", TAG_KEY, TAG_VALUE); + Assert.assertEquals( + new StringJoiner(MetricNames.DELIMITER) + .add(PIPELINE_NAME).add(PLUGIN_NAME) + .add("timer").toString(), + timer.getId().getName()); + + Assert.assertEquals(TAG_VALUE, timer.getId().getTag(TAG_KEY)); } @Test diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java index f3aa19a460..f0d39fa6be 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java @@ -16,7 +16,6 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/data-prepper-benchmarks/mapdb-benchmarks/README.md b/data-prepper-benchmarks/mapdb-benchmarks/README.md index 324308fb59..98c3e1d9cd 100644 --- a/data-prepper-benchmarks/mapdb-benchmarks/README.md +++ b/data-prepper-benchmarks/mapdb-benchmarks/README.md @@ -1,8 +1,8 @@ # MapDb Benchmarks -This package uses JMH (https://openjdk.java.net/projects/code-tools/jmh/) to benchmark the MapDb Prepper State plugin. -To use jmh benchmarking easily with gradle, this package uses a jmh gradle plugin (https://github.com/melix/jmh-gradle-plugin/) . -Details on configuration and other options can be found there. +This package uses JMH (https://openjdk.java.net/projects/code-tools/jmh/) to benchmark the MapDb Prepper State plugin. +To use jmh benchmarking easily with gradle, this package uses a jmh gradle plugin (https://github.com/melix/jmh-gradle-plugin/) . +Details on configuration and other options can be found there. To run the benchmarks from this directory, run the following command: @@ -14,4 +14,4 @@ To build an executable standalone jar of these benchmarks, run: ``` ../../gradlew jmhJar -``` \ No newline at end of file +``` diff --git a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle index c93a581074..4538c9d1ef 100644 --- a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle +++ b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle @@ -24,5 +24,5 @@ repositories { } dependencies { - compile project(':data-prepper-plugins:mapdb-prepper-state') + implementation project(':data-prepper-plugins:mapdb-prepper-state') } diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md b/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md index 4a1e721ba9..3e5e113da6 100644 --- a/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md +++ b/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md @@ -1,6 +1,6 @@ # Service Map Stateful Benchmarks -This package contains benchmarks for the service map stateful prepper using JMH: https://openjdk.java.net/projects/code-tools/jmh/ . +This package contains benchmarks for the service map stateful prepper using JMH: https://openjdk.java.net/projects/code-tools/jmh/ . Integration with gradle is done with the following gradle plugin for JMH: https://github.com/melix/jmh-gradle-plugin. @@ -9,11 +9,11 @@ The plugin creates a source set for the JMH benchmarks, and provides a few gradl ## Running the tests via gradle task Tests can be run via the "jmh" gradle task provided by the plugin. The README for the plugin provides the various parameters that -can be provided to the plugin. +can be provided to the plugin. ## Running the tests via JAR -To run the tests via JAR, you can build the benchmark jar using the gradle task "jmhJar". This jar is an executable jar +To run the tests via JAR, you can build the benchmark jar using the gradle task "jmhJar". This jar is an executable jar that runs the benchmark tests. Example command: ``` @@ -21,4 +21,4 @@ java -jar service-map-stateful-benchmarks-0.1-beta-jmh.jar -r 600 -i 2 -p batchS ``` The above command will run the benchmarks for 600 seconds (10 minutes) per iteration, 2 iterations. It also -sets the batchSize and windowDurationSeconds benchmark parameters. +sets the batchSize and windowDurationSeconds benchmark parameters. diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle index 1a3d7d33f6..ef35391197 100644 --- a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle +++ b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle @@ -24,6 +24,6 @@ repositories { } dependencies { - compile project(':data-prepper-plugins:service-map-stateful') + implementation project(':data-prepper-plugins:service-map-stateful') jmh "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" } diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 4eadf169b9..713e1fb74d 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -24,19 +24,22 @@ sourceSets { apply from: "integrationTest.gradle" dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins') - testCompile project(':data-prepper-plugins:common').sourceSets.test.output - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins') + implementation project(':data-prepper-plugins:common') + testImplementation project(':data-prepper-plugins:common').sourceSets.test.output + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "javax.validation:validation-api:2.0.1.Final" implementation "org.apache.bval:bval-extras:2.0.5" implementation "org.apache.bval:bval-jsr:2.0.5" implementation "org.reflections:reflections:0.9.12" - implementation "io.micrometer:micrometer-core:1.6.6" - implementation "io.micrometer:micrometer-registry-prometheus:1.6.6" + implementation "io.micrometer:micrometer-core:1.6.5" + implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" + implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.2" + implementation "software.amazon.awssdk:cloudwatch:2.17.15" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-core:3.9.0" + testImplementation "org.mockito:mockito-core:3.11.2" } jar { diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 93090fd21f..64449b5df6 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -1,13 +1,15 @@ buildscript { repositories { - jcenter() + maven { + url "https://plugins.gradle.org/m2/" + } } dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:6.6.1' + classpath 'com.bmuschko:gradle-docker-plugin:7.0.0' } } -apply plugin: 'com.bmuschko.docker-remote-api' +apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin /* * SPDX-License-Identifier: Apache-2.0 @@ -66,6 +68,7 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) { } def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" +def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" /** @@ -80,6 +83,7 @@ task createDataPrepperDockerFile(type: Dockerfile) { workingDir("/app") copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") + copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml") defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml") @@ -124,6 +128,25 @@ def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDo } } +task pullDataPrepperDockerImage(type: DockerPullImage) { + image = 'amazon/opendistro-for-elasticsearch-data-prepper:latest' +} + +def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, final String dataPrepperName, final int grpcPort, + final int serverPort, final String pipelineConfigYAML) { + return tasks.create("create${taskBaseName}", DockerCreateContainer) { + dependsOn createDataPrepperNetwork + dependsOn pullDataPrepperDockerImage + containerName = dataPrepperName + hostConfig.portBindings = [String.format('%d:21890', grpcPort), String.format('%d:4900', serverPort)] + exposePorts('tcp', [21890, 4900]) + hostConfig.network = createDataPrepperNetwork.getNetworkName() + hostConfig.binds = [(project.file(pipelineConfigYAML).toString()):"/usr/share/data-prepper/pipelines.yaml", + (project.file("src/integrationTest/resources/data_prepper.yml").toString()):"/usr/share/data-prepper/data-prepper-config.yaml"] + targetImageId pullDataPrepperDockerImage.image + } +} + /** * OpenSearch Docker tasks */ @@ -163,9 +186,9 @@ task rawSpanEndToEndTest(type: Test) { dependsOn build dependsOn startOpenSearchDockerContainer def createDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") + "rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") def createDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") + "rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task @@ -183,7 +206,44 @@ task rawSpanEndToEndTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest*" + includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" + } + + finalizedBy stopOdfeDockerContainer + def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) + def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) + finalizedBy stopDataPrepper1Task + finalizedBy stopDataPrepper2Task + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) + finalizedBy removeDataPrepperNetwork +} + +task rawSpanCompatibilityEndToEndTest(type: Test) { + dependsOn build + dependsOn startOdfeDockerContainer + def createDataPrepper1Task = createDataPrepperDockerContainer( + "rawSpanDataPrepperFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage( + "rawSpanDataPrepperFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) + def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) + dependsOn startDataPrepper1Task + dependsOn startDataPrepper2Task + startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' + startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' + // wait for data-preppers to be ready + doFirst { + sleep(10*1000) + } + + description = 'Runs the raw span compatibility integration tests.' + group = 'verification' + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + + filter { + includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" } finalizedBy stopOpenSearchDockerContainer @@ -200,9 +260,9 @@ task serviceMapEndToEndTest(type: Test) { dependsOn build dependsOn startOpenSearchDockerContainer def createDataPrepper1Task = createDataPrepperDockerContainer( - "serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") + "serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") def createDataPrepper2Task = createDataPrepperDockerContainer( - "serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") + "serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml new file mode 100644 index 0000000000..5f8cf74d13 --- /dev/null +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -0,0 +1,26 @@ +entry-pipeline: + source: + otel_trace_source: + ssl: false + prepper: + - peer_forwarder: + discovery_mode: "static" + static_endpoints: ["dataprepper1", "dataprepper2"] + ssl: false + sink: + - pipeline: + name: "raw-pipeline" +raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - otel_trace_raw_prepper: + root_span_flush_delay: 3 # TODO: remove after 1.1 release + trace_flush_interval: 5 + sink: + - elasticsearch: + hosts: [ "https://node-0.example.com:9200" ] + username: "admin" + password: "admin" + trace_analytics_raw: true \ No newline at end of file diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 0ea1521ec6..f49a8cf2e1 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -11,7 +11,7 @@ raw-pipeline: name: "entry-pipeline" prepper: - otel_trace_raw_prepper: - root_span_flush_delay: 1 + root_span_flush_delay: 1 # TODO: remove after 1.1 release trace_flush_interval: 5 - otel_trace_group_prepper: hosts: [ "https://node-0.example.com:9200" ] diff --git a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml index a526d8e239..90a6636479 100644 --- a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml @@ -5,7 +5,7 @@ entry-pipeline: prepper: - peer_forwarder: discovery_mode: "static" - static_endpoints: ["data-prepper1", "data-prepper2"] + static_endpoints: ["dataprepper1", "dataprepper2"] ssl: false sink: - pipeline: diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index dd4737e943..3bbb1f630a 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -13,6 +13,7 @@ import com.amazon.dataprepper.parser.PipelineParser; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.parser.model.MetricRegistryType; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; import io.micrometer.core.instrument.Metrics; @@ -21,12 +22,13 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; -import io.micrometer.prometheus.PrometheusConfig; -import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.core.instrument.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.List; import java.util.Map; /** @@ -37,33 +39,43 @@ */ public class DataPrepper { private static final Logger LOG = LoggerFactory.getLogger(DataPrepper.class); + private static final String DATAPREPPER_SERVICE_NAME = "DATAPREPPER_SERVICE_NAME"; + private static final String DEFAULT_SERVICE_NAME = "dataprepper"; - private static final PrometheusMeterRegistry sysJVMMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + private static final CompositeMeterRegistry systemMeterRegistry = new CompositeMeterRegistry(); private Map transformationPipelines; private static volatile DataPrepper dataPrepper; private static DataPrepperServer dataPrepperServer; - private static DataPrepperConfiguration configuration = DataPrepperConfiguration.DEFAULT_CONFIG; - - static { - new ClassLoaderMetrics().bindTo(sysJVMMeterRegistry); - new JvmMemoryMetrics().bindTo(sysJVMMeterRegistry); - new JvmGcMetrics().bindTo(sysJVMMeterRegistry); - new ProcessorMetrics().bindTo(sysJVMMeterRegistry); - new JvmThreadMetrics().bindTo(sysJVMMeterRegistry); - } + private static DataPrepperConfiguration configuration; /** - * Set the DataPrepperConfiguration from a file + * Set the DataPrepperConfiguration from file + * * @param configurationFile File containing DataPrepperConfiguration yaml */ public static void configure(final String configurationFile) { - final DataPrepperConfiguration dataPrepperConfiguration = - DataPrepperConfiguration.fromFile(new File(configurationFile)); + configuration = DataPrepperConfiguration.fromFile(new File(configurationFile)); + configureMeterRegistry(); + } + + /** + * Set the DataPrepperConfiguration with defaults + */ + public static void configureWithDefaults() { + configuration = DataPrepperConfiguration.DEFAULT_CONFIG; + configureMeterRegistry(); + } - configuration = dataPrepperConfiguration; + /** + * returns serviceName if exists or default serviceName + * @return serviceName for data-prepper + */ + public static String getServiceNameForMetrics() { + final String serviceName = System.getenv(DATAPREPPER_SERVICE_NAME); + return StringUtils.isNotBlank(serviceName) ? serviceName : DEFAULT_SERVICE_NAME; } public static DataPrepper getInstance() { @@ -80,20 +92,32 @@ private DataPrepper() { if (dataPrepper != null) { throw new RuntimeException("Please use getInstance() for an instance of this Data Prepper"); } - startPrometheusBackend(); + startMeterRegistryForDataPrepper(); dataPrepperServer = new DataPrepperServer(this); } - public static PrometheusMeterRegistry getSysJVMMeterRegistry() { - return sysJVMMeterRegistry; - } - /** - * Create a PrometheusMeterRegistry for this DataPrepper and register it with the global registry + * Creates instances of configured MeterRegistry and registers to {@link Metrics} globalRegistry to be used by + * Meters. */ - private static void startPrometheusBackend() { - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); - Metrics.addRegistry(prometheusMeterRegistry); + private static void startMeterRegistryForDataPrepper() { + final List configuredMetricRegistryTypes = configuration.getMetricRegistryTypes(); + configuredMetricRegistryTypes.forEach(metricRegistryType -> Metrics.addRegistry(MetricRegistryType + .getDefaultMeterRegistryForType(metricRegistryType))); + } + + private static void configureMeterRegistry() { + configuration.getMetricRegistryTypes().forEach(metricRegistryType -> + systemMeterRegistry.add(MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType))); + new ClassLoaderMetrics().bindTo(systemMeterRegistry); + new JvmMemoryMetrics().bindTo(systemMeterRegistry); + new JvmGcMetrics().bindTo(systemMeterRegistry); + new ProcessorMetrics().bindTo(systemMeterRegistry); + new JvmThreadMetrics().bindTo(systemMeterRegistry); + } + + public static CompositeMeterRegistry getSystemMeterRegistry() { + return systemMeterRegistry; } /** @@ -106,7 +130,7 @@ public boolean execute(final String configurationFileLocation) { LOG.info("Using {} configuration file", configurationFileLocation); final PipelineParser pipelineParser = new PipelineParser(configurationFileLocation); transformationPipelines = pipelineParser.parseConfiguration(); - if (transformationPipelines.size() == 0){ + if (transformationPipelines.size() == 0) { LOG.error("No valid pipeline is available for execution, exiting"); System.exit(1); } @@ -126,22 +150,23 @@ public void shutdown() { /** * Triggers shutdown of the Data Prepper server. */ - public void shutdownDataPrepperServer(){ + public void shutdownDataPrepperServer() { dataPrepperServer.stop(); } /** * Triggers shutdown of the provided pipeline, no-op if the pipeline does not exist. + * * @param pipeline name of the pipeline */ public void shutdown(final String pipeline) { - if(transformationPipelines.containsKey(pipeline)) { + if (transformationPipelines.containsKey(pipeline)) { transformationPipelines.get(pipeline).shutdown(); } } public Map getTransformationPipelines() { - return transformationPipelines; + return transformationPipelines; } public static DataPrepperConfiguration getConfiguration() { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index 573fb20090..33470b79de 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -25,6 +25,8 @@ public static void main(String[] args) { if(args.length > 1) { DataPrepper.configure(args[1]); + } else { + DataPrepper.configureWithDefaults(); } final DataPrepper dataPrepper = DataPrepper.getInstance(); if (args.length > 0) { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java index 561c6ff0bc..a52a9e2713 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java @@ -11,22 +11,27 @@ package com.amazon.dataprepper.parser.model; -import java.io.File; -import java.io.IOException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + /** * Class to hold configuration for DataPrepper, including server port and Log4j settings */ public class DataPrepperConfiguration { + private static final List DEFAULT_METRIC_REGISTRY_TYPE = Collections.singletonList(MetricRegistryType.Prometheus); private int serverPort = 4900; private boolean ssl = true; private String keyStoreFilePath = ""; private String keyStorePassword = ""; private String privateKeyPassword = ""; + private List metricRegistries = DEFAULT_METRIC_REGISTRY_TYPE; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); @@ -41,7 +46,7 @@ public static DataPrepperConfiguration fromFile(File file) { try { return OBJECT_MAPPER.readValue(file, DataPrepperConfiguration.class); } catch (IOException e) { - throw new IllegalArgumentException("Invalid DataPrepper configuration file."); + throw new IllegalArgumentException("Invalid DataPrepper configuration file.", e); } } @@ -53,12 +58,14 @@ public DataPrepperConfiguration( @JsonProperty("keyStoreFilePath") final String keyStoreFilePath, @JsonProperty("keyStorePassword") final String keyStorePassword, @JsonProperty("privateKeyPassword") final String privateKeyPassword, - @JsonProperty("serverPort") final String serverPort + @JsonProperty("serverPort") final String serverPort, + @JsonProperty("metricRegistries") final List metricRegistries ) { setSsl(ssl); this.keyStoreFilePath = keyStoreFilePath != null ? keyStoreFilePath : ""; this.keyStorePassword = keyStorePassword != null ? keyStorePassword : ""; this.privateKeyPassword = privateKeyPassword != null ? privateKeyPassword : ""; + this.metricRegistries = metricRegistries != null && !metricRegistries.isEmpty() ? metricRegistries : DEFAULT_METRIC_REGISTRY_TYPE; setServerPort(serverPort); } @@ -82,6 +89,10 @@ public String getPrivateKeyPassword() { return privateKeyPassword; } + public List getMetricRegistryTypes() { + return metricRegistries; + } + private void setSsl(final Boolean ssl) { if (ssl != null) { this.ssl = ssl; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java new file mode 100644 index 0000000000..c33038d3d7 --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java @@ -0,0 +1,34 @@ +package com.amazon.dataprepper.parser.model; + +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; + +import java.util.Arrays; + +import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; +import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; +import static java.lang.String.format; + +public enum MetricRegistryType { + Prometheus, + CloudWatch; + + public static MeterRegistry getDefaultMeterRegistryForType(final MetricRegistryType metricRegistryType) { + MeterRegistry meterRegistry = null; + switch (metricRegistryType) { + case Prometheus: + meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + break; + case CloudWatch: + meterRegistry = new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); + break; + default: + throw new IllegalArgumentException(format("Invalid metricRegistryType %s", metricRegistryType)); + } + meterRegistry.config().commonTags(Arrays.asList(Tag.of(SERVICE_NAME, getServiceNameForMetrics()))); + return meterRegistry; + } +} \ No newline at end of file diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java index 667e52212b..0197fd5b64 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java @@ -90,7 +90,8 @@ public Pipeline( this.prepperExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(prepperThreads, new PipelineThreadFactory(format("%s-prepper-worker", name)), this); - this.sinkExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(sinks.size(), + // TODO: allow this to be configurable as well? + this.sinkExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(prepperThreads, new PipelineThreadFactory(format("%s-sink-worker", name)), this); stopRequested = false; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java new file mode 100644 index 0000000000..9705369316 --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java @@ -0,0 +1,71 @@ +package com.amazon.dataprepper.pipeline.server; + +import io.micrometer.cloudwatch2.CloudWatchConfig; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import io.micrometer.core.instrument.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import static java.util.Objects.requireNonNull; + +/** + * Provides {@link CloudWatchMeterRegistry} that enables publishing metrics to AWS Cloudwatch. Registry + * uses the default aws credentials (i.e. credentials from .aws directory; + * refer https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html#credentials-file-format). + * {@link CloudWatchMeterRegistryProvider} also has a constructor with {@link CloudWatchAsyncClient} that will be used + * for communication with Cloudwatch. + */ +public class CloudWatchMeterRegistryProvider { + private static final String CLOUDWATCH_PROPERTIES = "cloudwatch.properties"; + private static final Logger LOG = LoggerFactory.getLogger(CloudWatchMeterRegistryProvider.class); + + private final CloudWatchMeterRegistry cloudWatchMeterRegistry; + + public CloudWatchMeterRegistryProvider() { + this(CLOUDWATCH_PROPERTIES, CloudWatchAsyncClient.create()); + } + + public CloudWatchMeterRegistryProvider( + final String cloudWatchPropertiesFilePath, + final CloudWatchAsyncClient cloudWatchAsyncClient) { + final CloudWatchConfig cloudWatchConfig = createCloudWatchConfig( + requireNonNull(cloudWatchPropertiesFilePath, "cloudWatchPropertiesFilePath must not be null")); + this.cloudWatchMeterRegistry = new CloudWatchMeterRegistry(cloudWatchConfig, Clock.SYSTEM, + requireNonNull(cloudWatchAsyncClient, "cloudWatchAsyncClient must not be null")); + } + + /** + * Returns the CloudWatchMeterRegistry created using the default aws credentials + */ + public CloudWatchMeterRegistry getCloudWatchMeterRegistry() { + return this.cloudWatchMeterRegistry; + } + + /** + * Returns CloudWatchConfig using the properties from {@link #CLOUDWATCH_PROPERTIES} + */ + private CloudWatchConfig createCloudWatchConfig(final String cloudWatchPropertiesFilePath) { + CloudWatchConfig cloudWatchConfig = null; + try (final InputStream inputStream = requireNonNull(getClass().getClassLoader() + .getResourceAsStream(cloudWatchPropertiesFilePath))) { + final Properties cloudwatchProperties = new Properties(); + cloudwatchProperties.load(inputStream); + cloudWatchConfig = new CloudWatchConfig() { + @Override + public String get(final String key) { + return cloudwatchProperties.getProperty(key); + } + }; + } catch (IOException ex) { + LOG.error("Encountered exception in creating CloudWatchConfig for CloudWatchMeterRegistry, " + + "Proceeding without metrics", ex); + //If there is no registry attached, micrometer will make NoopMeters which are discarded. + } + return cloudWatchConfig; + } +} diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index 9d1c769098..0ebc6d2ac3 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -12,10 +12,14 @@ package com.amazon.dataprepper.pipeline.server; import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsParameters; import com.sun.net.httpserver.HttpsServer; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +27,8 @@ import javax.net.ssl.SSLParameters; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Optional; +import java.util.Set; /** * Class to handle any serving that the data prepper instance needs to do. @@ -33,11 +39,12 @@ public class DataPrepperServer { private HttpServer server; public DataPrepperServer(final DataPrepper dataPrepper) { - final int port = DataPrepper.getConfiguration().getServerPort(); - final boolean ssl = DataPrepper.getConfiguration().ssl(); - final String keyStoreFilePath = DataPrepper.getConfiguration().getKeyStoreFilePath(); - final String keyStorePassword = DataPrepper.getConfiguration().getKeyStorePassword(); - final String privateKeyPassword = DataPrepper.getConfiguration().getPrivateKeyPassword(); + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); + final int port = dataPrepperConfiguration.getServerPort(); + final boolean ssl = dataPrepperConfiguration.ssl(); + final String keyStoreFilePath = dataPrepperConfiguration.getKeyStoreFilePath(); + final String keyStorePassword = dataPrepperConfiguration.getKeyStorePassword(); + final String privateKeyPassword = dataPrepperConfiguration.getPrivateKeyPassword(); try { if (ssl) { @@ -51,12 +58,25 @@ public DataPrepperServer(final DataPrepper dataPrepper) { throw new RuntimeException("Failed to create server", e); } - server.createContext("/metrics/prometheus", new PrometheusMetricsHandler()); - server.createContext("/metrics/sys", new PrometheusMetricsHandler(DataPrepper.getSysJVMMeterRegistry())); + getPrometheusMeterRegistryFromRegistries(Metrics.globalRegistry.getRegistries()).ifPresent(meterRegistry -> { + final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = (PrometheusMeterRegistry) meterRegistry; + server.createContext("/metrics/prometheus", new PrometheusMetricsHandler(prometheusMeterRegistryForDataPrepper)); + }); + + getPrometheusMeterRegistryFromRegistries(DataPrepper.getSystemMeterRegistry().getRegistries()).ifPresent( + meterRegistry -> { + final PrometheusMeterRegistry prometheusMeterRegistryForSystem = (PrometheusMeterRegistry) meterRegistry; + server.createContext("/metrics/sys", new PrometheusMetricsHandler(prometheusMeterRegistryForSystem)); + }); server.createContext("/list", new ListPipelinesHandler(dataPrepper)); server.createContext("/shutdown", new ShutdownHandler(dataPrepper)); } + private Optional getPrometheusMeterRegistryFromRegistries(final Set meterRegistries) { + return meterRegistries.stream().filter(meterRegistry -> + meterRegistry instanceof PrometheusMeterRegistry).findFirst(); + } + /** * Start the DataPrepperServer */ diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java index 723abb9fb9..ce281a0592 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java @@ -11,15 +11,15 @@ package com.amazon.dataprepper.pipeline.server; -import java.io.IOException; -import java.net.HttpURLConnection; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; -import io.micrometer.core.instrument.Metrics; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.HttpURLConnection; + /** * HttpHandler to handle requests for Prometheus metrics */ @@ -28,10 +28,6 @@ public class PrometheusMetricsHandler implements HttpHandler { private PrometheusMeterRegistry prometheusMeterRegistry; private final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsHandler.class); - public PrometheusMetricsHandler() { - prometheusMeterRegistry = (PrometheusMeterRegistry) Metrics.globalRegistry.getRegistries().iterator().next(); - } - public PrometheusMetricsHandler(final PrometheusMeterRegistry prometheusMeterRegistry) { this.prometheusMeterRegistry = prometheusMeterRegistry; } diff --git a/data-prepper-core/src/main/resources/cloudwatch.properties b/data-prepper-core/src/main/resources/cloudwatch.properties new file mode 100644 index 0000000000..c916d81f77 --- /dev/null +++ b/data-prepper-core/src/main/resources/cloudwatch.properties @@ -0,0 +1,4 @@ +cloudwatch.enabled=true +cloudwatch.namespace=dataprepper +cloudwatch.batchSize=20 +cloudwatch.step=PT1M \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index 8ab560a392..bf517dd105 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -54,21 +54,21 @@ public void testInstanceCreation() { } @Test - public void testDataPrepperSysMetrics() { + public void testDataPrepperSystemMetrics() { // Test retrieve gauge in ClassLoaderMetrics - final List classesLoaded = getSysMeasurementList("jvm.classes.loaded"); + final List classesLoaded = getSystemMeasurementList("jvm.classes.loaded"); Assert.assertEquals(1, classesLoaded.size()); // Test retrieve gauge in JvmMemoryMetrics - final List jvmBufferCount = getSysMeasurementList("jvm.buffer.count"); + final List jvmBufferCount = getSystemMeasurementList("jvm.buffer.count"); Assert.assertEquals(1, jvmBufferCount.size()); // Test retrieve gauge in JvmGcMetrics - final List jvmGcMaxDataSize = getSysMeasurementList("jvm.gc.max.data.size"); + final List jvmGcMaxDataSize = getSystemMeasurementList("jvm.gc.max.data.size"); Assert.assertEquals(1, jvmGcMaxDataSize.size()); // Test retrieve gauge in ProcessorMetrics - final List sysCPUCount = getSysMeasurementList("system.cpu.count"); + final List sysCPUCount = getSystemMeasurementList("system.cpu.count"); Assert.assertEquals(1, sysCPUCount.size()); // Test retrieve gauge in JvmThreadMetrics - final List jvmThreadsPeak = getSysMeasurementList("jvm.threads.peak"); + final List jvmThreadsPeak = getSystemMeasurementList("jvm.threads.peak"); Assert.assertEquals(1, jvmThreadsPeak.size()); } @@ -133,8 +133,8 @@ public int getExitStatus() { } } - private static List getSysMeasurementList(final String meterName) { - return StreamSupport.stream(DataPrepper.getSysJVMMeterRegistry().find(meterName).meter().measure().spliterator(), false) + private static List getSystemMeasurementList(final String meterName) { + return StreamSupport.stream(DataPrepper.getSystemMeterRegistry().find(meterName).meter().measure().spliterator(), false) .collect(Collectors.toList()); } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java index b6b3bf8c22..c64436c47c 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java @@ -51,6 +51,8 @@ public class TestDataProvider { public static final String VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS = "src/test/resources/valid_data_prepper_config_with_tls.yml"; public static final String VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE = "src/test/resources/valid_data_prepper_config_default_log4j.yml"; public static final String VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE = "src/test/resources/valid_data_prepper_some_default_config.yml"; + public static final String VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE = "src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml"; + public static final String VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE = "src/test/resources/valid_data_prepper_multiple_metrics_config.yml"; public static final String INVALID_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_data_prepper_config.yml"; public static final String INVALID_PORT_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_port_data_prepper_config.yml"; public static final String INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_data_prepper_config_with_bad_keystore_password.yml"; diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java index 36fc8f2d56..20fc21b653 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java @@ -11,6 +11,7 @@ package com.amazon.dataprepper.parser.model; +import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; @@ -18,8 +19,11 @@ import static com.amazon.dataprepper.TestDataProvider.INVALID_DATA_PREPPER_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.INVALID_PORT_DATA_PREPPER_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE; +import static org.hamcrest.MatcherAssert.assertThat; public class DataPrepperConfigurationTests { @@ -37,6 +41,30 @@ public void testSomeDefaultConfig() { Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); } + @Test + public void testDefaultMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + } + + @Test + public void testCloudWatchMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = + DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + + @Test + public void testMultipleMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = + DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + @Test(expected = IllegalArgumentException.class) public void testInvalidConfig() { final DataPrepperConfiguration dataPrepperConfiguration = diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java new file mode 100644 index 0000000000..84fbcdad50 --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java @@ -0,0 +1,34 @@ +package com.amazon.dataprepper.server; + +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class CloudWatchMeterRegistryProviderTest { + private static final String TEST_CLOUDWATCH_PROPERTIES = "cloudwatch_test.properties"; + + @Mock + CloudWatchAsyncClient cloudWatchAsyncClient; + + @Test(expected = NullPointerException.class) + public void testCreateWithInvalidPropertiesFile() { + new CloudWatchMeterRegistryProvider("does not exist", cloudWatchAsyncClient); + } + + @Test + public void testCreateCloudWatchMeterRegistry() { + final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider = new CloudWatchMeterRegistryProvider( + TEST_CLOUDWATCH_PROPERTIES, cloudWatchAsyncClient); + final CloudWatchMeterRegistry cloudWatchMeterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); + assertThat(cloudWatchMeterRegistry, notNullValue()); + } + +} diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index 7420b17793..74fa86a0be 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -16,9 +16,13 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -29,7 +33,9 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import java.io.File; import java.io.IOException; +import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -40,8 +46,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.Properties; +import java.util.Set; import java.util.UUID; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.mock; + public class DataPrepperServerTest { @@ -51,12 +63,17 @@ public class DataPrepperServerTest { private final int port = 1234; private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { - Metrics.globalRegistry.getRegistries().iterator().forEachRemaining(meterRegistry -> Metrics.globalRegistry.remove(meterRegistry)); + final Set meterRegistries = Metrics.globalRegistry.getRegistries(); + //to avoid ConcurrentModificationException + final Object[] registeredMeterRegistries = meterRegistries.toArray(); + for (final Object meterRegistry : registeredMeterRegistries) { + Metrics.removeRegistry((MeterRegistry) meterRegistry); + } Metrics.addRegistry(prometheusMeterRegistry); } private void setupDataPrepper() { - dataPrepper = Mockito.mock(DataPrepper.class); + dataPrepper = mock(DataPrepper.class); DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE); } @@ -108,18 +125,20 @@ public void testScrapeGlobalFailure() throws IOException, InterruptedException, } @Test - public void testGetSysMetrics() throws IOException, InterruptedException, URISyntaxException { + public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { final String scrape = UUID.randomUUID().toString(); final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(prometheusMeterRegistry); setupDataPrepper(); final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperMockedStatic.when(DataPrepper::getSysJVMMeterRegistry).thenReturn(prometheusMeterRegistry); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); dataPrepperServer = new DataPrepperServer(dataPrepper); dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:"+ port + "/metrics/sys")) + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) .GET().build(); HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); @@ -128,17 +147,19 @@ public void testGetSysMetrics() throws IOException, InterruptedException, URISyn } @Test - public void testScrapeSysMetricsFailure() throws IOException, InterruptedException, URISyntaxException { + public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(prometheusMeterRegistry); setupDataPrepper(); final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperMockedStatic.when(DataPrepper::getSysJVMMeterRegistry).thenReturn(prometheusMeterRegistry); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); dataPrepperServer = new DataPrepperServer(dataPrepper); dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:"+ port + "/metrics/sys")) + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) .GET().build(); HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); @@ -146,6 +167,62 @@ public void testScrapeSysMetricsFailure() throws IOException, InterruptedExcepti } } + @Test + public void testNonPrometheusMeterRegistry() throws Exception { + final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(cloudWatchMeterRegistry); + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( + new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); + setupDataPrepper(); + try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { + dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); + dataPrepperServer = new DataPrepperServer(dataPrepper); + dataPrepperServer.start(); + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) + .GET().build(); + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + } catch (ConnectException ex) { + dataPrepperServer.stop(); + //there should not be any Prometheus endpoints available + assertThat(ex.getMessage(), Matchers.is("Connection refused")); + } + } + + @Test + public void testMultipleMeterRegistries() throws Exception { + //for system metrics + final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); + final PrometheusMeterRegistry prometheusMeterRegistryForSystem = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(cloudWatchMeterRegistryForSystem); + compositeMeterRegistry.add(prometheusMeterRegistryForSystem); + + //for data prepper metrics + final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + setRegistry(prometheusMeterRegistryForDataPrepper); + + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( + new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); + setupDataPrepper(); + try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { + dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); + dataPrepperServer = new DataPrepperServer(dataPrepper); + dataPrepperServer.start(); + + //test prometheus registry + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + dataPrepperConfiguration.getServerPort() + + "/metrics/sys")).GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + dataPrepperServer.stop(); + } + } + @Test public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { setupDataPrepper(); diff --git a/data-prepper-core/src/test/resources/cloudwatch_test.properties b/data-prepper-core/src/test/resources/cloudwatch_test.properties new file mode 100644 index 0000000000..9023c91de3 --- /dev/null +++ b/data-prepper-core/src/test/resources/cloudwatch_test.properties @@ -0,0 +1,4 @@ +cloudwatch.enabled=true +cloudwatch.namespace=dataprepper-test +cloudwatch.batchSize=20 +cloudwatch.step=PT1S \ No newline at end of file diff --git a/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml b/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml new file mode 100644 index 0000000000..91a416eaaa --- /dev/null +++ b/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml @@ -0,0 +1,2 @@ +ssl: false +metricRegistries: [CloudWatch] \ No newline at end of file diff --git a/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml b/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml new file mode 100644 index 0000000000..cea3343be8 --- /dev/null +++ b/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml @@ -0,0 +1,2 @@ +ssl: false +metricRegistries: [Prometheus, CloudWatch] \ No newline at end of file diff --git a/data-prepper-plugins/blocking-buffer/README.md b/data-prepper-plugins/blocking-buffer/README.md index d84336469c..644b6f8cb0 100644 --- a/data-prepper-plugins/blocking-buffer/README.md +++ b/data-prepper-plugins/blocking-buffer/README.md @@ -8,7 +8,7 @@ Example `.yaml` configuration buffer: - bounded_blocking: ``` -*Note*: *By default, Data Prepper uses only one buffer. the `bounded_blocking` buffer, so this section in the `.yaml` need not be defined unless one wants to mention a custom buffer or tune the buffer settings* +*Note*: *By default, Data Prepper uses only one buffer. the `bounded_blocking` buffer, so this section in the `.yaml` need not be defined unless one wants to mention a custom buffer or tune the buffer settings* ## Configuration - buffer_size => An `int` representing max number of unchecked records the buffer accepts (num of unchecked records = num of records written into the buffer + num of in-flight records not yet checked by the Checkpointing API). Default is `512`. @@ -20,4 +20,4 @@ This plugin inherits the common metrics defined in [AbstractBuffer](https://gith ## Developer Guide This plugin is compatible with Java 14. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/blocking-buffer/build.gradle b/data-prepper-plugins/blocking-buffer/build.gradle index bd2c7e58a1..172eb994fc 100644 --- a/data-prepper-plugins/blocking-buffer/build.gradle +++ b/data-prepper-plugins/blocking-buffer/build.gradle @@ -13,7 +13,7 @@ plugins { id 'java' } dependencies { - compile project(':data-prepper-api') + implementation project(':data-prepper-api') testImplementation "junit:junit:4.13.2" } diff --git a/data-prepper-plugins/build.gradle b/data-prepper-plugins/build.gradle index a959047694..8c83c2c365 100644 --- a/data-prepper-plugins/build.gradle +++ b/data-prepper-plugins/build.gradle @@ -8,7 +8,10 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ +plugins { + id 'java-library' +} dependencies { - subprojects.forEach { compile project(':data-prepper-plugins:' + it.name) } + subprojects.forEach { api project(':data-prepper-plugins:' + it.name) } } \ No newline at end of file diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index 6e1beddd26..86da8ea331 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -24,4 +24,4 @@ A source plugin to read input data from console. ## `stdout` -A sink plugin to write output data to console. \ No newline at end of file +A sink plugin to write output data to console. diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 3004026bee..428dadef9c 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -10,15 +10,15 @@ */ plugins { - id 'java' + id 'java-library' } dependencies { - compile project(':data-prepper-api') - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + api project(':data-prepper-api') + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "org.reflections:reflections:0.9.12" testImplementation "junit:junit:4.13.2" - testImplementation "commons-io:commons-io:2.8.0" + testImplementation "commons-io:commons-io:2.11.0" } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index 1dd2733831..c43743e28e 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -23,13 +23,14 @@ repositories { } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.13.2' - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.4.32' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.4.32' - testCompile project(':data-prepper-plugins:common').sourceSets.test.output + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.21' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.21' + + testImplementation project(':data-prepper-plugins:common').sourceSets.test.output + testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation "org.hamcrest:hamcrest:2.2" } diff --git a/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java b/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java index d175d74b60..9af0521ba7 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java +++ b/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java @@ -11,19 +11,21 @@ package com.amazon.dataprepper.plugins.prepper.state; -import com.google.common.primitives.SignedBytes; import com.amazon.dataprepper.prepper.state.PrepperState; +import com.google.common.primitives.SignedBytes; +import org.mapdb.BTreeMap; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; +import org.mapdb.serializer.SerializerByteArray; + import java.io.File; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.BiFunction; -import org.mapdb.BTreeMap; -import org.mapdb.DBMaker; -import org.mapdb.Serializer; -import org.mapdb.serializer.SerializerByteArray; public class MapDbPrepperState implements PrepperState { @@ -41,14 +43,11 @@ public int compare(byte[] o1, byte[] o2) { private final File dbFile; public MapDbPrepperState(final File dbPath, final String dbName, final int concurrencyScale) { + // TODO: Cleanup references to file-based map this.dbFile = new File(String.join("/", dbPath.getPath(), dbName)); map = - (BTreeMap) DBMaker.fileDB(dbFile) - .fileDeleteAfterClose() - .fileMmapEnable() //MapDB uses the (slower) Random Access Files by default - .fileMmapPreclearDisable() + (BTreeMap) DBMaker.heapDB() .executorEnable() - .transactionEnable() .closeOnJvmShutdown() .concurrencyScale(concurrencyScale) .make() @@ -98,12 +97,18 @@ public List iterate(BiFunction fn, final int segments, fina return returnList; } + public Iterator> getIterator(final int segments, final int index) { + final KeyRange iterationEndpoints = getIterationEndpoints(segments, index); + return map.entryIterator(iterationEndpoints.low, true, iterationEndpoints.high, false); + } + /** * Gets iteration endpoints by taking the lowest and highest key and splitting the keyrange into segments. * These endpoints are an approximation of segments, and segments are guaranteed to cover the entire key range, * but there is no guarantee that all segments contain an equal number of elements. + * * @param segments Number of segments - * @param index Index to find segment endpoints for + * @param index Index to find segment endpoints for * @return KeyRange containing the two endpoints */ private KeyRange getIterationEndpoints(final int segments, final int index) { @@ -114,7 +119,7 @@ private KeyRange getIterationEndpoints(final int segments, final int index) { final byte[] highIndex = index == segments - 1 ? highEnd.add(new BigInteger("1")).toByteArray() : - lowEnd.add(step.multiply(new BigInteger(String.valueOf(index+1)))).toByteArray(); + lowEnd.add(step.multiply(new BigInteger(String.valueOf(index + 1)))).toByteArray(); return new KeyRange(lowIndex, highIndex); } @@ -129,6 +134,11 @@ public long sizeInBytes() { return dbFile.length(); } + @Override + public void clear() { + map.clear(); + } + @Override public void delete() { map.close(); diff --git a/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java b/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java index 925c56783e..88acec1442 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java +++ b/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java @@ -11,15 +11,16 @@ package com.amazon.dataprepper.plugins.prepper.state; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.function.BiFunction; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; public class MapDbPrepperStateTest extends PrepperStateTest { @@ -33,9 +34,9 @@ public void setPrepperState() throws Exception { @Test public void testIterateSegment() throws IOException { - final byte[] key1 = new byte[]{-64, 0, -64, 0}; + final byte[] key1 = new byte[]{-64, 0, -64, 0}; final byte[] key2 = new byte[]{0}; - final byte[] key3 = new byte[]{64, 64, 64 , 64}; + final byte[] key3 = new byte[]{64, 64, 64, 64}; final byte[] key4 = new byte[]{126, 126, 126, 126}; final DataClass data1 = new DataClass(UUID.randomUUID().toString(), random.nextInt()); @@ -72,8 +73,6 @@ public String apply(byte[] bytes, DataClass s) { data3.stringVal, data4.stringVal ))); - - Assert.assertEquals( 1048576, prepperState.sizeInBytes()); } } diff --git a/data-prepper-plugins/opensearch/README.md b/data-prepper-plugins/opensearch/README.md index e58c53f7d3..83c8863951 100644 --- a/data-prepper-plugins/opensearch/README.md +++ b/data-prepper-plugins/opensearch/README.md @@ -52,7 +52,7 @@ pipeline: sink: opensearch: hosts: ["https://your-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true cert: path/to/cert insecure: false trace_analytics_service_map: true @@ -64,12 +64,14 @@ pipeline: - `hosts`: A list of IP addresses of elasticsearch nodes. - `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that ODFE is using. -Default is null. +Default is null. -- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. +- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. +- `aws_sts_role_arn`: A IAM role arn which the sink plugin will assume to sign request to Amazon Elasticsearch. If not provided the plugin will use the default credentials. + - `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to `false`. - `username`(optional): A String of username used in the [internal users](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles) of ODFE cluster. Default is null. @@ -121,8 +123,8 @@ e.g. [otel-v1-apm-span-index-template.json](https://github.com/opensearch-projec - `dlq_file`(optional): A String of absolute file path for DLQ failed output records. Defaults to null. If not provided, failed records will be written into the default data-prepper log file (`logs/Data-Prepper.log`). -- `bulk_size` (optional): A long of bulk size in bulk requests in MB. Default to 5 MB. If set to be less than 0, -all the records received from the upstream prepper at a time will be sent as a single bulk request. +- `bulk_size` (optional): A long of bulk size in bulk requests in MB. Default to 5 MB. If set to be less than 0, +all the records received from the upstream prepper at a time will be sent as a single bulk request. If a single record turns out to be larger than the set bulk size, it will be sent as a bulk request of a single document. ## Metrics @@ -138,11 +140,11 @@ Besides common metrics in [AbstractSink](https://github.com/opensearch-project/d - `bulkRequestErrors`: measures number of errors encountered in sending bulk requests. - `documentsSuccess`: measures number of documents successfully sent to ES by bulk requests including retries. - `documentsSuccessFirstAttempt`: measures number of documents successfully sent to ES by bulk requests on first attempt. -- `documentErrors`: measures number of documents failed to be sent by bulk requests. +- `documentErrors`: measures number of documents failed to be sent by bulk requests. ## Developer Guide -This plugin is compatible with Java 8. See +This plugin is compatible with Java 8. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) - [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index c998b48bb5..35406efa3b 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -46,22 +46,28 @@ repositories { } dependencies { - compile project(':data-prepper-api') - testCompile project(':data-prepper-api').sourceSets.test.output - compile project(':data-prepper-plugins:common') + api project(':data-prepper-api') + testImplementation project(':data-prepper-api').sourceSets.test.output + api project(':data-prepper-plugins:common') implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1001" - implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" - implementation "io.micrometer:micrometer-core:1.6.6" + implementation 'software.amazon.awssdk:auth:2.17.15' + implementation 'software.amazon.awssdk:http-client-spi:2.17.15' + implementation 'software.amazon.awssdk:sdk-core:2.17.15' + implementation 'software.amazon.awssdk:aws-core:2.17.15' + implementation 'software.amazon.awssdk:regions:2.17.15' + implementation 'software.amazon.awssdk:utils:2.17.15' + implementation 'software.amazon.awssdk:sts:2.17.15' + implementation 'software.amazon.awssdk:url-connection-client:2.17.15' + implementation "io.micrometer:micrometer-core:1.7.2" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } - testImplementation "org.awaitility:awaitility:4.0.3" + testImplementation "org.awaitility:awaitility:4.1.0" testImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation "commons-io:commons-io:2.8.0" + testImplementation "commons-io:commons-io:2.11.0" } // Workaround for Werror @@ -70,21 +76,21 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.11.1000' + force 'com.amazonaws:aws-java-sdk-core:1.12.43' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' - force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' + force 'com.fasterxml.jackson.core:jackson-annotations:2.12.4' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' - force 'com.fasterxml.jackson.core:jackson-core:2.12.3' - force 'com.fasterxml.jackson:jackson-bom:2.12.3' - force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' + force 'com.fasterxml.jackson.core:jackson-core:2.12.4' + force 'com.fasterxml.jackson:jackson-bom:2.12.4' + force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.4' force 'commons-logging:commons-logging:1.2' force 'org.apache.httpcomponents:httpclient:4.5.10' force "org.hdrhistogram:HdrHistogram:2.1.12" force 'joda-time:joda-time:2.10.10' - force 'org.yaml:snakeyaml:1.28' - force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.3' + force 'org.yaml:snakeyaml:1.29' + force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.4' force 'junit:junit:4.13' - force "org.slf4j:slf4j-api:1.7.30" + force "org.slf4j:slf4j-api:1.7.32" force "org.apache.logging.log4j:log4j-api:2.14.1" force "org.apache.logging.log4j:log4j-core:2.14.1" } diff --git a/data-prepper-plugins/opensearch/odfe_security.md b/data-prepper-plugins/opensearch/odfe_security.md index 7fcff1f088..cf269b5e21 100644 --- a/data-prepper-plugins/opensearch/odfe_security.md +++ b/data-prepper-plugins/opensearch/odfe_security.md @@ -11,7 +11,7 @@ sink: password: "admin" ``` -or by using user credential assigned with a role that has the below required permissions. +or by using user credential assigned with a role that has the below required permissions. ### Cluster permissions @@ -31,4 +31,4 @@ Note that `indices:admin/template/*` need to be in cluster permissions. --------------- -With administrative privilege, one can create an internal user, a role and map the user to the role by following the ODFE [instructions](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/). \ No newline at end of file +With administrative privilege, one can create an internal user, a role and map the user to the role by following the ODFE [instructions](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/). diff --git a/data-prepper-plugins/opensearch/security.md b/data-prepper-plugins/opensearch/security.md index 1458d40eab..81a4fa3cd0 100644 --- a/data-prepper-plugins/opensearch/security.md +++ b/data-prepper-plugins/opensearch/security.md @@ -36,7 +36,7 @@ You should ensure that the credentials you configure have the required permissio } ] } -``` +``` Please check this [doc](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html) to know how to set IAM to your Elasticsearch domain, @@ -45,24 +45,24 @@ Please check this [doc](https://docs.aws.amazon.com/elasticsearch-service/latest The OpenSearch sink creates an [Index State Management (ISM)](https://opendistro.github.io/for-elasticsearch-docs/docs/ism/) policy for Trace Analytics indices but Amazon Elasticsearch Service allows only the `master user` to create an ISM policy. So, * If you use IAM for your master user in FGAC domain, configure the sink as below, - + ``` sink: opensearch: hosts: ["https://your-fgac-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true ``` -Run `aws configure` using the AWS CLI to set your credentials to the master IAM user. - +Run `aws configure` using the AWS CLI to set your credentials to the master IAM user. + * If you use internal database for your master user in FGAC domain, configure the sink as below, - + ``` sink: opensearch: hosts: ["https://your-fgac-amazon-elasticssearch-service-endpoint"] aws_sigv4: false username: "master-username" - password: "master-password" + password: "master-password" ``` Note: You can create a new IAM/internal user with `all_access` and use instead of the master IAM/internal user. diff --git a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/AwsRequestSigningApacheInterceptor.java new file mode 100644 index 0000000000..08c73f2346 --- /dev/null +++ b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/AwsRequestSigningApacheInterceptor.java @@ -0,0 +1,241 @@ +/* + * Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with + * the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package com.amazon.dataprepper.plugins.sink.opensearch; + +import org.apache.http.Header; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.regions.Region; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static org.apache.http.protocol.HttpCoreContext.HTTP_TARGET_HOST; + +/** + * An {@link HttpRequestInterceptor} that signs requests using any AWS {@link Signer} + * and {@link AwsCredentialsProvider}. + */ +final class AwsRequestSigningApacheInterceptor implements HttpRequestInterceptor { + + /** + * Constant to check content-length + */ + private static final String CONTENT_LENGTH = "content-length"; + /** + * Constant to check Zero content length + */ + private static final String ZERO_CONTENT_LENGTH = "0"; + /** + * Constant to check if host is the endpoint + */ + private static final String HOST = "host"; + + /** + * The service that we're connecting to. + */ + private final String service; + + /** + * The particular signer implementation. + */ + private final Signer signer; + + /** + * The source of AWS credentials for signing. + */ + private final AwsCredentialsProvider awsCredentialsProvider; + + /** + * The region signing region. + */ + private final Region region; + + /** + * + * @param service service that we're connecting to + * @param signer particular signer implementation + * @param awsCredentialsProvider source of AWS credentials for signing + * @param region signing region + */ + public AwsRequestSigningApacheInterceptor(final String service, + final Signer signer, + final AwsCredentialsProvider awsCredentialsProvider, + final Region region) { + this.service = Objects.requireNonNull(service); + this.signer = Objects.requireNonNull(signer); + this.awsCredentialsProvider = Objects.requireNonNull(awsCredentialsProvider); + this.region = Objects.requireNonNull(region); + } + + /** + * + * @param service service that we're connecting to + * @param signer particular signer implementation + * @param awsCredentialsProvider source of AWS credentials for signing + * @param region signing region + */ + public AwsRequestSigningApacheInterceptor(final String service, + final Signer signer, + final AwsCredentialsProvider awsCredentialsProvider, + final String region) { + this(service, signer, awsCredentialsProvider, Region.of(region)); + } + + /** + * {@inheritDoc} + */ + @Override + public void process(final HttpRequest request, final HttpContext context) + throws HttpException, IOException { + URIBuilder uriBuilder; + try { + uriBuilder = new URIBuilder(request.getRequestLine().getUri()); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI" , e); + } + + // Copy Apache HttpRequest to AWS Request + SdkHttpFullRequest.Builder requestBuilder = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.fromValue(request.getRequestLine().getMethod())) + .uri(buildUri(context, uriBuilder)); + + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest httpEntityEnclosingRequest = + (HttpEntityEnclosingRequest) request; + if (httpEntityEnclosingRequest.getEntity() != null) { + InputStream content = httpEntityEnclosingRequest.getEntity().getContent(); + requestBuilder.contentStreamProvider(() -> content); + } + } + requestBuilder.rawQueryParameters(nvpToMapParams(uriBuilder.getQueryParams())); + requestBuilder.headers(headerArrayToMap(request.getAllHeaders())); + + ExecutionAttributes attributes = new ExecutionAttributes(); + attributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, awsCredentialsProvider.resolveCredentials()); + attributes.putAttribute(AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME, service); + attributes.putAttribute(AwsSignerExecutionAttribute.SIGNING_REGION, region); + + // Sign it + SdkHttpFullRequest signedRequest = signer.sign(requestBuilder.build(), attributes); + + // Now copy everything back + request.setHeaders(mapToHeaderArray(signedRequest.headers())); + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest httpEntityEnclosingRequest = + (HttpEntityEnclosingRequest) request; + if (httpEntityEnclosingRequest.getEntity() != null) { + BasicHttpEntity basicHttpEntity = new BasicHttpEntity(); + basicHttpEntity.setContent(signedRequest.contentStreamProvider() + .orElseThrow(() -> new IllegalStateException("There must be content")) + .newStream()); + httpEntityEnclosingRequest.setEntity(basicHttpEntity); + } + } + } + + private URI buildUri(final HttpContext context, URIBuilder uriBuilder) throws IOException { + try { + HttpHost host = (HttpHost) context.getAttribute(HTTP_TARGET_HOST); + + if (host != null) { + uriBuilder.setHost(host.getHostName()); + uriBuilder.setScheme(host.getSchemeName()); + uriBuilder.setPort(host.getPort()); + } + + return uriBuilder.build(); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI", e); + } + } + + /** + * + * @param params list of HTTP query params as NameValuePairs + * @return a multimap of HTTP query params + */ + private static Map> nvpToMapParams(final List params) { + Map> parameterMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (NameValuePair nvp : params) { + List argsList = + parameterMap.computeIfAbsent(nvp.getName(), k -> new ArrayList<>()); + argsList.add(nvp.getValue()); + } + return parameterMap; + } + + /** + * @param headers modelled Header objects + * @return a Map of header entries + */ + private static Map> headerArrayToMap(final Header[] headers) { + Map> headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Header header : headers) { + if (!skipHeader(header)) { + headersMap.put(header.getName(), headersMap + .getOrDefault(header.getName(), + new LinkedList<>(Collections.singletonList(header.getValue())))); + } + } + return headersMap; + } + + /** + * @param header header line to check + * @return true if the given header should be excluded when signing + */ + private static boolean skipHeader(final Header header) { + return (CONTENT_LENGTH.equalsIgnoreCase(header.getName()) + && ZERO_CONTENT_LENGTH.equals(header.getValue())) // Strip Content-Length: 0 + || HOST.equalsIgnoreCase(header.getName()); // Host comes from endpoint + } + + /** + * @param mapHeaders Map of header entries + * @return modelled Header objects + */ + private static Header[] mapToHeaderArray(final Map> mapHeaders) { + Header[] headers = new Header[mapHeaders.size()]; + int i = 0; + for (Map.Entry> headerEntry : mapHeaders.entrySet()) { + for (String value : headerEntry.getValue()) { + headers[i++] = new BasicHeader(headerEntry.getKey(), value); + } + } + return headers; + } +} diff --git a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index d0baf0ed23..807720b2b3 100644 --- a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -12,10 +12,6 @@ package com.amazon.dataprepper.plugins.sink.opensearch; import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -33,6 +29,12 @@ import org.opensearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import javax.net.ssl.SSLContext; import java.io.InputStream; @@ -43,6 +45,7 @@ import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.util.List; +import java.util.UUID; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -62,6 +65,7 @@ public class ConnectionConfiguration { public static final String INSECURE = "insecure"; public static final String AWS_SIGV4 = "aws_sigv4"; public static final String AWS_REGION = "aws_region"; + public static final String AWS_STS_ROLE_ARN = "aws_sts_role_arn"; private final List hosts; private final String username; @@ -72,6 +76,8 @@ public class ConnectionConfiguration { private final boolean insecure; private final boolean awsSigv4; private final String awsRegion; + private final String awsStsRoleArn; + private final String pipelineName; public List getHosts() { return hosts; @@ -93,6 +99,14 @@ public String getAwsRegion() { return awsRegion; } + public String getAwsStsRoleArn() { + return awsStsRoleArn; + } + + public Path getCertPath() { + return certPath; + } + public Integer getSocketTimeout() { return socketTimeout; } @@ -111,6 +125,8 @@ private ConnectionConfiguration(final Builder builder) { this.insecure = builder.insecure; this.awsSigv4 = builder.awsSigv4; this.awsRegion = builder.awsRegion; + this.awsStsRoleArn = builder.awsStsRoleArn; + this.pipelineName = builder.pipelineName; } public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ @@ -118,6 +134,7 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); + builder.withPipelineName(pluginSetting.getPipelineName()); if (username != null) { builder = builder.withUsername(username); } @@ -135,7 +152,10 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe } builder.withAwsSigv4(pluginSetting.getBooleanOrDefault(AWS_SIGV4, false)); - builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); + if (builder.awsSigv4) { + builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); + builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); + } final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); @@ -148,8 +168,8 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe return builder.build(); } - public Path getCertPath() { - return certPath; + public String getPipelineName() { + return pipelineName; } public RestHighLevelClient createClient() { @@ -160,7 +180,7 @@ public RestHighLevelClient createClient() { i++; } final RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); - /** + /* * Given that this is a patch release, we will support only the IAM based access policy AES domains. * We will not support FGAC and Custom endpoint domains. This will be followed in the next version. */ @@ -186,12 +206,22 @@ private void attachSigV4(final RestClientBuilder restClientBuilder) { //if aws signing is enabled we will add AWSRequestSigningApacheInterceptor interceptor, //if not follow regular credentials process LOG.info("{} is set, will sign requests using AWSRequestSigningApacheInterceptor", AWS_SIGV4); - final AWS4Signer aws4Signer = new AWS4Signer(); - aws4Signer.setServiceName(SERVICE_NAME); - aws4Signer.setRegionName(awsRegion); - final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - final HttpRequestInterceptor httpRequestInterceptor = new AWSRequestSigningApacheInterceptor(SERVICE_NAME, aws4Signer, - credentialsProvider); + final Aws4Signer aws4Signer = Aws4Signer.create(); + AwsCredentialsProvider credentialsProvider; + if (awsStsRoleArn != null && !awsStsRoleArn.isEmpty()) { + credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .stsClient(StsClient.create()) + .refreshRequest(AssumeRoleRequest.builder() + .roleSessionName("Elasticsearch-Sink-" + UUID.randomUUID() + .toString()) + .roleArn(awsStsRoleArn) + .build()) + .build(); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } + final HttpRequestInterceptor httpRequestInterceptor = new AwsRequestSigningApacheInterceptor(SERVICE_NAME, aws4Signer, + credentialsProvider, awsRegion); restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.addInterceptorLast(httpRequestInterceptor); attachSSLContext(httpClientBuilder); @@ -262,6 +292,8 @@ public static class Builder { private boolean insecure; private boolean awsSigv4; private String awsRegion; + private String awsStsRoleArn; + private String pipelineName; public Builder(final List hosts) { @@ -316,6 +348,16 @@ public Builder withAwsRegion(final String awsRegion) { return this; } + public Builder withAWSStsRoleArn(final String awsStsRoleArn) { + this.awsStsRoleArn = awsStsRoleArn; + return this; + } + + public Builder withPipelineName(final String pipelineName) { + this.pipelineName = pipelineName; + return this; + } + public ConnectionConfiguration build() { return new ConnectionConfiguration(this); } diff --git a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 3ebf03e327..1d0c8a9af4 100644 --- a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -28,6 +28,10 @@ import org.opensearch.client.RestHighLevelClient; import org.opensearch.client.indices.CreateIndexRequest; import org.opensearch.client.indices.GetIndexRequest; +import org.opensearch.client.indices.GetIndexTemplatesRequest; +import org.opensearch.client.indices.GetIndexTemplatesResponse; +import org.opensearch.client.indices.IndexTemplateMetadata; +import org.opensearch.client.indices.IndexTemplatesExistRequest; import org.opensearch.client.indices.PutIndexTemplateRequest; import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -57,6 +61,7 @@ public class OpenSearchSink extends AbstractSink> { private static final Logger LOG = LoggerFactory.getLogger(OpenSearchSink.class); // Pulled from BulkRequest to make estimation of bytes consistent private static final int REQUEST_OVERHEAD = 50; + protected static final String INDEX_ALIAS_USED_AS_INDEX_ERROR = "Invalid alias name [%s], an index exists with the same name as the alias"; private BufferedWriter dlqWriter; private final OpenSearchSinkConfiguration esSinkConfig; @@ -80,22 +85,24 @@ public OpenSearchSink(final PluginSetting pluginSetting) { this.indexType = esSinkConfig.getIndexConfiguration().getIndexType(); this.documentIdField = esSinkConfig.getIndexConfiguration().getDocumentIdField(); try { - start(); + initialize(); } catch (final IOException e) { + this.shutdown(); throw new RuntimeException(e.getMessage(), e); } } - public void start() throws IOException { - LOG.info("Starting OpenSearch sink"); + public void initialize() throws IOException { + LOG.info("Initializing OpenSearch sink"); restHighLevelClient = esSinkConfig.getConnectionConfiguration().createClient(); final boolean isISMEnabled = IndexStateManagement.checkISMEnabled(restHighLevelClient); - final Optional policyIdOptional = isISMEnabled? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : Optional.empty(); + final Optional policyIdOptional = isISMEnabled ? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : + Optional.empty(); if (!esSinkConfig.getIndexConfiguration().getIndexTemplate().isEmpty()) { - createIndexTemplate(isISMEnabled, policyIdOptional.orElse(null)); + checkAndCreateIndexTemplate(isISMEnabled, policyIdOptional.orElse(null)); } final String dlqFile = esSinkConfig.getRetryConfiguration().getDlqFile(); - if ( dlqFile != null) { + if (dlqFile != null) { dlqWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND); } checkAndCreateIndex(); @@ -105,7 +112,7 @@ public void start() throws IOException { this::logFailure, pluginMetrics, bulkRequestSupplier); - LOG.info("Started OpenSearch sink"); + LOG.info("Initialized OpenSearch sink"); } @Override @@ -114,7 +121,7 @@ public void doOutput(final Collection> records) { return; } BulkRequest bulkRequest = bulkRequestSupplier.get(); - for (final Record record: records) { + for (final Record record : records) { final String document = record.getData(); final IndexRequest indexRequest = new IndexRequest().source(document, XContentType.JSON); try { @@ -157,9 +164,16 @@ private void flushBatch(final BulkRequest bulkRequest) { }); } - private void createIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { + private void checkAndCreateIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias(); - final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexAlias + "-index-template"); + final String indexTemplateName = indexAlias + "-index-template"; + + // Check existing index template version - only overwrite if version is less than or does not exist + if (!shouldCreateIndexTemplate(indexTemplateName)) { + return; + } + + final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexTemplateName); final boolean isRaw = indexType.equals(IndexConstants.RAW); if (isRaw) { putIndexTemplateRequest.patterns(Collections.singletonList(indexAlias + "-*")); @@ -169,16 +183,65 @@ private void createIndexTemplate(final boolean isISMEnabled, final String ismPol if (isISMEnabled) { IndexStateManagement.attachPolicy(esSinkConfig.getIndexConfiguration(), ismPolicyId, indexAlias); } + putIndexTemplateRequest.source(esSinkConfig.getIndexConfiguration().getIndexTemplate()); restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); } + // TODO: Unit tests for this (and for the rest of the class) + private boolean shouldCreateIndexTemplate(final String indexTemplateName) throws IOException { + final Optional indexTemplateMetadataOptional = getIndexTemplateMetadata(indexTemplateName); + if (indexTemplateMetadataOptional.isPresent()) { + final Integer existingTemplateVersion = indexTemplateMetadataOptional.get().version(); + LOG.info("Found version {} for existing index template {}", existingTemplateVersion, indexTemplateName); + + final int newTemplateVersion = (int) esSinkConfig.getIndexConfiguration().getIndexTemplate().getOrDefault("version", 0); + + if (existingTemplateVersion != null && existingTemplateVersion >= newTemplateVersion) { + LOG.info("Index template {} should not be updated, current version {} >= existing version {}", + indexTemplateName, + existingTemplateVersion, + newTemplateVersion); + return false; + + } else { + LOG.info("Index template {} should be updated from version {} to version {}", + indexTemplateName, + existingTemplateVersion, + newTemplateVersion); + return true; + } + } else { + LOG.info("Index template {} does not exist and should be created", indexTemplateName); + return true; + } + } + + private Optional getIndexTemplateMetadata(final String indexTemplateName) throws IOException { + final IndexTemplatesExistRequest existsRequest = new IndexTemplatesExistRequest(indexTemplateName); + final boolean exists = restHighLevelClient.indices().existsTemplate(existsRequest, RequestOptions.DEFAULT); + if (!exists) { + return Optional.empty(); + } + + final GetIndexTemplatesRequest request = new GetIndexTemplatesRequest(indexTemplateName); + final GetIndexTemplatesResponse response = restHighLevelClient.indices().getIndexTemplate(request, RequestOptions.DEFAULT); + + if (response.getIndexTemplates().size() == 1) { + return Optional.of(response.getIndexTemplates().get(0)); + } else { + throw new RuntimeException(String.format("Found multiple index templates (%s) result when querying for %s", + response.getIndexTemplates().size(), + indexTemplateName)); + } + } + private void checkAndCreateIndex() throws IOException { // Check alias exists final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias(); final boolean isRaw = indexType.equals(IndexConstants.RAW); - final boolean exists = isRaw? - restHighLevelClient.indices().existsAlias(new GetAliasesRequest().aliases(indexAlias), RequestOptions.DEFAULT): + final boolean exists = isRaw ? + restHighLevelClient.indices().existsAlias(new GetAliasesRequest().aliases(indexAlias), RequestOptions.DEFAULT) : restHighLevelClient.indices().exists(new GetIndexRequest(indexAlias), RequestOptions.DEFAULT); if (!exists) { // TODO: use date as suffix? @@ -198,8 +261,13 @@ private void checkAndCreateIndex() throws IOException { if (e.getMessage().contains("resource_already_exists_exception")) { // Do nothing - likely caused by a race condition where the resource was created // by another host before this host's restClient made its request + } else if (e.getMessage().contains(String.format(INDEX_ALIAS_USED_AS_INDEX_ERROR, indexAlias))) { + // TODO: replace IOException with custom data-prepper exception + throw new IOException( + String.format("An index exists with the same name as the reserved index alias name [%s], please delete or migrate the existing index", + indexAlias)); } else { - throw e; + throw new IOException(e); } } } @@ -221,7 +289,7 @@ private void logFailure(final DocWriteRequest docWriteRequest, final Throwabl } } else { LOG.warn("Document [{}] has failure: {}", docWriteRequest.toString(), failure); - }; + } } @Override diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-service-map-index-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-service-map-index-template.json index 399263ce28..99aa8ea796 100644 --- a/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-service-map-index-template.json +++ b/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-service-map-index-template.json @@ -1,4 +1,5 @@ { + "version": 0, "mappings": { "date_detection": false, "dynamic_templates": [ diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json index 25df9ff8e7..9fefc33c18 100644 --- a/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json +++ b/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json @@ -1,4 +1,5 @@ { + "version": 1, "mappings": { "date_detection": false, "dynamic_templates": [ @@ -11,11 +12,11 @@ } }, { - "attributes_map": { + "span_attributes_map": { "mapping": { "type":"keyword" }, - "path_match":"attributes.*" + "path_match":"span.attributes.*" } } ], diff --git a/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index 5756cd941f..7405af012e 100644 --- a/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -32,6 +32,7 @@ public class ConnectionConfigurationTests { private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); private final String TEST_USERNAME = "admin"; private final String TEST_PASSWORD = "admin"; + private final String TEST_PIPELINE_NAME = "Test-Pipeline"; private final Integer TEST_CONNECT_TIMEOUT = 5; private final Integer TEST_SOCKET_TIMEOUT = 10; private final String TEST_CERT_PATH = Objects.requireNonNull(getClass().getClassLoader().getResource("test-ca.pem")).getFile(); @@ -39,7 +40,7 @@ public class ConnectionConfigurationTests { @Test public void testReadConnectionConfigurationDefault() { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, false); + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -49,12 +50,13 @@ public void testReadConnectionConfigurationDefault() { assertNull(connectionConfiguration.getCertPath()); assertNull(connectionConfiguration.getConnectTimeout()); assertNull(connectionConfiguration.getSocketTimeout()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientDefault() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, false); + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -65,7 +67,7 @@ public void testCreateClientDefault() throws IOException { @Test public void testReadConnectionConfigurationNoCert() { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -74,12 +76,13 @@ public void testReadConnectionConfigurationNoCert() { assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); assertFalse(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientNoCert() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -90,7 +93,7 @@ public void testCreateClientNoCert() throws IOException { @Test public void testCreateClientInsecure() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, true); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -101,7 +104,7 @@ public void testCreateClientInsecure() throws IOException { @Test public void testCreateClientWithCertPath() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, TEST_CERT_PATH, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -112,7 +115,7 @@ public void testCreateClientWithCertPath() throws IOException { @Test public void testCreateClientWithAWSSigV4AndRegion() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, "us-west-2", null, false); + TEST_HOSTS, null, null, null, null, true, "us-west-2", null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-west-2", connectionConfiguration.getAwsRegion()); @@ -122,37 +125,52 @@ public void testCreateClientWithAWSSigV4AndRegion() throws IOException { @Test public void testCreateClientWithAWSSigV4DefaultRegion() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, false); + TEST_HOSTS, null, null, null, null, true, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); - assertTrue(connectionConfiguration.isAwsSigv4());; + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientWithAWSSigV4AndInsecure() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, true); + TEST_HOSTS, null, null, null, null, true, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientWithAWSSigV4AndCertPath() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, TEST_CERT_PATH, false); + TEST_HOSTS, null, null, null, null, true, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); - assertTrue(connectionConfiguration.isAwsSigv4());; + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); + } + + @Test + public void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { + final PluginSetting pluginSetting = generatePluginSetting( + TEST_HOSTS, null, null, null, null, true, null, "some-iam-role", TEST_CERT_PATH, false); + final ConnectionConfiguration connectionConfiguration = + ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals("some-iam-role", connectionConfiguration.getAwsStsRoleArn()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } private PluginSetting generatePluginSetting( final List hosts, final String username, final String password, final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String certPath, final boolean insecure) { + final String awsStsRoleArn, final String certPath, final boolean insecure) { final Map metadata = new HashMap<>(); metadata.put("hosts", hosts); metadata.put("username", username); @@ -163,9 +181,11 @@ private PluginSetting generatePluginSetting( if (awsRegion != null) { metadata.put("aws_region", awsRegion); } + metadata.put("aws_sts_role_arn", awsStsRoleArn); metadata.put("cert", certPath); metadata.put("insecure", insecure); - - return new PluginSetting("opensearch", metadata); + final PluginSetting pluginSetting = new PluginSetting("opensearch", metadata); + pluginSetting.setPipelineName(TEST_PIPELINE_NAME); + return pluginSetting; } } diff --git a/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 1c14810326..d8dcc6ad14 100644 --- a/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/test/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -72,8 +72,9 @@ public class OpenSearchSinkIT extends OpenSearchRestTestCase { private static final String PLUGIN_NAME = "opensearch"; private static final String PIPELINE_NAME = "integTestPipeline"; public List HOSTS = Arrays.stream(System.getProperty("tests.rest.cluster").split(",")) - .map(ip -> String.format("%s://%s", getProtocol(), ip)).collect(Collectors.toList()); - private static final String DEFAULT_TEMPLATE_FILE = "test-index-template.json"; + .map(ip -> String.format("%s://%s", getProtocol(), ip)).collect(Collectors.toList()); + private static final String TEST_TEMPLATE_V1_FILE = "test-index-template.json"; + private static final String TEST_TEMPLATE_V2_FILE = "test-index-template-v2.json"; private static final String DEFAULT_RAW_SPAN_FILE_1 = "raw-span-1.json"; private static final String DEFAULT_RAW_SPAN_FILE_2 = "raw-span-2.json"; private static final String DEFAULT_SERVICE_MAP_FILE = "service-map-1.json"; @@ -93,7 +94,7 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { final String index = String.format("%s-000001", indexAlias); final Map mappings = getIndexMappings(index); assertNotNull(mappings); - assertFalse((boolean)mappings.get("date_detection")); + assertFalse((boolean) mappings.get("date_detection")); sink.shutdown(); if (isOSBundle()) { @@ -124,6 +125,15 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { } } + public void testInstantiateSinkRawSpanReservedAliasAlreadyUsedAsIndex() throws IOException { + final String reservedIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexConstants.RAW); + final Request request = new Request(HttpMethod.PUT, reservedIndexAlias); + client().performRequest(request); + final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null); + assertThrows(String.format(ElasticsearchSink.INDEX_ALIAS_USED_AS_INDEX_ERROR, reservedIndexAlias), + RuntimeException.class, () -> new ElasticsearchSink(pluginSetting)); + } + public void testOutputRawSpanDefault() throws IOException, InterruptedException { final String testDoc1 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1); final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_2); @@ -140,7 +150,7 @@ public void testOutputRawSpanDefault() throws IOException, InterruptedException final List> retSources = getSearchResponseDocSources(expIndexAlias); assertEquals(2, retSources.size()); assertTrue(retSources.containsAll(Arrays.asList(expData1, expData2))); - assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String)expData1.get("spanId"))); + assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String) expData1.get("spanId"))); sink.shutdown(); // Verify metrics @@ -227,7 +237,7 @@ public void testInstantiateSinkServiceMapDefault() throws IOException { assertEquals(SC_OK, response.getStatusLine().getStatusCode()); final Map mappings = getIndexMappings(indexAlias); assertNotNull(mappings); - assertFalse((boolean)mappings.get("date_detection")); + assertFalse((boolean) mappings.get("date_detection")); sink.shutdown(); if (isOSBundle()) { @@ -239,8 +249,7 @@ public void testInstantiateSinkServiceMapDefault() throws IOException { public void testOutputServiceMapDefault() throws IOException, InterruptedException { final String testDoc = readDocFromFile(DEFAULT_SERVICE_MAP_FILE); final ObjectMapper mapper = new ObjectMapper(); - @SuppressWarnings("unchecked") - final Map expData = mapper.readValue(testDoc, Map.class); + @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc, Map.class); final List> testRecords = Collections.singletonList(new Record<>(testDoc)); final PluginSetting pluginSetting = generatePluginSetting(false, true, null, null); @@ -250,7 +259,7 @@ public void testOutputServiceMapDefault() throws IOException, InterruptedExcepti final List> retSources = getSearchResponseDocSources(expIndexAlias); assertEquals(1, retSources.size()); assertEquals(expData, retSources.get(0)); - assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String)expData.get("hashId"))); + assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String) expData.get("hashId"))); sink.shutdown(); // verify metrics @@ -269,7 +278,7 @@ public void testOutputServiceMapDefault() throws IOException, InterruptedExcepti public void testInstantiateSinkCustomIndex() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( - getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); + getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile); OpenSearchSink sink = new OpenSearchSink(pluginSetting); final Request request = new Request(HttpMethod.HEAD, testIndexAlias); @@ -282,10 +291,67 @@ public void testInstantiateSinkCustomIndex() throws IOException { sink.shutdown(); } + public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates() throws IOException { + final String testIndexAlias = "test-alias"; + final String expectedIndexTemplateName = testIndexAlias + "-index-template"; + final String testTemplateFileV1 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE).getFile(); + final String testTemplateFileV2 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V2_FILE).getFile(); + + // Create sink with template version 1 + PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1); + ElasticsearchSink sink = new ElasticsearchSink(pluginSetting); + + Request getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + Response getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + String responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer firstResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + assertEquals(Integer.valueOf(1), firstResponseVersion); + sink.shutdown(); + + // Create sink with template version 2 + pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV2); + sink = new ElasticsearchSink(pluginSetting); + + getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer secondResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + assertEquals(Integer.valueOf(2), secondResponseVersion); + sink.shutdown(); + + // Create sink with template version 1 again + pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1); + sink = new ElasticsearchSink(pluginSetting); + + getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer thirdResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + // Assert version 2 was not overwritten by version 1 + assertEquals(Integer.valueOf(2), thirdResponseVersion); + sink.shutdown(); + + } + public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( - getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); + getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(generateCustomRecord(testIdField, testId)); @@ -307,7 +373,8 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0); } - private PluginSetting generatePluginSetting(final boolean isRaw, final boolean isServiceMap, final String indexAlias, final String templateFilePath) { + private PluginSetting generatePluginSetting(final boolean isRaw, final boolean isServiceMap, final String indexAlias, + final String templateFilePath) { final Map metadata = new HashMap<>(); metadata.put(IndexConfiguration.TRACE_ANALYTICS_RAW_FLAG, isRaw); metadata.put(IndexConfiguration.TRACE_ANALYTICS_SERVICE_MAP_FLAG, isServiceMap); @@ -328,19 +395,19 @@ private PluginSetting generatePluginSetting(final boolean isRaw, final boolean i private Record generateCustomRecord(final String idField, final String documentId) throws IOException { return new Record<>( - Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .field(idField, documentId) - .endObject() - ) + Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .field(idField, documentId) + .endObject() + ) ); } private String readDocFromFile(final String filename) throws IOException { final StringBuilder jsonBuilder = new StringBuilder(); try (final InputStream inputStream = Objects.requireNonNull( - getClass().getClassLoader().getResourceAsStream(filename))){ + getClass().getClassLoader().getResourceAsStream(filename))) { final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); bufferedReader.lines().forEach(jsonBuilder::append); } @@ -348,9 +415,10 @@ private String readDocFromFile(final String filename) throws IOException { } private Boolean checkIsWriteIndex(final String responseBody, final String aliasName, final String indexName) throws IOException { - @SuppressWarnings("unchecked") final Map indexBlob = (Map)createParser(XContentType.JSON.xContent(), responseBody).map().get(indexName); - @SuppressWarnings("unchecked") final Map aliasesBlob = (Map)indexBlob.get("aliases"); - @SuppressWarnings("unchecked") final Map aliasBlob = (Map)aliasesBlob.get(aliasName); + @SuppressWarnings("unchecked") final Map indexBlob = (Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(indexName); + @SuppressWarnings("unchecked") final Map aliasesBlob = (Map) indexBlob.get("aliases"); + @SuppressWarnings("unchecked") final Map aliasBlob = (Map) aliasesBlob.get(aliasName); return (Boolean) aliasBlob.get("is_write_index"); } @@ -358,19 +426,19 @@ private Integer getDocumentCount(final String index, final String field, final S final Request request = new Request(HttpMethod.GET, index + "/_count"); if (field != null && value != null) { final String jsonEntity = Strings.toString( - XContentFactory.jsonBuilder().startObject() - .startObject("query") - .startObject("match") - .field(field, value) - .endObject() - .endObject() - .endObject() + XContentFactory.jsonBuilder().startObject() + .startObject("query") + .startObject("match") + .field(field, value) + .endObject() + .endObject() + .endObject() ); request.setJsonEntity(jsonEntity); } final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - return (Integer)createParser(XContentType.JSON.xContent(), responseBody).map().get("count"); + return (Integer) createParser(XContentType.JSON.xContent(), responseBody).map().get("count"); } private List> getSearchResponseDocSources(final String index) throws IOException { @@ -380,10 +448,11 @@ private List> getSearchResponseDocSources(final String index final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") final List hits = (List) ((Map)createParser(XContentType.JSON.xContent(), - responseBody).map().get("hits")).get("hits"); + @SuppressWarnings("unchecked") final List hits = + (List) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get("hits")).get("hits"); @SuppressWarnings("unchecked") final List> sources = hits.stream() - .map(hit -> (Map)((Map) hit).get("_source")) + .map(hit -> (Map) ((Map) hit).get("_source")) .collect(Collectors.toList()); return sources; } @@ -393,9 +462,9 @@ private Map getIndexMappings(final String index) throws IOExcept final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") - final Map mappings = (Map) ((Map)createParser(XContentType.JSON.xContent(), - responseBody).map().get(index)).get("mappings"); + @SuppressWarnings("unchecked") final Map mappings = + (Map) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(index)).get("mappings"); return mappings; } @@ -405,8 +474,7 @@ private String getIndexPolicyId(final String index) throws IOException { final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") - final String policyId = (String) ((Map)createParser(XContentType.JSON.xContent(), + @SuppressWarnings("unchecked") final String policyId = (String) ((Map) createParser(XContentType.JSON.xContent(), responseBody).map().get(index)).get("index.opendistro.index_state_management.policy_id"); return policyId; } diff --git a/data-prepper-plugins/opensearch/src/test/resources/raw-span-1.json b/data-prepper-plugins/opensearch/src/test/resources/raw-span-1.json index d2a2dc575f..ef3c514c21 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/raw-span-1.json +++ b/data-prepper-plugins/opensearch/src/test/resources/raw-span-1.json @@ -12,17 +12,17 @@ "startTime":"2020-08-20T05:40:46.041011600Z", "endTime":"2020-08-20T05:40:46.089556800Z", "durationInNanos": 48545200, - "attributes.http@status_code":200, - "attributes.net@peer@port":41168, - "attributes.servlet@path":"/logs", - "attributes.http@response_content_length":7, - "attributes.http@user_agent":"curl/7.54.0", - "attributes.http@flavor":"HTTP/1.1", - "attributes.servlet@context":"", - "attributes.http@url":"http://0.0.0.0:8087/logs", - "attributes.net@peer@ip":"172.29.0.1", - "attributes.http@method":"POST", - "attributes.http@client_ip":"172.29.0.1", + "span.attributes.http@status_code":200, + "span.attributes.net@peer@port":41168, + "span.attributes.servlet@path":"/logs", + "span.attributes.http@response_content_length":7, + "span.attributes.http@user_agent":"curl/7.54.0", + "span.attributes.http@flavor":"HTTP/1.1", + "span.attributes.servlet@context":"", + "span.attributes.http@url":"http://0.0.0.0:8087/logs", + "span.attributes.net@peer@ip":"172.29.0.1", + "span.attributes.http@method":"POST", + "span.attributes.http@client_ip":"172.29.0.1", "resource.attributes@telemetry@sdk@language":"java", "resource.attributes@telemetry@sdk@name":"opentelemetry", "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" diff --git a/data-prepper-plugins/opensearch/src/test/resources/raw-span-2.json b/data-prepper-plugins/opensearch/src/test/resources/raw-span-2.json index 57e06dea33..4580705132 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/raw-span-2.json +++ b/data-prepper-plugins/opensearch/src/test/resources/raw-span-2.json @@ -12,17 +12,17 @@ "startTime":"2020-08-20T05:40:46.042011600Z", "endTime":"2020-08-20T05:40:46.088556800Z", "durationInNanos": 46545200, - "attributes.http@status_code":200, - "attributes.net@peer@port":41168, - "attributes.servlet@path":"/logs", - "attributes.http@response_content_length":7, - "attributes.http@user_agent":"curl/7.54.0", - "attributes.http@flavor":"HTTP/1.1", - "attributes.servlet@context":"", - "attributes.http@url":"http://0.0.0.0:8087/logs", - "attributes.net@peer@ip":"172.29.0.1", - "attributes.http@method":"POST", - "attributes.http@client_ip":"172.29.0.1", + "span.attributes.http@status_code":200, + "span.attributes.net@peer@port":41168, + "span.attributes.servlet@path":"/logs", + "span.attributes.http@response_content_length":7, + "span.attributes.http@user_agent":"curl/7.54.0", + "span.attributes.http@flavor":"HTTP/1.1", + "span.attributes.servlet@context":"", + "span.attributes.http@url":"http://0.0.0.0:8087/logs", + "span.attributes.net@peer@ip":"172.29.0.1", + "span.attributes.http@method":"POST", + "span.attributes.http@client_ip":"172.29.0.1", "resource.attributes@telemetry@sdk@language":"java", "resource.attributes@telemetry@sdk@name":"opentelemetry", "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" diff --git a/data-prepper-plugins/opensearch/src/test/resources/raw-span-error.json b/data-prepper-plugins/opensearch/src/test/resources/raw-span-error.json index 522220fe17..3a7956523b 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/raw-span-error.json +++ b/data-prepper-plugins/opensearch/src/test/resources/raw-span-error.json @@ -7,14 +7,14 @@ "status":{}, "startTime":"2020-09-01T22:01:55.103171300Z", "endTime":"2020-09-01T22:01:55.113684900Z", - "attributes.component":"mysql", - "attributes.db.type":"sql", - "attributes.db.instance":"APM", - "attributes.db.statement":"INSERT INTO Inventory_Items (ItemId, TotalQty) VALUES (%(ItemId)s, %(Qty)s) ON DUPLICATE KEY UPDATE TotalQty = TotalQty + %(Qty)s", - "attributes.db.user":"root", - "attributes.net.peer.name":"mysql", - "attributes.net.peer.port":3306, - "attributes.db.statement.parameters":"{'ItemId': 'banana', 'Qty': '6'}", + "span.attributes.component":"mysql", + "span.attributes.db.type":"sql", + "span.attributes.db.instance":"APM", + "span.attributes.db.statement":"INSERT INTO Inventory_Items (ItemId, TotalQty) VALUES (%(ItemId)s, %(Qty)s) ON DUPLICATE KEY UPDATE TotalQty = TotalQty + %(Qty)s", + "span.attributes.db.statement.parameters":"{'ItemId': 'banana', 'Qty': '6'}", + "span.attributes.db.user":"root", + "span.attributes.net.peer.name":"mysql", + "span.attributes.net.peer.port":3306, "resource.attributes.service.name":"database", "resource.attributes.service.instance.id":"140683459769616", "resource.attributes.telemetry.sdk.name":"opentelemetry", diff --git a/data-prepper-plugins/opensearch/src/test/resources/test-index-template.json b/data-prepper-plugins/opensearch/src/test/resources/test-index-template.json index 16bb72d011..6b942a8ec2 100644 --- a/data-prepper-plugins/opensearch/src/test/resources/test-index-template.json +++ b/data-prepper-plugins/opensearch/src/test/resources/test-index-template.json @@ -1,4 +1,5 @@ { + "version": 1, "mappings": { "date_detection": false, "dynamic_templates": [ diff --git a/data-prepper-plugins/otel-trace-group-prepper/README.md b/data-prepper-plugins/otel-trace-group-prepper/README.md index c87315a00f..c493610d5d 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/README.md +++ b/data-prepper-plugins/otel-trace-group-prepper/README.md @@ -16,7 +16,7 @@ pipeline: cert: path/to/cert username: YOUR_USERNAME_HERE password: YOUR_PASSWORD_HERE -``` +``` See [odfe_security.md](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/elasticsearch/odfe_security.md) for detailed explanation. @@ -28,7 +28,7 @@ pipeline: prepper: - otel-trace-group-prepper: hosts: ["https://your-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true cert: path/to/cert insecure: false ``` @@ -40,9 +40,9 @@ See [security.md](https://github.com/opensearch-project/data-prepper/blob/main/d - `hosts`: A list of IP addresses of elasticsearch nodes. - `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that ODFE is using. -Default is null. +Default is null. -- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. +- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. @@ -61,7 +61,7 @@ Default is null. ## Developer Guide -This plugin is compatible with Java 8. See +This plugin is compatible with Java 8. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index 242fa222af..63c99448bb 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -23,13 +23,13 @@ repositories { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:opensearch') - testCompile project(':data-prepper-api').sourceSets.test.output - implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - implementation "io.micrometer:micrometer-core:1.6.6" - testImplementation "org.mockito:mockito-core:3.9.0" + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:opensearch') + testImplementation project(':data-prepper-api').sourceSets.test.output + implementation "org.opensearch.client:opensearch-rest-high-level-client:${es_version}" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" + implementation "io.micrometer:micrometer-core:1.7.2" + testImplementation "org.mockito:mockito-core:3.11.2" testImplementation "junit:junit:4.13.2" } \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/README.md b/data-prepper-plugins/otel-trace-raw-prepper/README.md index 74899d5142..bb99547d4b 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/README.md +++ b/data-prepper-plugins/otel-trace-raw-prepper/README.md @@ -1,6 +1,6 @@ # OTel Trace Raw Prepper -This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. +This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. ## Usages Example `.yaml` configuration @@ -11,7 +11,6 @@ prepper: ## Configuration -* `root_span_flush_delay`: An `int` represents the time interval in seconds to flush all the root spans in the prepper together with their descendants. Default to 30. * `trace_flush_interval`: An `int` represents the time interval in seconds to flush all the descendant spans without any root span. Default to 180. ## Metrics @@ -25,4 +24,4 @@ Apart from common metrics in [AbstractPrepper](https://github.com/opensearch-pro ## Developer Guide This plugin is compatible with Java 8. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index d0117173fe..989db83950 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -14,21 +14,21 @@ plugins { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') - compile 'commons-codec:commons-codec:1.15' - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + implementation 'commons-codec:commons-codec:1.15' + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.15.8' - implementation "com.linecorp.armeria:armeria:1.6.0" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation 'com.google.protobuf:protobuf-java-util:3.17.3' + implementation "com.linecorp.armeria:armeria:1.9.2" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' - testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation 'org.assertj:assertj-core:3.20.2' + testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.awaitility:awaitility:4.0.3" + testImplementation "org.awaitility:awaitility:4.1.0" } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java index bd5f233809..cf3dc0ccf5 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java @@ -22,12 +22,10 @@ import com.amazon.dataprepper.plugins.prepper.oteltrace.model.RawSpanSet; import com.amazon.dataprepper.plugins.prepper.oteltrace.model.TraceGroup; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableList; -import com.google.common.primitives.Ints; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.util.StringUtils; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; @@ -40,18 +38,15 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @DataPrepperPlugin(name = "otel_trace_raw_prepper", type = PluginType.PREPPER) public class OTelTraceRawPrepper extends AbstractPrepper, Record> { - private static final long SEC_TO_MILLIS = 1_000L; private static final Logger LOG = LoggerFactory.getLogger(OTelTraceRawPrepper.class); @@ -60,17 +55,11 @@ public class OTelTraceRawPrepper extends AbstractPrepper delayedParentSpanQueue = new DelayQueue<>(); - // TODO: replace with file store, e.g. MapDB? - // TODO: introduce a gauge to monitor the size private final Map traceIdRawSpanSetMap = new ConcurrentHashMap<>(); private final Cache traceIdTraceGroupCache; @@ -80,15 +69,13 @@ public class OTelTraceRawPrepper extends AbstractPrepper> doExecute(Collection> records) { + final List rawSpans = new LinkedList<>(); + for (Record ets : records) { for (ResourceSpans rs : ets.getData().getResourceSpansList()) { try { @@ -119,7 +108,8 @@ public Collection> doExecute(Collection> doExecute(Collection rawSpans = new LinkedList<>(); - rawSpans.addAll(getTracesToFlushByRootSpan()); rawSpans.addAll(getTracesToFlushByGarbageCollection()); return convertRawSpansToJsonRecords(rawSpans); } - private void processRawSpan(final RawSpan rawSpan) { - if (rawSpan.getParentSpanId() == null || "".equals(rawSpan.getParentSpanId())) { - processRootRawSpan(rawSpan); + /** + * Branching logic to handle root and child spans. + * A root span is the first span of a trace, it has no parentSpanId. + * + * @param rawSpan Span to be evaluated + * @param spanSet Collection to insert spans to + */ + private void processRawSpan(final RawSpan rawSpan, final Collection spanSet) { + if (StringUtils.isBlank(rawSpan.getParentSpanId())) { + final List rootSpanAndChildren = processRootSpan(rawSpan); + spanSet.addAll(rootSpanAndChildren); } else { - processDescendantRawSpan(rawSpan); + final Optional populatedChildSpanOptional = processChildSpan(rawSpan); + if (populatedChildSpanOptional.isPresent()) { + spanSet.add(populatedChildSpanOptional.get()); + } } } - private void processRootRawSpan(final RawSpan rawSpan) { - // TODO: flush descendants here to get rid of DelayQueue - // TODO: safe-guard against null traceGroup for rootSpan? - traceIdTraceGroupCache.put(rawSpan.getTraceId(), rawSpan.getTraceGroup()); - final long now = System.currentTimeMillis(); - final long nowPlusOffset = now + rootSpanFlushDelay; - final DelayedParentSpan delayedParentSpan = new DelayedParentSpan(rawSpan, nowPlusOffset); - delayedParentSpanQueue.add(delayedParentSpan); - } + /** + * Retrieves all child spans from memory and returns them as a set with the root span. + * Also adds an entry to the traceID cache so that later child spans can be tagged, + * in the case where a child span is processed AFTER the root span. + * + * @param parentSpan + * @return List containing root span, along with any child spans that have already been processed. + */ + private List processRootSpan(final RawSpan parentSpan) { + traceIdTraceGroupCache.put(parentSpan.getTraceId(), parentSpan.getTraceGroup()); + + final List recordsToFlush = new LinkedList<>(); + recordsToFlush.add(parentSpan); + + final TraceGroup traceGroup = parentSpan.getTraceGroup(); + final String parentSpanTraceId = parentSpan.getTraceId(); - private void processDescendantRawSpan(final RawSpan rawSpan) { - traceIdRawSpanSetMap.compute(rawSpan.getTraceId(), (traceId, rawSpanSet) -> { - if (rawSpanSet == null) { - rawSpanSet = new RawSpanSet(); + final RawSpanSet rawSpanSet = traceIdRawSpanSetMap.get(parentSpanTraceId); + if (rawSpanSet != null) { + for (final RawSpan rawSpan : rawSpanSet.getRawSpans()) { + rawSpan.setTraceGroup(traceGroup); + recordsToFlush.add(rawSpan); } - rawSpanSet.addRawSpan(rawSpan); - return rawSpanSet; - }); + + traceIdRawSpanSetMap.remove(parentSpanTraceId); + } + + return recordsToFlush; + } + + /** + * Attempts to populate the traceGroup of the child span by fetching from a cache. If the traceGroup is not in the cache, + * the child span is kept in memory to be populated when its corresponding root span arrives. + * + * @param childSpan + * @return Optional containing childSpan if its traceGroup is in memory, otherwise an empty Optional + */ + private Optional processChildSpan(final RawSpan childSpan) { + final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(childSpan.getTraceId()); + + if (traceGroup != null) { + childSpan.setTraceGroup(traceGroup); + return Optional.of(childSpan); + } else { + traceIdRawSpanSetMap.compute(childSpan.getTraceId(), (traceId, rawSpanSet) -> { + if (rawSpanSet == null) { + rawSpanSet = new RawSpanSet(); + } + rawSpanSet.addRawSpan(childSpan); + return rawSpanSet; + }); + + return Optional.empty(); + } } private List> convertRawSpansToJsonRecords(final List rawSpans) { @@ -185,51 +220,32 @@ private List> convertRawSpansToJsonRecords(final List ra return records; } - private List getTracesToFlushByRootSpan() { - final List recordsToFlush = new LinkedList<>(); - - for (DelayedParentSpan delayedParentSpan; (delayedParentSpan = delayedParentSpanQueue.poll()) != null;) { - RawSpan parentSpan = delayedParentSpan.getRawSpan(); - recordsToFlush.add(parentSpan); - - TraceGroup traceGroup = parentSpan.getTraceGroup(); - String parentSpanTraceId = parentSpan.getTraceId(); - - RawSpanSet rawSpanSet = traceIdRawSpanSetMap.get(parentSpanTraceId); - if (rawSpanSet != null) { - for (RawSpan rawSpan : rawSpanSet.getRawSpans()) { - rawSpan.setTraceGroup(traceGroup); - recordsToFlush.add(rawSpan); - } - - traceIdRawSpanSetMap.remove(parentSpanTraceId); - } - } - - return recordsToFlush; - } - - + /** + * Periodically flush spans from memory. Typically all spans of a trace are written + * once the trace's root span arrives, however some child spans my arrive after the root span. + * This method ensures "orphaned" child spans are eventually flushed from memory. + * @return List of RawSpans to be sent down the pipeline + */ private List getTracesToFlushByGarbageCollection() { final List recordsToFlush = new LinkedList<>(); if (shouldGarbageCollect()) { - boolean isLockAcquired = traceFlushLock.tryLock(); + final boolean isLockAcquired = traceFlushLock.tryLock(); if (isLockAcquired) { try { final long now = System.currentTimeMillis(); lastTraceFlushTime = now; - Iterator> entryIterator = traceIdRawSpanSetMap.entrySet().iterator(); + final Iterator> entryIterator = traceIdRawSpanSetMap.entrySet().iterator(); while (entryIterator.hasNext()) { - Map.Entry entry = entryIterator.next(); - String traceId = entry.getKey(); - TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(traceId); - RawSpanSet rawSpanSet = entry.getValue(); - long traceTime = rawSpanSet.getTimeSeen(); - if (now - traceTime >= traceFlushInterval) { - Set rawSpans = rawSpanSet.getRawSpans(); + final Map.Entry entry = entryIterator.next(); + final String traceId = entry.getKey(); + final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(traceId); + final RawSpanSet rawSpanSet = entry.getValue(); + final long traceTime = rawSpanSet.getTimeSeen(); + if (now - traceTime >= traceFlushInterval || isShuttingDown) { + final Set rawSpans = rawSpanSet.getRawSpans(); if (traceGroup != null) { rawSpans.forEach(rawSpan -> { rawSpan.setTraceGroup(traceGroup); @@ -258,11 +274,11 @@ private List getTracesToFlushByGarbageCollection() { } private boolean shouldGarbageCollect() { - return System.currentTimeMillis() - lastTraceFlushTime >= traceFlushInterval; + return System.currentTimeMillis() - lastTraceFlushTime >= traceFlushInterval || isShuttingDown; } /** - * Re-enqueues all spans with time "0" so that all will be available for consumption. + * Forces a flush of all spans in memory */ @Override public void prepareForShutdown() { @@ -270,16 +286,8 @@ public void prepareForShutdown() { if (isLockAcquired) { try { - LOG.info("Preparing for shutdown, re-enqueueing {} spans", delayedParentSpanQueue.size()); - Iterator iterator = delayedParentSpanQueue.iterator(); - List delayedParentSpanList = ImmutableList.copyOf(iterator); - - delayedParentSpanQueue.clear(); - - for (final DelayedParentSpan delayedParentSpan : delayedParentSpanList) { - final DelayedParentSpan newSpan = new DelayedParentSpan(delayedParentSpan.getRawSpan(), 0L); - delayedParentSpanQueue.add(newSpan); - } + LOG.info("Preparing for shutdown, will attempt to flush {} spans", traceIdRawSpanSetMap.size()); + isShuttingDown = true; } finally { prepareForShutdownLock.unlock(); } @@ -288,38 +296,11 @@ public void prepareForShutdown() { @Override public boolean isReadyForShutdown() { - return delayedParentSpanQueue.isEmpty() - && traceIdRawSpanSetMap.isEmpty(); + return traceIdRawSpanSetMap.isEmpty(); } @Override public void shutdown() { traceIdTraceGroupCache.cleanUp(); } - - class DelayedParentSpan implements Delayed { - private RawSpan rawSpan; - private long delayTime; - - public DelayedParentSpan(RawSpan rawSpan, long startTime) { - this.rawSpan = rawSpan; - this.delayTime = startTime; - } - - @Override - public long getDelay(TimeUnit unit) { - long diff = delayTime - System.currentTimeMillis(); - return unit.convert(diff, TimeUnit.MILLISECONDS); - } - - @Override - public int compareTo(Delayed o) { - return Ints.saturatedCast( - this.delayTime - ((DelayedParentSpan) o).delayTime); - } - - public RawSpan getRawSpan() { - return rawSpan; - } - } } diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java index 0093ccd24f..3b17e80321 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java @@ -14,8 +14,6 @@ public class OtelTraceRawPrepperConfig { static final String TRACE_FLUSH_INTERVAL = "trace_flush_interval"; static final long DEFAULT_TG_FLUSH_INTERVAL_SEC = 180L; - static final String ROOT_SPAN_FLUSH_DELAY = "root_span_flush_delay"; - static final long DEFAULT_ROOT_SPAN_FLUSH_DELAY_SEC = 30L; - static final long DEFAULT_TRACE_ID_TTL_SEC = 300L; + static final long DEFAULT_TRACE_ID_TTL_SEC = 15L; static final long MAX_TRACE_ID_CACHE_SIZE = 1000_000L; } diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java index 1cb3c7f454..dcc4555f71 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java @@ -19,8 +19,6 @@ import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -30,9 +28,7 @@ public final class OTelProtoHelper { - private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final BigDecimal MILLIS_TO_NANOS = new BigDecimal(1_000_000); - private static final BigDecimal SEC_TO_MILLIS = new BigDecimal(1_000); + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String SERVICE_NAME = "service.name"; private static final String SPAN_ATTRIBUTES = "span.attributes"; static final String RESOURCE_ATTRIBUTES = "resource.attributes"; @@ -147,10 +143,7 @@ public static Map getSpanStatusAttributes(final Status status) { } private static String convertUnixNanosToISO8601(final long unixNano) { - final BigDecimal nanos = new BigDecimal(unixNano); - final long epochSeconds = nanos.divide(MILLIS_TO_NANOS.multiply(SEC_TO_MILLIS), RoundingMode.DOWN).longValue(); - final int nanoAdj = nanos.remainder(MILLIS_TO_NANOS.multiply(SEC_TO_MILLIS)).intValue(); - return Instant.ofEpochSecond(epochSeconds, nanoAdj).toString(); + return Instant.ofEpochSecond(0L, unixNano).toString(); } public static String getStartTimeISO8601(final Span span) { diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java index 6c43404745..1b01255606 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java @@ -12,7 +12,6 @@ package com.amazon.dataprepper.plugins.prepper.oteltrace.model; import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java index f8d1a2edd5..8133664d3f 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java @@ -61,7 +61,6 @@ public class OTelTraceRawPrepperTest { private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final long TEST_TRACE_FLUSH_INTERVAL = 3L; - private static final long TEST_ROOT_SPAN_FLUSH_DELAY = 1L; private static final int TEST_CONCURRENCY_SCALE = 2; private static final String TEST_REQUEST_ONE_FULL_TRACE_GROUP_JSON_FILE = "sample-request-one-full-trace-group.json"; @@ -82,7 +81,6 @@ public void setup() { "OTelTrace", new HashMap() {{ put(OtelTraceRawPrepperConfig.TRACE_FLUSH_INTERVAL, TEST_TRACE_FLUSH_INTERVAL); - put(OtelTraceRawPrepperConfig.ROOT_SPAN_FLUSH_DELAY, TEST_ROOT_SPAN_FLUSH_DELAY); }}); pluginSetting.setPipelineName("pipelineOTelTrace"); pluginSetting.setProcessWorkers(TEST_CONCURRENCY_SCALE); @@ -141,12 +139,12 @@ public void testEmptySpans() { @Test public void testExportRequestFlushByParentSpan() throws IOException { final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE); - oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); - await().atMost(2 * TEST_ROOT_SPAN_FLUSH_DELAY, TimeUnit.SECONDS).untilAsserted(() -> { - final List> processedRecords = (List>) oTelTraceRawPrepper.doExecute(Collections.emptyList()); - Assertions.assertThat(processedRecords.size()).isEqualTo(6); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); - }); + final List> processedRecords = (List>)oTelTraceRawPrepper.doExecute( + Collections.singletonList(new Record<>(exportTraceServiceRequest)) + ); + + Assertions.assertThat(processedRecords.size()).isEqualTo(6); + Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); } @Test @@ -160,7 +158,7 @@ public void testExportRequestFlushByParentSpanMultiThread() throws IOException, for (Future>> future : futures) { processedRecords.addAll(future.get()); } - await().atMost(2 * TEST_ROOT_SPAN_FLUSH_DELAY, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(2 * TEST_TRACE_FLUSH_INTERVAL, TimeUnit.SECONDS).untilAsserted(() -> { List>>> futureList = submitExportTraceServiceRequests(Collections.emptyList()); for (Future>> future : futureList) { processedRecords.addAll(future.get()); @@ -208,7 +206,7 @@ public void testPrepareForShutdown() throws Exception { assertTrue(oTelTraceRawPrepper.isReadyForShutdown()); // Add records to memory/queue - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE); + final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_MISSING_ROOTS_JSON_FILE); oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); // Assert records exist in memory diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java index 6ed105c0c5..d3f87a908e 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java @@ -28,7 +28,6 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java index ce3eb293f0..12554b5721 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java @@ -21,8 +21,6 @@ import org.junit.Test; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index f58ad9f216..b001e9a592 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -1,4 +1,4 @@ -# OTel Trace Source +# OTel Trace Source This is a source which follows the [OTLP Protocol](https://github.com/open-telemetry/oteps/blob/master/text/0035-opentelemetry-protocol.md). This source supports ```OTLP/grpc``` and ```OTLP/HTTP```. Support for ```OTLP/HTTP+JSON``` is not complete due as the traceId and spanId will be ```base64``` and not ```HexString```. @@ -12,21 +12,28 @@ source: ## Configurations -* port(Optional) => An `int` represents the port Otel trace source is running on. Default is ```21890```. +* port(Optional) => An `int` represents the port Otel trace source is running on. Default is ```21890```. * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables a gRPC health check service under ```grpc.health.v1 / Health / Check```. Default is ```false```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. +* unframed_requests(Optional) => A boolean to enable requests not framed using the gRPC wire protocol. +* thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200`. +* max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500`. + +### SSL + * ssl(Optional) => A boolean enables TLS/SSL. Default is ```true```. -* sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path. Required if ```ssl``` is set to ```true``` -* sslKeyFile(Optional) => A `String` represents the SSL key file path. Required if ```ssl``` is set to ```true``` -* thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200` -* max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500` +* sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* sslKeyFile(Optional) => A `String` represents the SSL key file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* useAcmCertForSSL(Optional) => A boolean enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is ```false```. +* acmCertificateArn(Optional) => A `String` represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if ```useAcmCertForSSL``` is set to ```true```. +* awsRegion(Optional) => A `String` represents the AWS region to use ACM or S3. Required if ```useAcmCertForSSL``` is set to ```true``` or ```sslKeyCertChainFile``` and ```sslKeyFile``` is ```AWS S3 path```. ## Metrics ### Counter - `requestTimeouts`: measures total number of requests that time out. -- `requestsReceived`: measures total number of requests received by otel trace source. +- `requestsReceived`: measures total number of requests received by otel trace source. ## Developer Guide This plugin is compatible with Java 8. See diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 79f68a3e1b..880d509daf 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -14,24 +14,31 @@ plugins { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:blocking-buffer') - compile 'commons-codec:commons-codec:1.15' - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:blocking-buffer') + implementation 'commons-codec:commons-codec:1.15' + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.15.8' - implementation "com.linecorp.armeria:armeria:1.6.0" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.9.0" + implementation "commons-io:commons-io:2.11.0" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" + implementation 'com.google.protobuf:protobuf-java-util:3.17.3' + implementation "com.linecorp.armeria:armeria:1.9.2" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" + implementation "org.apache.commons:commons-lang3:3.12.0" + implementation "org.bouncycastle:bcprov-jdk15on:1.69" + implementation "org.bouncycastle:bcpkix-jdk15on:1.69" + testImplementation 'org.assertj:assertj-core:3.20.2' + testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation(platform('org.junit:junit-bom:5.7.1')) + testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects - testImplementation("org.mockito:mockito-core:3.9.0") - testImplementation("org.mockito:mockito-junit-jupiter:3.9.0") + testImplementation("org.mockito:mockito-core:3.11.2") + testImplementation("org.mockito:mockito-junit-jupiter:3.11.2") + testImplementation("commons-io:commons-io:2.10.0") } test { diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt b/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt new file mode 100644 index 0000000000..26c78d1411 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHTCCAYYCCQD4hqYeYDQZADANBgkqhkiG9w0BAQUFADBSMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCVFgxDzANBgNVBAcMBkF1c3RpbjEPMA0GA1UECgwGQW1hem9u +MRQwEgYDVQQLDAtEYXRhcHJlcHBlcjAgFw0yMTA2MjUxOTIzMTBaGA8yMTIxMDYw +MTE5MjMxMFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlRYMQ8wDQYDVQQHDAZB +dXN0aW4xDzANBgNVBAoMBkFtYXpvbjEUMBIGA1UECwwLRGF0YXByZXBwZXIwgZ8w +DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrb3YhdKbQ5PtLHall10iLZC9ZdDVrq +HOvqVSM8NHlL8f82gJ3l0n9k7hYc5eKisutaS9eDTmJ+Dnn8xn/qPSKTIq9Wh+OZ +O+e9YEEpI/G4F9KpGULgMyRg9sJK0GlZdEt9o5GJNJIJUkptJU5eiLuE0IV+jyJo +Nvm8OE6EJPqxAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAjgnX5n/Tt7eo9uakIGAb +uBhvYdR8JqKXqF9rjFJ/MIK7FdQSF/gCdjnvBhzLlZFK/Nb6MGKoSKm5Lcr75LgC +FyhIwp3WlqQksiMFnOypYVY71vqDgj6UKdMaOBgthsYhngj8lC+wsVzWqQvkJ2Qg +/GAIzJwiZfXiaevQHRk79qI= +-----END CERTIFICATE----- diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key b/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key new file mode 100644 index 0000000000..479b877131 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCq292IXSm0OT7Sx2pZddIi2QvWXQ1a6hzr6lUjPDR5S/H/NoCd +5dJ/ZO4WHOXiorLrWkvXg05ifg55/MZ/6j0ikyKvVofjmTvnvWBBKSPxuBfSqRlC +4DMkYPbCStBpWXRLfaORiTSSCVJKbSVOXoi7hNCFfo8iaDb5vDhOhCT6sQIDAQAB +AoGANrrhFqpJDpr7vcb1ER0Fp/YArbT27zVo+EUC6puBb41dQlQyFOImcHpjLaAq +H1PgnjU5cBp2hGQ+vOK0rwrYc/HNl6vfh6N3NbDptMiuoBafRJA9JzYourAM09BU +zmXyr61Yn3KHzx1PRwWe37icX93oXP3P0qHb3dI1ZF4jG0ECQQDU5N/a7ogoz2zn +ZssD6FvUOUQDsdBWdXmhUvg+YdZrV44e4xk+FVzwEONoRktEYKz9MFXlsgNHr445 +KRguHWcJAkEAzXQkwOkN8WID1wrwoobUIMbZSGAZzofwkKXgTTnllnT1qOQXuRbS +aCMejFEymBBef4aXP6N4+va2FKW/MF34aQJAO2oMl1sOoOUSrZngepy0VAwPUUCk +thxe74jqQu6nGpn6zd/vQYZQw6bS8Fz90H1yic6dilcd1znFZWp0lxoZkQJBALeI +xoBycRsuFQIYasi1q3AwUtBd0Q/3zkZZeBtk2hzjFMUwJaUZpxKSNOrialD/ZnuD +jz+xWBTRKe0d98JMX+kCQCmsJEj/HYQAC1GamZ7JQWogRSRF2KTgTWRaDXDxy0d4 +yUQgwHB+HZLFcbi1JEK6eIixCsX8iifrrkteh+1npJ0= +-----END RSA PRIVATE KEY----- diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key b/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key new file mode 100644 index 0000000000..285efc8d82 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key @@ -0,0 +1,17 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIICojAcBgoqhkiG9w0BDAEDMA4ECAd2FKZw2oGwAgIIAASCAoDTgiaXkazaotc7 +SxQK3bX34sEvdkLXg/O4ZpHTb0f4gLPxNhDe7ZPKrAS2TdywpSHT0189MVl+PIvw +4YQDaGVHL1SM5ukJu+PQkfQAMigdCJ+bUsG6hkrUDC74qYhHZHj1yVGavL6I4KHT +Ixh9IV2GMRS4m6HGJ17nYsdiTFFNdK++WcTMxbVWv3SNdKGZG79T32pjMxuIUPWr +3dB+ZXM+FSqwlBLZxPvvjlP6ETw7QXrlBHcQh1tHSh10bM+q0c5CktZoXLwpc+gv +ZsGXzpVjqFrAw4Vw0ikJl1mUCoGOtqqP0P6QWwbIJZBxNoO0MvWcXW+U3AGNFFze +nMR8UTXdga2l1Lx7pokQkWUpp48SDRjDx/RdZTRXCgtRcKuBcm0x2lxNILdwOzjJ +5GlKMvvc2OXXTnYqSCTqdfbuR3XBYmWgFki92D6JnVIYq+QJr5qj8IJDJ7mADQ1i +Za6PEJnrT641fLeSKRq7QiTydMQ3JXa9DFqUPwdZPPHLr/hC19sWHrq7gxvhkcLI +wrTtTIi8/iV4IVaiHk7YU83IM6sGExabQ3BRXcHMr+7i1vVxtEsFNC6ycTfJ8gpJ +YsnpXUQe912l5sk7GRSP1InNRF7kzMD0QeOAQ0UVfmsbHOPSXvD7fXkJWIb6N+zW +qCQYRmBwc7Bz2KZein5MLsMeNz2AWj/DcA2fMC+4+QtI0nF5BFtaR0V0npWhsbPu +3rj+AXipnvVhDIkfl8495j7ybCBj7HAZk8Ux8GmiZ3PGFO1C7XCQaLPWJ4Aw4Kb3 +EUqtVtpbwsCov5PDmMDXgz8qOxWMdQsP/dPF1HnVAg7SoFG9xA4nHdZ2WAFZwYtf +rRxEd7br +-----END ENCRYPTED PRIVATE KEY----- diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java new file mode 100644 index 0000000000..f34092450a --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java @@ -0,0 +1,7 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; + +public interface CertificateProvider { + Certificate getCertificate(); +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java new file mode 100644 index 0000000000..4c3237cf48 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java @@ -0,0 +1,52 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CertificateProviderFactory { + private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); + + final OTelTraceSourceConfig oTelTraceSourceConfig; + public CertificateProviderFactory(final OTelTraceSourceConfig oTelTraceSourceConfig) { + this.oTelTraceSourceConfig = oTelTraceSourceConfig; + } + + public CertificateProvider getCertificateProvider() { + // ACM Cert for SSL takes preference + if (oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("Using ACM certificate and private key for SSL/TLS."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final ClientConfiguration clientConfig = new ClientConfiguration() + .withThrottledRetries(true); + final AWSCertificateManager awsCertificateManager = AWSCertificateManagerClientBuilder.standard() + .withRegion(oTelTraceSourceConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig) + .build(); + return new ACMCertificateProvider(awsCertificateManager, oTelTraceSourceConfig.getAcmCertificateArn(), + oTelTraceSourceConfig.getAcmCertIssueTimeOutMillis(), oTelTraceSourceConfig.getAcmPrivateKeyPassword()); + } else if (oTelTraceSourceConfig.isSslCertAndKeyFileInS3()) { + LOG.info("Using S3 to fetch certificate and private key for SSL/TLS."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(oTelTraceSourceConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .build(); + return new S3CertificateProvider(s3Client, oTelTraceSourceConfig.getSslKeyCertChainFile(), oTelTraceSourceConfig.getSslKeyFile()); + } else { + LOG.info("Using local file system to get certificate and private key for SSL/TLS."); + return new FileCertificateProvider(oTelTraceSourceConfig.getSslKeyCertChainFile(), oTelTraceSourceConfig.getSslKeyFile()); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java new file mode 100644 index 0000000000..06de8bc8d6 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java @@ -0,0 +1,156 @@ +package com.amazon.dataprepper.plugins.certificate.acm; + +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.ExportCertificateRequest; +import com.amazonaws.services.certificatemanager.model.ExportCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.apache.commons.lang3.RandomStringUtils; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMEncryptedKeyPair; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.bc.BcPEMDecryptorProvider; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; +import org.bouncycastle.pkcs.PKCSException; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.Security; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; + +public class ACMCertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); + private static final long SLEEP_INTERVAL = 10000L; + private static final int PASSPHRASE_CHAR_COUNT = 36; + private static final String BOUNCY_CASTLE_PROVIDER = "BC"; + private static final Random SECURE_RANDOM = new SecureRandom(); + + private final AWSCertificateManager awsCertificateManager; + private final String acmArn; + private final long totalTimeout; + private final String passphrase; + + public ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, + final String acmArn, + final long totalTimeout, + final String passphrase) { + this.awsCertificateManager = Objects.requireNonNull(awsCertificateManager); + this.acmArn = Objects.requireNonNull(acmArn); + this.totalTimeout = Objects.requireNonNull(totalTimeout); + // Passphrase can be null. If null a random passphrase will be generated. + this.passphrase = passphrase; + Security.addProvider(new BouncyCastleProvider()); + } + + public Certificate getCertificate() { + ExportCertificateResult exportCertificateResult = null; + long timeSlept = 0L; + + // The private key from ACM is encrypted. Passphrase is the privateKey password that will be used to decrypt the + // private key. If it's not provided, generate a random password. The configured passphrase can + // be used to decrypt the private key manually using openssl commands for any inspection or debugging. + final String pkPassphrase = Optional.ofNullable(passphrase).orElse(generatePassphrase(PASSPHRASE_CHAR_COUNT)); + while (exportCertificateResult == null && timeSlept < totalTimeout) { + try { + final ExportCertificateRequest exportCertificateRequest = new ExportCertificateRequest() + .withCertificateArn(acmArn) + .withPassphrase(ByteBuffer.wrap(pkPassphrase.getBytes())); + exportCertificateResult = awsCertificateManager.exportCertificate(exportCertificateRequest); + + } catch (final RequestInProgressException ex) { + try { + Thread.sleep(SLEEP_INTERVAL); + } catch (InterruptedException iex) { + throw new RuntimeException(iex); + } + } catch (final ResourceNotFoundException | InvalidArnException ex) { + LOG.error("Exception retrieving the certificate with arn: {}", acmArn, ex); + throw ex; + } + timeSlept += SLEEP_INTERVAL; + } + if (exportCertificateResult != null) { + final String decryptedPrivateKey = getDecryptedPrivateKey(exportCertificateResult.getPrivateKey(), pkPassphrase); + return new Certificate(exportCertificateResult.getCertificate(), decryptedPrivateKey); + } else { + throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is" + + " %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); + } + } + + private String generatePassphrase(final int characterCount) { + String passphrase = RandomStringUtils.random( + characterCount, + 0, + 0, + true, + true, + null, + SECURE_RANDOM); + + return passphrase; + } + + private String getDecryptedPrivateKey(final String encryptedPrivateKey, final String keyPassword) { + try { + final PrivateKey rsaPrivateKey = encryptedPrivateKeyStringToPrivateKey(encryptedPrivateKey, keyPassword.toCharArray()); + return privateKeyAsString(rsaPrivateKey); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private String privateKeyAsString(final PrivateKey key) throws IOException { + final StringWriter sw = new StringWriter(); + try (JcaPEMWriter pw = new JcaPEMWriter(sw)) { + pw.writeObject(key); + } + return sw.toString(); + } + + private PrivateKey encryptedPrivateKeyStringToPrivateKey(final String encryptedPrivateKey, final char[] password) + throws IOException, PKCSException { + final PrivateKeyInfo pki; + try (final PEMParser pemParser = new PEMParser(new StringReader(encryptedPrivateKey))) { + final Object o = pemParser.readObject(); + if (o instanceof PKCS8EncryptedPrivateKeyInfo) { // encrypted private key in pkcs8-format + LOG.debug("key in pkcs8 encoding"); + final PKCS8EncryptedPrivateKeyInfo epki = (PKCS8EncryptedPrivateKeyInfo) o; + final JcePKCSPBEInputDecryptorProviderBuilder builder = + new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(BOUNCY_CASTLE_PROVIDER); + final InputDecryptorProvider idp = builder.build(password); + pki = epki.decryptPrivateKeyInfo(idp); + } else if (o instanceof PEMEncryptedKeyPair) { // encrypted private key in pkcs1-format + LOG.debug("key in pkcs1 encoding"); + final PEMEncryptedKeyPair epki = (PEMEncryptedKeyPair) o; + final PEMKeyPair pkp = epki.decryptKeyPair(new BcPEMDecryptorProvider(password)); + pki = pkp.getPrivateKeyInfo(); + } else if (o instanceof PEMKeyPair) { // unencrypted private key + LOG.debug("key unencrypted"); + final PEMKeyPair pkp = (PEMKeyPair) o; + pki = pkp.getPrivateKeyInfo(); + } else { + throw new PKCSException("Invalid encrypted private key class: " + o != null ? o.getClass().getName() : null); + } + final JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider(BOUNCY_CASTLE_PROVIDER); + return converter.getPrivateKey(pki); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java new file mode 100644 index 0000000000..f18892bac2 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java @@ -0,0 +1,38 @@ +package com.amazon.dataprepper.plugins.certificate.file; + +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +public class FileCertificateProvider implements CertificateProvider { + private final String certificateFilePath; + private final String privateKeyFilePath; + + public FileCertificateProvider(final String certificateFilePath, + final String privateKeyFilePath) { + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + this.privateKeyFilePath = Objects.requireNonNull(privateKeyFilePath); + } + + private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); + + public Certificate getCertificate() { + try { + final Path certFilePath = Path.of(certificateFilePath); + final Path pkFilePath = Path.of(privateKeyFilePath); + + final String certAsString = Files.readString(certFilePath); + final String privateKeyAsString = Files.readString(pkFilePath); + + return new Certificate(certAsString, privateKeyAsString); + } catch (final Exception ex) { + LOG.error("Error encountered while reading the certificate.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java new file mode 100644 index 0000000000..647eaa5ab9 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java @@ -0,0 +1,29 @@ +package com.amazon.dataprepper.plugins.certificate.model; + +import static java.util.Objects.requireNonNull; + +public class Certificate { + /** + * The base64 PEM-encoded certificate. + */ + private String certificate; + + /** + * The decrypted private key associated with the public key in the certificate. The key is output in PKCS #8 format + * and is base64 PEM-encoded. + */ + private String privateKey; + + public Certificate(final String certificate, final String privateKey) { + this.certificate = requireNonNull(certificate, "certificate must not be null"); + this.privateKey = requireNonNull(privateKey, "privateKey must not be null"); + } + + public String getCertificate() { + return certificate; + } + + public String getPrivateKey() { + return privateKey; + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java new file mode 100644 index 0000000000..f72ae4e303 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java @@ -0,0 +1,49 @@ +package com.amazon.dataprepper.plugins.certificate.s3; + +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class S3CertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(S3CertificateProvider.class); + private final AmazonS3 s3Client; + private final String certificateFilePath; + private final String privateKeyFilePath; + + public S3CertificateProvider(final AmazonS3 s3Client, + final String certificateFilePath, + final String privateKeyFilePath) { + this.s3Client = Objects.requireNonNull(s3Client); + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + this.privateKeyFilePath = Objects.requireNonNull(privateKeyFilePath); + } + + public Certificate getCertificate() { + final AmazonS3URI certificateS3URI = new AmazonS3URI(certificateFilePath); + final AmazonS3URI privateKeyS3URI = new AmazonS3URI(privateKeyFilePath); + final String certificate = getObjectWithKey(certificateS3URI.getBucket(), certificateS3URI.getKey()); + final String privateKey = getObjectWithKey(privateKeyS3URI.getBucket(), privateKeyS3URI.getKey()); + + return new Certificate(certificate, privateKey); + } + + private String getObjectWithKey(final String bucketName, final String key) { + + // Download the object + try (final S3Object s3Object = s3Client.getObject(bucketName, key)) { + LOG.info("Object with key \"{}\" downloaded.", key); + return IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8); + } catch (final Exception ex) { + LOG.error("Error encountered while processing the response from Amazon S3.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index 1abdcf5b19..ebd11c085a 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -14,16 +14,20 @@ import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.buffer.Buffer; import com.amazon.dataprepper.model.record.Record; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import io.micrometer.core.instrument.Counter; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.TimeoutException; public class OTelTraceGrpcService extends TraceServiceGrpc.TraceServiceImplBase { + private static final Logger LOG = LoggerFactory.getLogger(OTelTraceGrpcService.class); public static final String REQUEST_TIMEOUTS = "requestTimeouts"; public static final String REQUESTS_RECEIVED = "requestsReceived"; @@ -48,16 +52,24 @@ public OTelTraceGrpcService(int bufferWriteTimeoutInMillis, @Override public void export(ExportTraceServiceRequest request, StreamObserver responseObserver) { + requestsReceivedCounter.increment(); + + if (Context.current().isCancelled()) { + requestTimeoutCounter.increment(); + responseObserver.onError(Status.CANCELLED.withDescription("Cancelled by client").asRuntimeException()); + return; + } + try { - requestsReceivedCounter.increment(); buffer.write(new Record<>(request), bufferWriteTimeoutInMillis); responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build()); responseObserver.onCompleted(); } catch (TimeoutException e) { + LOG.error("Buffer is full, unable to write"); + requestTimeoutCounter.increment(); responseObserver .onError(Status.RESOURCE_EXHAUSTED.withDescription("Buffer is full, request timed out.") .asException()); - requestTimeoutCounter.increment(); } } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index fbfa9e04a2..cffa932317 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -18,6 +18,9 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.source.Source; +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProviderFactory; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.amazon.dataprepper.plugins.health.HealthGrpcService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; @@ -28,7 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -38,10 +42,19 @@ public class OTelTraceSource implements Source private final OTelTraceSourceConfig oTelTraceSourceConfig; private Server server; private final PluginMetrics pluginMetrics; + private final CertificateProviderFactory certificateProviderFactory; public OTelTraceSource(final PluginSetting pluginSetting) { oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + } + + // accessible only in the same package for unit test + OTelTraceSource(final PluginSetting pluginSetting, final CertificateProviderFactory certificateProviderFactory) { + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); + this.certificateProviderFactory = certificateProviderFactory; } @Override @@ -58,7 +71,6 @@ public void start(Buffer> buffer) { buffer, pluginMetrics )) - .addService(new HealthGrpcService()) .useClientTimeoutHeader(false); if (oTelTraceSourceConfig.hasHealthCheck()) { @@ -71,14 +83,22 @@ public void start(Buffer> buffer) { grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } + grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); + final ServerBuilder sb = Server.builder(); sb.service(grpcServiceBuilder.build()); sb.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); - if (oTelTraceSourceConfig.isSsl()) { - LOG.info("SSL/TLS is enabled"); - sb.https(oTelTraceSourceConfig.getPort()).tls(new File(oTelTraceSourceConfig.getSslKeyCertChainFile()), - new File(oTelTraceSourceConfig.getSslKeyFile())); + // ACM Cert for SSL takes preference + if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("SSL/TLS is enabled."); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + final Certificate certificate = certificateProvider.getCertificate(); + sb.https(oTelTraceSourceConfig.getPort()).tls( + new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) + ) + ); } else { sb.http(oTelTraceSourceConfig.getPort()); } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index 11f95499e9..4a87f412d6 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -13,28 +13,46 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; +import org.apache.commons.lang3.StringUtils; + public class OTelTraceSourceConfig { static final String REQUEST_TIMEOUT = "request_timeout"; static final String PORT = "port"; static final String SSL = "ssl"; + static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; + static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; static final String HEALTH_CHECK_SERVICE = "health_check_service"; static final String PROTO_REFLECTION_SERVICE = "proto_reflection_service"; static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; static final String SSL_KEY_FILE = "sslKeyFile"; + static final String ACM_CERT_ARN = "acmCertificateArn"; + static final String ACM_PRIVATE_KEY_PASSWORD = "acmPrivateKeyPassword"; + static final String AWS_REGION = "awsRegion"; static final String THREAD_COUNT = "thread_count"; static final String MAX_CONNECTION_COUNT = "max_connection_count"; + static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21890; static final int DEFAULT_THREAD_COUNT = 200; static final int DEFAULT_MAX_CONNECTION_COUNT = 500; static final boolean DEFAULT_SSL = true; + static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; + private static final String S3_PREFIX = "s3://"; private final int requestTimeoutInMillis; private final int port; private final boolean healthCheck; private final boolean protoReflectionService; + private final boolean enableUnframedRequests; private final boolean ssl; + private final boolean useAcmCertForSSL; + private final long acmCertIssueTimeOutMillis; private final String sslKeyCertChainFile; private final String sslKeyFile; + private final boolean sslCertAndKeyFileInS3; + private final String acmCertificateArn; + private final String acmPrivateKeyPassword; + private final String awsRegion; private final int threadCount; private final int maxConnectionCount; @@ -42,36 +60,76 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, final int port, final boolean healthCheck, final boolean protoReflectionService, + final boolean enableUnframedRequests, final boolean isSSL, + final boolean useAcmCertForSSL, + final long acmCertIssueTimeOutMillis, final String sslKeyCertChainFile, final String sslKeyFile, + final String acmCertificateArn, + final String acmPrivateKeyPassword, + final String awsRegion, final int threadCount, final int maxConnectionCount) { this.requestTimeoutInMillis = requestTimeoutInMillis; this.port = port; this.healthCheck = healthCheck; this.protoReflectionService = protoReflectionService; + this.enableUnframedRequests = enableUnframedRequests; this.ssl = isSSL; + this.useAcmCertForSSL = useAcmCertForSSL; + this.acmCertIssueTimeOutMillis = acmCertIssueTimeOutMillis; this.sslKeyCertChainFile = sslKeyCertChainFile; this.sslKeyFile = sslKeyFile; + this.acmCertificateArn = acmCertificateArn; + this.acmPrivateKeyPassword = acmPrivateKeyPassword; + this.awsRegion = awsRegion; this.threadCount = threadCount; this.maxConnectionCount = maxConnectionCount; - if (ssl && (sslKeyCertChainFile == null || sslKeyCertChainFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + boolean certAndKeyFileInS3 = false; + if (useAcmCertForSSL) { + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), acmCertificateArn, ACM_CERT_ARN); + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), awsRegion, AWS_REGION); + } else if(ssl) { + validateSSLCertificateFiles(); + certAndKeyFileInS3 = isSSLCertificateLocatedInS3(); + if (certAndKeyFileInS3) { + validateSSLArgument("The certificate and key files are located in S3", awsRegion, AWS_REGION); + } } - if (ssl && (sslKeyFile == null || sslKeyFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + this.sslCertAndKeyFileInS3 = certAndKeyFileInS3; + } + + private void validateSSLArgument(final String sslTypeMessage, final String argument, final String argumentName) { + if (StringUtils.isEmpty(argument)) { + throw new IllegalArgumentException(String.format("%s, %s can not be empty or null", sslTypeMessage, argumentName)); } } + private void validateSSLCertificateFiles() { + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyCertChainFile, SSL_KEY_CERT_FILE); + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyFile, SSL_KEY_FILE); + } + + private boolean isSSLCertificateLocatedInS3() { + return sslKeyCertChainFile.toLowerCase().startsWith(S3_PREFIX) && + sslKeyFile.toLowerCase().startsWith(S3_PREFIX); + } + public static OTelTraceSourceConfig buildConfig(final PluginSetting pluginSetting) { return new OTelTraceSourceConfig(pluginSetting.getIntegerOrDefault(REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT_MS), pluginSetting.getIntegerOrDefault(PORT, DEFAULT_PORT), pluginSetting.getBooleanOrDefault(HEALTH_CHECK_SERVICE, false), pluginSetting.getBooleanOrDefault(PROTO_REFLECTION_SERVICE, false), + pluginSetting.getBooleanOrDefault(ENABLE_UNFRAMED_REQUESTS, false), pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL), + pluginSetting.getBooleanOrDefault(USE_ACM_CERT_FOR_SSL, DEFAULT_USE_ACM_CERT_FOR_SSL), + pluginSetting.getLongOrDefault(ACM_CERT_ISSUE_TIME_OUT_MILLIS, DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS), pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null), pluginSetting.getStringOrDefault(SSL_KEY_FILE, null), + pluginSetting.getStringOrDefault(ACM_CERT_ARN, null), + pluginSetting.getStringOrDefault(ACM_PRIVATE_KEY_PASSWORD, null), + pluginSetting.getStringOrDefault(AWS_REGION, null), pluginSetting.getIntegerOrDefault(THREAD_COUNT, DEFAULT_THREAD_COUNT), pluginSetting.getIntegerOrDefault(MAX_CONNECTION_COUNT, DEFAULT_MAX_CONNECTION_COUNT)); } @@ -92,10 +150,22 @@ public boolean hasProtoReflectionService() { return protoReflectionService; } + public boolean enableUnframedRequests() { + return enableUnframedRequests; + } + public boolean isSsl() { return ssl; } + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis; + } + public String getSslKeyCertChainFile() { return sslKeyCertChainFile; } @@ -104,6 +174,22 @@ public String getSslKeyFile() { return sslKeyFile; } + public String getAcmCertificateArn() { + return acmCertificateArn; + } + + public String getAcmPrivateKeyPassword() { + return acmPrivateKeyPassword; + } + + public boolean isSslCertAndKeyFileInS3() { + return sslCertAndKeyFileInS3; + } + + public String getAwsRegion() { + return awsRegion; + } + public int getThreadCount() { return threadCount; } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java index c2d9e21aa6..16dba054cd 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java @@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; public class HealthGrpcServiceTest { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java new file mode 100644 index 0000000000..634dad17e9 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java @@ -0,0 +1,76 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import org.hamcrest.core.IsInstanceOf; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +public class CertificateProviderFactoryTest { + private OTelTraceSourceConfig oTelTraceSourceConfig; + private CertificateProviderFactory certificateProviderFactory; + + @Test + public void getCertificateProviderAcmProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); + } + + @Test + public void getCertificateProviderS3ProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("sslKeyCertChainFile", "s3://data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "s3://data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); + } + + @Test + public void getCertificateProviderFileProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java new file mode 100644 index 0000000000..54e0773815 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java @@ -0,0 +1,95 @@ +package com.amazon.dataprepper.plugins.certificate.acm; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ACMCertificateProviderTest { + private static final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + private static final long acmCertIssueTimeOutMillis = 2000L; + private static final String acmPrivateKeyPassword = "password"; + @Mock + private AWSCertificateManager awsCertificateManager; + + @Mock + private ExportCertificateResult exportCertificateResult; + + private ACMCertificateProvider acmCertificateProvider; + + @BeforeEach + public void beforeEach() { + acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, acmCertificateArn, acmCertIssueTimeOutMillis, acmPrivateKeyPassword); + } + + @Test + public void getACMCertificateWithEncryptedPrivateKeySuccess() throws IOException { + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path encryptedKeyFilePath = Path.of("data/certificate/test_encrypted_key.key"); + final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String encryptedKeyAsString = Files.readString(encryptedKeyFilePath); + final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); + when(exportCertificateResult.getCertificate()).thenReturn(certAsString); + when(exportCertificateResult.getPrivateKey()).thenReturn(encryptedKeyAsString); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + final Certificate certificate = acmCertificateProvider.getCertificate(); + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); + } + + @Test + public void getACMCertificateWithUnencryptedPrivateKeySuccess() throws IOException { + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); + when(exportCertificateResult.getCertificate()).thenReturn(certAsString); + when(exportCertificateResult.getPrivateKey()).thenReturn(decryptedKeyAsString); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + final Certificate certificate = acmCertificateProvider.getCertificate(); + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); + } + + @Test + public void getACMCertificateWithInvalidPrivateKeyException() { + when(exportCertificateResult.getPrivateKey()).thenReturn(UUID.randomUUID().toString()); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + assertThrows(RuntimeException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateRequestInProgressException() { + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new RequestInProgressException("Request in progress.")); + assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateResourceNotFoundException() { + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new ResourceNotFoundException("Resource not found.")); + assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateInvalidArnException() { + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new InvalidArnException("Invalid certificate arn.")); + assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java new file mode 100644 index 0000000000..7b0fbc9e44 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java @@ -0,0 +1,48 @@ +package com.amazon.dataprepper.plugins.certificate.file; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +public class FileCertificateProviderTest { + + private FileCertificateProvider fileCertificateProvider; + + @Test + public void getCertificateValidPathSuccess() throws IOException { + final String certificateFilePath = "data/certificate/test_cert.crt"; + final String privateKeyFilePath = "data/certificate/test_decrypted_key.key"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath, privateKeyFilePath); + + final Certificate certificate = fileCertificateProvider.getCertificate(); + + final Path certFilePath = Path.of(certificateFilePath); + final Path keyFilePath = Path.of(privateKeyFilePath); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(keyAsString)); + } + + @Test + public void getCertificateInvalidPathSuccess() { + final String certificateFilePath = "path_does_not_exit/test_cert.crt"; + final String privateKeyFilePath = "path_does_not_exit/test_decrypted_key.key"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath, privateKeyFilePath); + + Assertions.assertThrows(RuntimeException.class, () -> fileCertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java new file mode 100644 index 0000000000..cdf7c36dfe --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java @@ -0,0 +1,78 @@ +package com.amazon.dataprepper.plugins.certificate.s3; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class S3CertificateProviderTest { + @Mock + private AmazonS3 amazonS3; + + @Mock + private S3Object certS3Object; + + @Mock + private S3Object privateKeyS3Object; + + private S3CertificateProvider s3CertificateProvider; + + @Test + public void getCertificateValidKeyPathSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + final String privateKeyContent = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String certificatePath = UUID.randomUUID().toString(); + final String privateKeyPath = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + final String s3SslKeyFile = String.format("s3://%s/%s",bucketName, privateKeyPath); + + final InputStream certObjectStream = IOUtils.toInputStream(certificateContent, StandardCharsets.UTF_8); + final InputStream privateKeyObjectStream = IOUtils.toInputStream(privateKeyContent, StandardCharsets.UTF_8); + + when(certS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(certObjectStream,null)); + when(privateKeyS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(privateKeyObjectStream,null)); + + when(amazonS3.getObject(bucketName, certificatePath)).thenReturn(certS3Object); + when(amazonS3.getObject(bucketName, privateKeyPath)).thenReturn(privateKeyS3Object); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile, s3SslKeyFile); + + final Certificate certificate = s3CertificateProvider.getCertificate(); + + assertThat(certificate.getCertificate(), is(certificateContent)); + assertThat(certificate.getPrivateKey(), is(privateKeyContent)); + } + + @Test + public void getCertificateValidKeyPathS3Exception() { + final String certificatePath = UUID.randomUUID().toString(); + final String privateKeyPath = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + final String s3SslKeyFile = String.format("s3://%s/%s",bucketName, privateKeyPath); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile, s3SslKeyFile); + when(amazonS3.getObject(anyString(), anyString())).thenThrow(new RuntimeException("S3 exception")); + + Assertions.assertThrows(RuntimeException.class, () -> s3CertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 14c703c77e..cdfdba46f5 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -14,9 +14,13 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProviderFactory; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazon.dataprepper.plugins.health.HealthGrpcService; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpMethod; @@ -26,34 +30,48 @@ import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import io.grpc.BindableService; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import static com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -65,10 +83,25 @@ public class OTelTraceSourceTest { @Mock private Server server; + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + @Mock private CompletableFuture completableFuture; - PluginSetting pluginSetting; - PluginSetting testPluginSetting; + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; private BlockingBuffer> buffer; @@ -76,17 +109,11 @@ public class OTelTraceSourceTest { .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() .addSpans(Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); - private static OTelTraceSource SOURCE; + private OTelTraceSource SOURCE; private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); - private static TraceServiceGrpc.TraceServiceBlockingStub CLIENT; - - private static String getUri() { - return "gproto+http://127.0.0.1:" + SOURCE.getoTelTraceSourceConfig().getPort() + '/'; - } - private BlockingBuffer> getBuffer() { final HashMap integerHashMap = new HashMap<>(); integerHashMap.put("buffer_size", 1); @@ -98,20 +125,21 @@ private BlockingBuffer> getBuffer() { public void beforeEach() { lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); lenient().when(serverBuilder.build()).thenReturn(server); lenient().when(server.start()).thenReturn(completableFuture); - final HashMap settingsMap = new HashMap<>(); + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + + final Map settingsMap = new HashMap<>(); settingsMap.put("request_timeout", 5); settingsMap.put(SSL, false); pluginSetting = new PluginSetting("otel_trace", settingsMap); pluginSetting.setPipelineName("pipeline"); SOURCE = new OTelTraceSource(pluginSetting); - buffer = getBuffer(); - SOURCE.start(buffer); - - CLIENT = Clients.newClient(getUri(), TraceServiceGrpc.TraceServiceBlockingStub.class); } @AfterEach @@ -121,6 +149,7 @@ public void afterEach() { @Test void testHttpFullJson() throws InvalidProtocolBufferException { + SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -130,9 +159,7 @@ void testHttpFullJson() throws InvalidProtocolBufferException { .build(), HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); - }).join(); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -142,15 +169,54 @@ void testHttpFullJson() throws InvalidProtocolBufferException { .build(), HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415) //validateBuffer(); - }).join(); + ).join(); + } + + @Test + void testHttpsFullJson() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("request_timeout", 5); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + pluginSetting = new PluginSetting("otel_trace", settingsMap); + pluginSetting.setPipelineName("pipeline"); + SOURCE = new OTelTraceSource(pluginSetting); + buffer = getBuffer(); + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415) + //validateBuffer(); + ).join(); } @Test void testHttpFullBytes() { + SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -160,9 +226,7 @@ void testHttpFullBytes() { .build(), HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); - }).join(); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -178,13 +242,159 @@ void testHttpFullBytes() { }).join(); } + @Test + public void testServerStartCertFileSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate).isEqualTo(certAsString); + assertThat(actualPrivateKey).isEqualTo(keyAsString); + } + } + + @Test + public void testServerStartACMCertSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate).isEqualTo(certAsString); + assertThat(actualPrivateKey).isEqualTo(keyAsString); + } + } + + @Test + void start_with_Health_configured_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + } + + @Test + void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + } + @Test public void testDoubleStart() { + // starting server + SOURCE.start(buffer); + // double start server Assertions.assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); } @Test public void testRunAnotherSourceWithSamePort() { + // starting server + SOURCE.start(buffer); + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); testPluginSetting.setPipelineName("pipeline"); final OTelTraceSource source = new OTelTraceSource(testPluginSetting); @@ -205,7 +415,7 @@ public void testStartWithServerExecutionExceptionNoCause() throws ExecutionExcep // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); when(completableFuture.get()).thenThrow(new ExecutionException("", null)); // When/Then @@ -218,7 +428,7 @@ public void testStartWithServerExecutionExceptionWithCause() throws ExecutionExc // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); final NullPointerException expCause = new NullPointerException(); when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); @@ -233,7 +443,7 @@ public void testStopWithServerExecutionExceptionNoCause() throws ExecutionExcept // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); @@ -248,7 +458,7 @@ public void testStartWithInterruptedException() throws ExecutionException, Inter // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); when(completableFuture.get()).thenThrow(new InterruptedException()); // When/Then @@ -262,7 +472,7 @@ public void testStopWithServerExecutionExceptionWithCause() throws ExecutionExce // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); final NullPointerException expCause = new NullPointerException(); @@ -279,7 +489,7 @@ public void testStopWithInterruptedException() throws ExecutionException, Interr // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); when(completableFuture.get()).thenThrow(new InterruptedException()); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index 3c08c0c1d2..8f63f11602 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -64,6 +64,7 @@ public void testValidConfig() { TEST_PORT, true, true, + false, true, TEST_KEY_CERT, TEST_KEY, @@ -92,6 +93,7 @@ public void testInvalidConfig() { DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_PORT, false, false, + false, true, null, TEST_KEY, DEFAULT_THREAD_COUNT, @@ -105,6 +107,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, "", TEST_KEY, @@ -119,6 +122,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, TEST_KEY_CERT, null, @@ -133,6 +137,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, TEST_KEY_CERT, "", @@ -146,6 +151,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT final int port, final boolean healthCheck, final boolean protoReflectionService, + final boolean enableUnframedRequests, final boolean isSSL, final String sslKeyCertChainFile, final String sslKeyFile, @@ -156,6 +162,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.PORT, port); settings.put(OTelTraceSourceConfig.HEALTH_CHECK_SERVICE, healthCheck); settings.put(OTelTraceSourceConfig.PROTO_REFLECTION_SERVICE, protoReflectionService); + settings.put(OTelTraceSourceConfig.ENABLE_UNFRAMED_REQUESTS, enableUnframedRequests); settings.put(OTelTraceSourceConfig.SSL, isSSL); settings.put(OTelTraceSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); diff --git a/data-prepper-plugins/peer-forwarder/README.md b/data-prepper-plugins/peer-forwarder/README.md index 4e443ef8a4..e509bde0da 100644 --- a/data-prepper-plugins/peer-forwarder/README.md +++ b/data-prepper-plugins/peer-forwarder/README.md @@ -1,5 +1,5 @@ # Peer Forwarder -This prepper forwards `ExportTraceServiceRequests` via gRPC to other Data Prepper instances. +This prepper forwards `ExportTraceServiceRequests` via gRPC to other Data Prepper instances. ## Usage The primary usecase of this prepper is to ensure that groups of traces are aggregated by trace ID and processed by the same Prepper instance. @@ -26,7 +26,8 @@ prepper: discovery_mode: "dns" domain_name: "data-prepper-cluster.my-domain.net" ``` -For DNS cluster setup, see [Operating a Cluster with DNS Discovery](#DNS_Discovery). +For DNS cluster setup, see [Operating a Cluster with DNS Discovery](#DNS_Discovery). To +setup for AWS Cloud Map, see [Using AWS CloudMap](#AWS_CloudMap_Discovery). ### Operating a Cluster with DNS Discovery DNS discovery is recommended when scaling out a Data Prepper cluster. The core concept is to configure a DNS provider to return a list of Data Prepper hosts when given a single domain name. @@ -42,15 +43,70 @@ A DNS server (like [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html)) can #### With Amazon Route 53 Private Hosted Zones [Private hosted zones](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/hosted-zones-private.html) enable Amazon Route 53 to "respond to DNS queries for a domain and its subdomains within one or more VPCs that you create with the Amazon VPC service." Similar to the custom DNS server approach, except that Route 53 maintains the list of Data Prepper hosts. Suffers from the same drawback in that the list must be manually kept up-to-date. +### Using AWS CloudMap + +An alternative to DNS discovery is [AWS CloudMap](https://docs.aws.amazon.com/cloud-map/latest/dg/what-is-cloud-map.html). +CloudMap provides API-based service discovery as well as DNS-based service discovery. + +Peer forwarder can use the API-based service discovery. To support this you must have an existing +namespace configured for API instance discovery. You can create a new one following the instructions +provided by the [CloudMap documentation](https://docs.aws.amazon.com/cloud-map/latest/dg/working-with-namespaces.html). + +Your pipeline configuration needs to include: + +* `awsCloudMapNamespaceName` - Set to your CloudMap Namespace name +* `awsCloudMapServiceName` - Set to the service name within your specified Namespace +* `awsRegion` - The AWS region where your namespace exists. +* `discovery_mode` - Set to `aws_cloud_map` + +Example configuration: + +``` + prepper: + - peer_forwarder: + discovery_mode: aws_cloud_map + awsCloudMapNamespaceName: my-namespace + awsCloudMapServiceName: data-prepper-cluster + awsRegion: us-east-1 +``` + +The DataPrepper must also be running with the necessary permissions. The following +IAM policy shows the necessary permissions. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CloudMapPeerForwarder", + "Effect": "Allow", + "Action": "servicediscovery:DiscoverInstances", + "Resource": "*" + } + ] +} +``` + ## Configuration * `time_out`: timeout in seconds for sending `ExportTraceServiceRequest`. Defaults to 3 seconds. * `span_agg_count`: batch size for number of spans per `ExportTraceServiceRequest`. Defaults to 48. -* `discovery_mode`: peer discovery mode to be used. Allowable values are `static` and `dns`. Defaults to `static` +* `target_port`: the destination port to forward requests to. Defaults to `21890` +* `discovery_mode`: peer discovery mode to be used. Allowable values are `static`, `dns`, and `aws_cloud_map`. Defaults to `static` * `static_endpoints`: list containing endpoints of all Data Prepper instances. * `domain_name`: single domain name to query DNS against. Typically used by creating multiple [DNS A Records](https://www.cloudflare.com/learning/dns/dns-records/dns-a-record/) for the same domain. -* `ssl` => Default is ```true```. -* `sslKeyCertChainFile` => Should be provided if ```ssl``` is set to ```true``` +* `awsCloudMapNamespaceName` - specifies the CloudMap namespace when using AWS CloudMap service discovery +* `awsCloudMapServiceName` - specifies the CloudMap service when using AWS CloudMap service discovery + +### SSL +The SSL configuration for setting up trust manager for peer forwarding client to connect to other Data Prepper instances. The SSL configuration should be same as the one used for OTel Trace Source. + +* `ssl(Optional)` => A boolean enables TLS/SSL. Default is ```true```. +* `sslKeyCertChainFile(Optional)` => A `String` represents the SSL certificate chain file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* `useAcmCertForSSL(Optional)` => A boolean enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is ```false```. +* `acmCertificateArn(Optional)` => A `String` represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if ```useAcmCertForSSL``` is set to ```true```. +* `awsRegion(Optional)` => A `String` represents the AWS region to use ACM or S3. Required if ```useAcmCertForSSL``` is set to ```true``` or ```sslKeyCertChainFile``` and ```sslKeyFile``` is ```AWS S3 path```. + ## Metrics @@ -58,20 +114,20 @@ Besides common metrics in [AbstractPrepper](https://github.com/opensearch-projec ### Timer -- `forwardRequestLatency`: measures latency in forwarding each request. +- `latency`: measures latency of forwarded requests. ### Counter -- `forwardRequestSuccess`: measures number of successfully sent requests. -- `forwardRequestErrors`: measures number of failed requests. +- `requests`: measures total number of forwarded requests. +- `errors`: measures number of failed requests. ### Gauge -- `peerEndpoints`: measures number of dynamically discovered peer data-prepper endpoints. For `static` mode, the size is fixed. +- `peerEndpoints`: measures number of dynamically discovered peer data-prepper endpoints. For `static` mode, the size is fixed. ## Developer Guide -This plugin is compatible with Java 14. See +This plugin is compatible with Java 14. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 4289e022dc..94f13d4d48 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -21,14 +21,31 @@ repositories { } dependencies { - compile project(':data-prepper-api') - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "com.linecorp.armeria:armeria:1.6.0" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" + implementation "com.linecorp.armeria:armeria:1.9.2" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" + implementation(platform('software.amazon.awssdk:bom:2.17.15')) + implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" + implementation 'software.amazon.awssdk:servicediscovery' + implementation "commons-io:commons-io:2.11.0" + implementation "org.apache.commons:commons-lang3:3.12.0" implementation "commons-validator:commons-validator:1.7" - testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation(platform('org.junit:junit-bom:5.7.2')) + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.junit.vintage:junit-vintage-engine' + testImplementation "org.hamcrest:hamcrest:2.2" + testImplementation "org.mockito:mockito-inline:3.11.2" + testImplementation "org.mockito:mockito-core:3.11.2" + testImplementation 'org.mockito:mockito-junit-jupiter:3.11.2' + testImplementation "commons-io:commons-io:2.10.0" + testImplementation 'org.awaitility:awaitility:4.1.0' +} + +test { + useJUnitPlatform() } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java index 624bac447e..0c0a9a31f3 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java @@ -11,12 +11,14 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.linecorp.armeria.client.ClientBuilder; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.Clients; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import java.io.File; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -27,26 +29,34 @@ public class PeerClientPool { private static final PeerClientPool INSTANCE = new PeerClientPool(); private final Map peerClients; + private int port; private int clientTimeoutSeconds = 3; private boolean ssl; - private File sslKeyCertChainFile; + private Certificate certificate; private PeerClientPool() { peerClients = new ConcurrentHashMap<>(); } public static PeerClientPool getInstance() { + // TODO: remove singleton now that port is configurable return INSTANCE; } public void setClientTimeoutSeconds(int clientTimeoutSeconds) { this.clientTimeoutSeconds = clientTimeoutSeconds; } + public void setSsl(boolean ssl) { this.ssl = ssl; } - public void setSslKeyCertChainFile(File sslKeyCertChainFile) { - this.sslKeyCertChainFile = sslKeyCertChainFile; + + public void setPort(int port) { + this.port = port; + } + + public void setCertificate(final Certificate certificate) { + this.certificate = certificate; } public TraceServiceGrpc.TraceServiceBlockingStub getClient(final String address) { @@ -58,14 +68,20 @@ private TraceServiceGrpc.TraceServiceBlockingStub createGRPCClient(final String // TODO: replace hardcoded port with customization final ClientBuilder clientBuilder; if (ssl) { - clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTPS, ipAddress)) + clientBuilder = Clients.builder(String.format("%s://%s:%s/", GRPC_HTTPS, ipAddress, port)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)) .factory(ClientFactory.builder() - .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager(sslKeyCertChainFile)).build()); + .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager( + new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)) + ) + ).tlsNoVerifyHosts(ipAddress) + .build() + ); } else { clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTP, ipAddress)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)); } + return clientBuilder.build(TraceServiceGrpc.TraceServiceBlockingStub.class); } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java index 08bdca54c1..659b2fb6cf 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java @@ -34,12 +34,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @DataPrepperPlugin(name = "peer_forwarder", type = PluginType.PREPPER) public class PeerForwarder extends AbstractPrepper, Record> { - public static final String FORWARD_REQUEST_LATENCY_PREFIX = "forwardRequestLatency"; - public static final String FORWARD_REQUEST_SUCCESS_PREFIX = "forwardRequestSuccess"; - public static final String FORWARD_REQUEST_ERRORS_PREFIX = "forwardRequestErrors"; + public static final String REQUESTS = "requests"; + public static final String LATENCY = "latency"; + public static final String ERRORS = "errors"; + public static final String DESTINATION = "destination"; + + public static final int ASYNC_REQUEST_THREAD_COUNT = 200; private static final Logger LOG = LoggerFactory.getLogger(PeerForwarder.class); @@ -51,6 +59,8 @@ public class PeerForwarder extends AbstractPrepper forwardedRequestCounters; private final Map forwardRequestErrorCounters; + private final ExecutorService executorService; + public PeerForwarder(final PluginSetting pluginSetting, final PeerClientPool peerClientPool, final HashRing hashRing, @@ -59,9 +69,11 @@ public PeerForwarder(final PluginSetting pluginSetting, this.peerClientPool = peerClientPool; this.hashRing = hashRing; this.maxNumSpansPerRequest = maxNumSpansPerRequest; - forwardedRequestCounters = new HashMap<>(); - forwardRequestErrorCounters = new HashMap<>(); - forwardRequestTimers = new HashMap<>(); + forwardedRequestCounters = new ConcurrentHashMap<>(); + forwardRequestErrorCounters = new ConcurrentHashMap<>(); + forwardRequestTimers = new ConcurrentHashMap<>(); + + executorService = Executors.newFixedThreadPool(ASYNC_REQUEST_THREAD_COUNT); } public PeerForwarder(final PluginSetting pluginSetting) { @@ -94,8 +106,8 @@ public List> doExecute(final Collection> results = new ArrayList<>(); + final List> recordsToProcessLocally = new ArrayList<>(); + final List> forwardedRequestFutures = new ArrayList<>(); for (final Map.Entry> entry : groupedRS.entrySet()) { final TraceServiceGrpc.TraceServiceBlockingStub client; @@ -112,8 +124,11 @@ public List> doExecute(final Collection= maxNumSpansPerRequest) { final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - // Send the batch request to designated remote peer or ingest into localhost - processRequest(client, currRequest, results); + if (client == null) { + recordsToProcessLocally.add(new Record<>(currRequest)); + } else { + forwardedRequestFutures.add(processRequest(client, currRequest)); + } currRequestBuilder = ExportTraceServiceRequest.newBuilder(); currSpansCount = 0; } @@ -123,37 +138,57 @@ public List> doExecute(final Collection 0) { final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - processRequest(client, currRequest, results); + if (client == null) { + recordsToProcessLocally.add(new Record<>(currRequest)); + } else { + forwardedRequestFutures.add(processRequest(client, currRequest)); + } } } - return results; + + for (final CompletableFuture future : forwardedRequestFutures) { + try { + final Record record = future.get(); + if (record != null) { + recordsToProcessLocally.add(record); + } + } catch (InterruptedException | ExecutionException e) { + LOG.error("Problem with asynchronous peer forwarding", e); + } + } + + return recordsToProcessLocally; } /** - * Forward request to the peer address if client is given, otherwise push the request to local buffer. + * Asynchronously forwards a request to the peer address. Returns a record with an empty payload if + * the request succeeds, otherwise the payload will contain the failed ExportTraceServiceRequest to + * be processed locally. */ - private void processRequest(final TraceServiceGrpc.TraceServiceBlockingStub client, - final ExportTraceServiceRequest request, - final List> localBuffer) { - if (client != null) { - final String peerIp = client.getChannel().authority(); - final Timer forwardRequestTimer = forwardRequestTimers.computeIfAbsent( - peerIp, ip -> pluginMetrics.timer(String.format("%s:%s", FORWARD_REQUEST_LATENCY_PREFIX, ip))); - final Counter forwardedRequestCounter = forwardedRequestCounters.computeIfAbsent( - peerIp, ip -> pluginMetrics.counter(String.format("%s:%s", FORWARD_REQUEST_SUCCESS_PREFIX, ip))); - final Counter forwardRequestErrorCounter = forwardRequestErrorCounters.computeIfAbsent( - peerIp, ip -> pluginMetrics.counter(String.format("%s:%s", FORWARD_REQUEST_ERRORS_PREFIX, ip))); + private CompletableFuture processRequest(final TraceServiceGrpc.TraceServiceBlockingStub client, + final ExportTraceServiceRequest request) { + final String peerIp = client.getChannel().authority(); + final Timer forwardRequestTimer = forwardRequestTimers.computeIfAbsent( + peerIp, ip -> pluginMetrics.timerWithTags(LATENCY, DESTINATION, ip)); + final Counter forwardedRequestCounter = forwardedRequestCounters.computeIfAbsent( + peerIp, ip -> pluginMetrics.counterWithTags(REQUESTS, DESTINATION, ip)); + final Counter forwardRequestErrorCounter = forwardRequestErrorCounters.computeIfAbsent( + peerIp, ip -> pluginMetrics.counterWithTags(ERRORS, DESTINATION, ip)); + + final CompletableFuture callFuture = CompletableFuture.supplyAsync(() -> + { + forwardedRequestCounter.increment(); try { forwardRequestTimer.record(() -> client.export(request)); - forwardedRequestCounter.increment(); + return null; } catch (Exception e) { - LOG.error(String.format("Failed to forward the request:\n%s\n", request.toString())); + LOG.error("Failed to forward request to address: {}", peerIp, e); forwardRequestErrorCounter.increment(); - localBuffer.add(new Record<>(request)); + return new Record<>(request); } - } else { - localBuffer.add(new Record<>(request)); - } + }, executorService); + + return callFuture; } private boolean isAddressDefinedLocally(final String address) { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java index b98caf8ac2..032bd6b731 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java @@ -12,25 +12,35 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProviderConfig; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProviderFactory; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.PeerListProvider; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.PeerListProviderFactory; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Paths; +import org.apache.commons.lang3.StringUtils; import static com.google.common.base.Preconditions.checkNotNull; public class PeerForwarderConfig { public static final String TIME_OUT = "time_out"; public static final String MAX_NUM_SPANS_PER_REQUEST = "span_agg_count"; - public static final int NUM_VIRTUAL_NODES = 10; + public static final int NUM_VIRTUAL_NODES = 128; + public static final String TARGET_PORT = "target_port"; public static final String DISCOVERY_MODE = "discovery_mode"; public static final String DOMAIN_NAME = "domain_name"; public static final String STATIC_ENDPOINTS = "static_endpoints"; public static final String SSL = "ssl"; public static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; private static final boolean DEFAULT_SSL = true; + private static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; + private static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + private static final int DEFAULT_TARGET_PORT = 21890; + private static final int DEFAULT_TIMEOUT_SECONDS = 2; + private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; + private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; + private static final String ACM_CERT_ARN = "acmCertificateArn"; + public static final String AWS_REGION = "awsRegion"; + public static final String AWS_CLOUD_MAP_NAMESPACE_NAME = "awsCloudMapNamespaceName"; + public static final String AWS_CLOUD_MAP_SERVICE_NAME = "awsCloudMapServiceName"; private final HashRing hashRing; private final PeerClientPool peerClientPool; @@ -55,27 +65,38 @@ public static PeerForwarderConfig buildConfig(final PluginSetting pluginSetting) final HashRing hashRing = new HashRing(peerListProvider, NUM_VIRTUAL_NODES); final PeerClientPool peerClientPool = PeerClientPool.getInstance(); peerClientPool.setClientTimeoutSeconds(3); + + final int targetPort = pluginSetting.getIntegerOrDefault(TARGET_PORT, DEFAULT_TARGET_PORT); + peerClientPool.setPort(targetPort); + final boolean ssl = pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL); final String sslKeyCertChainFilePath = pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null); - final File sslKeyCertChainFile; - if (ssl) { - if (sslKeyCertChainFilePath == null || sslKeyCertChainFilePath.isEmpty()) { + final boolean useAcmCertForSsl = pluginSetting.getBooleanOrDefault(USE_ACM_CERT_FOR_SSL, DEFAULT_USE_ACM_CERT_FOR_SSL); + + if (ssl || useAcmCertForSsl) { + if (ssl && StringUtils.isEmpty(sslKeyCertChainFilePath)) { throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); - } else if (!Files.exists(Paths.get(sslKeyCertChainFilePath))) { - throw new IllegalArgumentException(String.format("%s is enabled, %s does not exist", SSL, SSL_KEY_CERT_FILE)); - } else { - sslKeyCertChainFile = new File(sslKeyCertChainFilePath); } - } else { - sslKeyCertChainFile = null; + peerClientPool.setSsl(true); + final String acmCertificateArn = pluginSetting.getStringOrDefault(ACM_CERT_ARN, null); + final long acmCertIssueTimeOutMillis = pluginSetting.getLongOrDefault(ACM_CERT_ISSUE_TIME_OUT_MILLIS, DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS); + final String awsRegion = pluginSetting.getStringOrDefault(AWS_REGION, null); + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSsl, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFilePath + ); + final CertificateProviderFactory certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + peerClientPool.setCertificate(certificateProviderFactory.getCertificateProvider().getCertificate()); + } - peerClientPool.setSsl(ssl); - peerClientPool.setSslKeyCertChainFile(sslKeyCertChainFile); return new PeerForwarderConfig( peerClientPool, hashRing, - pluginSetting.getIntegerOrDefault(TIME_OUT, 3), + pluginSetting.getIntegerOrDefault(TIME_OUT, DEFAULT_TIMEOUT_SECONDS), pluginSetting.getIntegerOrDefault(MAX_NUM_SPANS_PER_REQUEST, 48)); } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java index 605508def9..f096bd0bb5 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java @@ -16,10 +16,6 @@ import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.net.UnknownHostException; import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java new file mode 100644 index 0000000000..a9ef3665d1 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java @@ -0,0 +1,7 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; + +public interface CertificateProvider { + Certificate getCertificate(); +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java new file mode 100644 index 0000000000..dc29ca9b24 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java @@ -0,0 +1,43 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +public class CertificateProviderConfig { + private static final String S3_PREFIX = "s3://"; + + private final boolean useAcmCertForSSL; + private final String acmCertificateArn; + private final String awsRegion; + private final long acmCertIssueTimeOutMillis; + private final String sslKeyCertChainFile; + + public CertificateProviderConfig(final boolean useAcmCertForSSL, final String acmCertificateArn, final String awsRegion, final long acmCertIssueTimeOutMillis, final String sslKeyCertChainFile) { + this.useAcmCertForSSL = useAcmCertForSSL; + this.acmCertificateArn = acmCertificateArn; + this.awsRegion = awsRegion; + this.acmCertIssueTimeOutMillis = acmCertIssueTimeOutMillis; + this.sslKeyCertChainFile = sslKeyCertChainFile; + } + + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public String getAcmCertificateArn() { + return acmCertificateArn; + } + + public String getAwsRegion() { + return awsRegion; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis; + } + + public String getSslKeyCertChainFile() { + return sslKeyCertChainFile; + } + + public boolean isSslCertFileInS3() { + return sslKeyCertChainFile.toLowerCase().startsWith(S3_PREFIX); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java new file mode 100644 index 0000000000..c6e5c2b9b7 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java @@ -0,0 +1,51 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3.S3CertificateProvider; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CertificateProviderFactory { + private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); + + final CertificateProviderConfig certificateProviderConfig; + public CertificateProviderFactory(final CertificateProviderConfig certificateProviderConfig) { + this.certificateProviderConfig = certificateProviderConfig; + } + + public CertificateProvider getCertificateProvider() { + // ACM Cert for SSL takes preference + if (certificateProviderConfig.useAcmCertForSSL()) { + LOG.info("Using ACM certificate for SSL/TLS to setup trust store."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final ClientConfiguration clientConfig = new ClientConfiguration() + .withThrottledRetries(true); + final AWSCertificateManager awsCertificateManager = AWSCertificateManagerClientBuilder.standard() + .withRegion(certificateProviderConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig) + .build(); + return new ACMCertificateProvider(awsCertificateManager, certificateProviderConfig.getAcmCertificateArn(), + certificateProviderConfig.getAcmCertIssueTimeOutMillis()); + } else if (certificateProviderConfig.isSslCertFileInS3()) { + LOG.info("Using S3 to fetch certificate for SSL/TLS to setup trust store."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(certificateProviderConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .build(); + return new S3CertificateProvider(s3Client, certificateProviderConfig.getSslKeyCertChainFile()); + } else { + LOG.info("Using local file system to get certificate for SSL/TLS to setup trust store."); + return new FileCertificateProvider(certificateProviderConfig.getSslKeyCertChainFile()); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java new file mode 100644 index 0000000000..0281542912 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java @@ -0,0 +1,58 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.GetCertificateRequest; +import com.amazonaws.services.certificatemanager.model.GetCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +public class ACMCertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); + private static final long SLEEP_INTERVAL = 10000L; + private final AWSCertificateManager awsCertificateManager; + private final String acmArn; + private final long totalTimeout; + public ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, + final String acmArn, + final long totalTimeout) { + this.awsCertificateManager = Objects.requireNonNull(awsCertificateManager); + this.acmArn = Objects.requireNonNull(acmArn); + this.totalTimeout = totalTimeout; + } + + public Certificate getCertificate() { + GetCertificateResult getCertificateResult = null; + long timeSlept = 0L; + + while (getCertificateResult == null && timeSlept < totalTimeout) { + try { + final GetCertificateRequest getCertificateRequest = new GetCertificateRequest() + .withCertificateArn(acmArn); + getCertificateResult = awsCertificateManager.getCertificate(getCertificateRequest); + + } catch (final RequestInProgressException ex) { + try { + Thread.sleep(SLEEP_INTERVAL); + } catch (InterruptedException iex) { + throw new RuntimeException(iex); + } + } catch (final ResourceNotFoundException | InvalidArnException ex) { + LOG.error("Exception retrieving the certificate with arn: {}", acmArn, ex); + throw ex; + } + timeSlept += SLEEP_INTERVAL; + } + if(getCertificateResult != null) { + return new Certificate(getCertificateResult.getCertificate()); + } else { + throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java new file mode 100644 index 0000000000..ca9b867c34 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java @@ -0,0 +1,33 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +public class FileCertificateProvider implements CertificateProvider { + private final String certificateFilePath; + + public FileCertificateProvider(final String certificateFilePath) { + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + } + + private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); + + public Certificate getCertificate() { + try { + final Path certFilePath = Path.of(certificateFilePath); + + final String certAsString = Files.readString(certFilePath); + + return new Certificate(certAsString); + } catch (final Exception ex) { + LOG.error("Error encountered while reading the certificate.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java new file mode 100644 index 0000000000..68df09e3e8 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java @@ -0,0 +1,18 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model; + +import static java.util.Objects.requireNonNull; + +public class Certificate { + /** + * The base64 PEM-encoded certificate. + */ + private String certificate; + + public Certificate(final String certificate) { + this.certificate = requireNonNull(certificate, "certificate must not be null"); + } + + public String getCertificate() { + return certificate; + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java new file mode 100644 index 0000000000..0e7a1937f6 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java @@ -0,0 +1,44 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class S3CertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(S3CertificateProvider.class); + private final AmazonS3 s3Client; + private final String certificateFilePath; + + public S3CertificateProvider(final AmazonS3 s3Client, + final String certificateFilePath) { + this.s3Client = Objects.requireNonNull(s3Client); + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + } + + public Certificate getCertificate() { + final AmazonS3URI certificateS3URI = new AmazonS3URI(certificateFilePath); + final String certificate = getObjectWithKey(certificateS3URI.getBucket(), certificateS3URI.getKey()); + + return new Certificate(certificate); + } + + private String getObjectWithKey(final String bucketName, final String key) { + + // Download the object + try (final S3Object s3Object = s3Client.getObject(bucketName, key)) { + LOG.info("Object with key \"{}\" downloaded.", key); + return IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8); + } catch (final Exception ex) { + LOG.error("Error encountered while processing the response from Amazon S3.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java new file mode 100644 index 0000000000..e2b89af27e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java @@ -0,0 +1,205 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.linecorp.armeria.client.Endpoint; +import com.linecorp.armeria.client.endpoint.DynamicEndpointGroup; +import com.linecorp.armeria.client.retry.Backoff; +import com.linecorp.armeria.common.CommonPools; +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.ScheduledFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.servicediscovery.ServiceDiscoveryAsyncClient; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesRequest; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesResponse; +import software.amazon.awssdk.services.servicediscovery.model.HttpInstanceSummary; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Implementation of {@link PeerListProvider} which uses AWS CloudMap's + * service discovery capability to discover peers. + */ +class AwsCloudMapPeerListProvider implements PeerListProvider, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(AwsCloudMapPeerListProvider.class); + private static final int ONE_SECOND = 1000; + private static final int TWENTY_SECONDS = 32000; + private static final double TWENTY_PERCENT = 0.2; + private static final String INSTANCE_IP4_ATTRIBUTE_NAME = "AWS_INSTANCE_IPV4"; + + private final ServiceDiscoveryAsyncClient awsServiceDiscovery; + private final String namespaceName; + private final String serviceName; + private final AwsCloudMapDynamicEndpointGroup endpointGroup; + private final int timeToRefreshSeconds; + private final Backoff backoff; + private final EventLoop eventLoop; + private final String domainName; + + AwsCloudMapPeerListProvider( + final ServiceDiscoveryAsyncClient awsServiceDiscovery, + final String namespaceName, + final String serviceName, + final int timeToRefreshSeconds, + final Backoff backoff, + final PluginMetrics pluginMetrics) { + this.awsServiceDiscovery = Objects.requireNonNull(awsServiceDiscovery); + this.namespaceName = Objects.requireNonNull(namespaceName); + this.serviceName = Objects.requireNonNull(serviceName); + this.timeToRefreshSeconds = timeToRefreshSeconds; + this.backoff = Objects.requireNonNull(backoff); + + if (timeToRefreshSeconds < 1) + throw new IllegalArgumentException("timeToRefreshSeconds must be positive. Actual: " + timeToRefreshSeconds); + + eventLoop = CommonPools.workerGroup().next(); + LOG.info("Using AWS CloudMap for Peer Forwarding. namespace='{}', serviceName='{}'", + namespaceName, serviceName); + + endpointGroup = new AwsCloudMapDynamicEndpointGroup(); + + domainName = serviceName + "." + namespaceName; + + pluginMetrics.gauge(PEER_ENDPOINTS, endpointGroup, group -> group.endpoints().size()); + } + + static AwsCloudMapPeerListProvider createPeerListProvider(final PluginSetting pluginSetting, final PluginMetrics pluginMetrics) { + final String awsRegion = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_REGION); + final String namespace = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME); + final String serviceName = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME); + + final Backoff standardBackoff = Backoff.exponential(ONE_SECOND, TWENTY_SECONDS).withJitter(TWENTY_PERCENT); + final int timeToRefreshSeconds = 20; + + final ServiceDiscoveryAsyncClient serviceDiscoveryAsyncClient = ServiceDiscoveryAsyncClient + .builder() + .region(Region.of(awsRegion)) + .credentialsProvider(DefaultCredentialsProvider.create()) + .build(); + + return new AwsCloudMapPeerListProvider( + serviceDiscoveryAsyncClient, + namespace, + serviceName, + timeToRefreshSeconds, + standardBackoff, + pluginMetrics); + } + + private static String getRequiredSettingString(final PluginSetting pluginSetting, final String propertyName) { + final String propertyValue = pluginSetting.getStringOrDefault(propertyName, null); + return Objects.requireNonNull(propertyValue, String.format("Missing '%s' configuration value", propertyName)); + } + + @Override + public List getPeerList() { + return endpointGroup.endpoints() + .stream() + .map(Endpoint::ipAddr) + .collect(Collectors.toList()); + } + + @Override + public void addListener(final Consumer> listener) { + endpointGroup.addListener(listener); + } + + @Override + public void removeListener(final Consumer listener) { + endpointGroup.removeListener(listener); + } + + @Override + public void close() { + endpointGroup.close(); + } + + /** + * The {@link DynamicEndpointGroup} class serves as a useful base class for updating + * endpoints by supporting the endpoint observer pattern for us. We just need to + * periodically check for updates and call {@link DynamicEndpointGroup#setEndpoints(Iterable)}. + */ + private class AwsCloudMapDynamicEndpointGroup extends DynamicEndpointGroup { + + private int failedAttemptCount = 0; + private volatile ScheduledFuture scheduledDiscovery; + + private AwsCloudMapDynamicEndpointGroup() { + eventLoop.execute(this::discoverInstances); + } + + private void discoverInstances() { + if (isClosing()) { + return; + } + + final DiscoverInstancesRequest discoverInstancesRequest = DiscoverInstancesRequest + .builder() + .namespaceName(namespaceName) + .serviceName(serviceName) + .build(); + + LOG.info("Discovering instances."); + + awsServiceDiscovery.discoverInstances(discoverInstancesRequest).whenComplete( + (discoverInstancesResponse, throwable) -> { + if (discoverInstancesResponse != null) { + try { + failedAttemptCount = 0; + updateEndpointsWithDiscoveredInstances(discoverInstancesResponse); + } catch (final Throwable ex) { + LOG.warn("Failed to update endpoints.", ex); + } finally { + scheduledDiscovery = eventLoop.schedule(this::discoverInstances, + timeToRefreshSeconds, TimeUnit.SECONDS); + } + } + + if (throwable != null) { + failedAttemptCount++; + final long delayMillis = backoff.nextDelayMillis(failedAttemptCount); + LOG.error("Failed to discover instances for: namespace='{}', serviceName='{}'. Will retry in {} ms.", + namespaceName, serviceName, delayMillis, throwable); + + scheduledDiscovery = eventLoop.schedule(this::discoverInstances, + delayMillis, TimeUnit.MILLISECONDS); + } + }); + } + + private void updateEndpointsWithDiscoveredInstances(final DiscoverInstancesResponse discoverInstancesResponse) { + final List instances = discoverInstancesResponse.instances(); + + LOG.info("Discovered {} instances.", instances.size()); + + final List endpoints = instances + .stream() + .map(HttpInstanceSummary::attributes) + .map(attributes -> attributes.get(INSTANCE_IP4_ATTRIBUTE_NAME)) + .map(ip -> Endpoint.of(domainName).withIpAddr(ip)) + .collect(Collectors.toList()); + + setEndpoints(endpoints); + } + + @Override + protected void doCloseAsync(final CompletableFuture future) { + final ScheduledFuture scheduledDiscovery = this.scheduledDiscovery; + if (scheduledDiscovery != null) { + scheduledDiscovery.cancel(true); + } + + future.complete(null); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java index 72bec9f8aa..5b34b08229 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java @@ -11,7 +11,33 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; + +import java.util.Objects; +import java.util.function.BiFunction; + public enum DiscoveryMode { - STATIC, - DNS, + STATIC(StaticPeerListProvider::createPeerListProvider), + DNS(DnsPeerListProvider::createPeerListProvider), + AWS_CLOUD_MAP(AwsCloudMapPeerListProvider::createPeerListProvider); + + private final BiFunction creationFunction; + + DiscoveryMode(final BiFunction creationFunction) { + Objects.requireNonNull(creationFunction); + + this.creationFunction = creationFunction; + } + + /** + * Creates a new {@link PeerListProvider} for the this discovery mode. + * + * @param pluginSetting The plugin settings + * @param pluginMetrics The plugin metrics + * @return The new {@link PeerListProvider} for this discovery mode + */ + PeerListProvider create(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + return creationFunction.apply(pluginSetting, pluginMetrics); + } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java new file mode 100644 index 0000000000..b7bc0c870a --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java @@ -0,0 +1,10 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.google.common.net.InternetDomainName; +import org.apache.commons.validator.routines.InetAddressValidator; + +class DiscoveryUtils { + static boolean validateEndpoint(final String endpoint) { + return InternetDomainName.isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java index d5b6581601..b2cb4ed132 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java @@ -12,6 +12,9 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.google.common.base.Preconditions; import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; import org.slf4j.Logger; @@ -23,9 +26,13 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.DiscoveryUtils.validateEndpoint; + public class DnsPeerListProvider implements PeerListProvider { private static final Logger LOG = LoggerFactory.getLogger(DnsPeerListProvider.class); + private static final int MIN_TTL = 10; + private static final int MAX_TTL = 20; private final DnsAddressEndpointGroup endpointGroup; @@ -44,6 +51,18 @@ public DnsPeerListProvider(final DnsAddressEndpointGroup endpointGroup, final Pl pluginMetrics.gauge(PEER_ENDPOINTS, endpointGroup, group -> group.endpoints().size()); } + static DnsPeerListProvider createPeerListProvider(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + final String domainName = pluginSetting.getStringOrDefault(PeerForwarderConfig.DOMAIN_NAME, null); + Objects.requireNonNull(domainName, String.format("Missing '%s' configuration value",PeerForwarderConfig. DOMAIN_NAME)); + Preconditions.checkState(validateEndpoint(domainName), "Invalid domain name: %s", domainName); + + final DnsAddressEndpointGroup endpointGroup = DnsAddressEndpointGroup.builder(domainName) + .ttl(MIN_TTL, MAX_TTL) + .build(); + + return new DnsPeerListProvider(endpointGroup, pluginMetrics); + } + @Override public List getPeerList() { return endpointGroup.endpoints() diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java index 2892d0d0c6..3b2e8a34a3 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java @@ -14,19 +14,10 @@ import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; -import com.google.common.base.Preconditions; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; -import org.apache.commons.validator.routines.DomainValidator; -import org.apache.commons.validator.routines.InetAddressValidator; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; public class PeerListProviderFactory { - // TODO: Make these configurable? - private static final int MIN_TTL = 10; - private static final int MAX_TTL = 20; public PeerListProvider createProvider(final PluginSetting pluginSetting) { Objects.requireNonNull(pluginSetting); @@ -38,30 +29,7 @@ public PeerListProvider createProvider(final PluginSetting pluginSetting) { final PluginMetrics pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); - switch (discoveryMode) { - case DNS: - final String domainName = pluginSetting.getStringOrDefault(PeerForwarderConfig.DOMAIN_NAME, null); - Objects.requireNonNull(domainName, String.format("Missing '%s' configuration value",PeerForwarderConfig. DOMAIN_NAME)); - Preconditions.checkState(validateEndpoint(domainName), "Invalid domain name: %s", domainName); - - final DnsAddressEndpointGroup endpointGroup = DnsAddressEndpointGroup.builder(domainName) - .ttl(MIN_TTL, MAX_TTL) - .build(); - - return new DnsPeerListProvider(endpointGroup, pluginMetrics); - case STATIC: - final List endpoints = (List) pluginSetting.getAttributeOrDefault(PeerForwarderConfig.STATIC_ENDPOINTS, null); - Objects.requireNonNull(endpoints, String.format("Missing '%s' configuration value", PeerForwarderConfig.STATIC_ENDPOINTS)); - final List invalidEndpoints = endpoints.stream().filter(endpoint -> !this.validateEndpoint(endpoint)).collect(Collectors.toList()); - Preconditions.checkState(invalidEndpoints.isEmpty(), "Including invalid endpoints: %s", invalidEndpoints); - - return new StaticPeerListProvider(endpoints, pluginMetrics); - default: - throw new UnsupportedOperationException(String.format("Unsupported discovery mode: %s", discoveryMode)); - } + return discoveryMode.create(pluginSetting, pluginMetrics); } - private boolean validateEndpoint(final String endpoint) { - return DomainValidator.getInstance(true).isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); - } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java index 0158779990..a0131813ee 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java @@ -12,13 +12,18 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.google.common.base.Preconditions; import com.linecorp.armeria.client.Endpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Consumer; +import java.util.stream.Collectors; public class StaticPeerListProvider implements PeerListProvider { @@ -40,6 +45,15 @@ public StaticPeerListProvider(final List dataPrepperEndpoints, final Plu pluginMetrics.gauge(PEER_ENDPOINTS, endpoints, List::size); } + static StaticPeerListProvider createPeerListProvider(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + final List endpoints = (List) pluginSetting.getAttributeOrDefault(PeerForwarderConfig.STATIC_ENDPOINTS, null); + Objects.requireNonNull(endpoints, String.format("Missing '%s' configuration value", PeerForwarderConfig.STATIC_ENDPOINTS)); + final List invalidEndpoints = endpoints.stream().filter(endpoint -> !DiscoveryUtils.validateEndpoint(endpoint)).collect(Collectors.toList()); + Preconditions.checkState(invalidEndpoints.isEmpty(), "Including invalid endpoints: %s", invalidEndpoints); + + return new StaticPeerListProvider(endpoints, pluginMetrics); + } + @Override public List getPeerList() { return endpoints; diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java index 4d29d63d9c..7ca69000df 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java @@ -11,6 +11,7 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -21,6 +22,9 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import static org.junit.Assert.assertNotNull; @@ -36,6 +40,7 @@ public class PeerClientPoolTest { @Test public void testGetClientValidAddress() { PeerClientPool pool = PeerClientPool.getInstance(); + pool.setPort(PORT); TraceServiceGrpc.TraceServiceBlockingStub client = pool.getClient(VALID_ADDRESS); @@ -43,7 +48,7 @@ public void testGetClientValidAddress() { } @Test - public void testGetClientWithSSL() { + public void testGetClientWithSSL() throws IOException { // Set up test server with SSL ServerBuilder sb = Server.builder(); sb.service(GrpcService.builder() @@ -57,7 +62,11 @@ public void testGetClientWithSSL() { // Configure client pool PeerClientPool pool = PeerClientPool.getInstance(); pool.setSsl(true); - pool.setSslKeyCertChainFile(SSL_CRT_FILE); + + final Path certFilePath = Path.of(PeerClientPoolTest.class.getClassLoader().getResource("test-crt.crt").getPath()); + final String certAsString = Files.readString(certFilePath); + final Certificate certificate = new Certificate(certAsString); + pool.setCertificate(certificate); TraceServiceGrpc.TraceServiceBlockingStub client = pool.getClient(LOCALHOST); assertNotNull(client); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java index 7dc9e2e48b..661333a54a 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java @@ -12,22 +12,25 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.DiscoveryMode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.junit.MockitoJUnitRunner; -import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mockStatic; @@ -53,7 +56,6 @@ public void setUp() { mockedPeerClientPoolClass = mockStatic(PeerClientPool.class); mockedPeerClientPoolClass.when(PeerClientPool::getInstance).thenReturn(peerClientPool); doNothing().when(peerClientPool).setSsl(anyBoolean()); - doNothing().when(peerClientPool).setSslKeyCertChainFile(any(File.class)); } @After @@ -98,13 +100,13 @@ public void testBuildConfigInvalidSSL() { }); settings.put(PeerForwarderConfig.SSL_KEY_CERT_FILE, INVALID_SSL_KEY_CERT_FILE); - Assert.assertThrows(IllegalArgumentException.class, () -> { + Assert.assertThrows(RuntimeException.class, () -> { PeerForwarderConfig.buildConfig(new PluginSetting("peer_forwarder", settings){{ setPipelineName(PIPELINE_NAME); }}); }); } @Test - public void testBuildConfigValidSSL() { + public void testBuildConfigValidSSL() throws IOException { final HashMap settings = new HashMap<>(); settings.put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); settings.put(PeerForwarderConfig.STATIC_ENDPOINTS, TEST_ENDPOINTS); @@ -113,6 +115,12 @@ public void testBuildConfigValidSSL() { PeerForwarderConfig.buildConfig(new PluginSetting("peer_forwarder", settings){{ setPipelineName(PIPELINE_NAME); }}); verify(peerClientPool, times(1)).setSsl(true); - verify(peerClientPool, times(1)).setSslKeyCertChainFile(new File(VALID_SSL_KEY_CERT_FILE)); + final ArgumentCaptor certificateArgumentCaptor = ArgumentCaptor.forClass(Certificate.class); + verify(peerClientPool, times(1)).setCertificate(certificateArgumentCaptor.capture()); + final Certificate certificate = certificateArgumentCaptor.getValue(); + + final Path certFilePath = Path.of(VALID_SSL_KEY_CERT_FILE); + final String certAsString = Files.readString(certFilePath); + Assert.assertEquals(certificate.getCertificate(), certAsString); } } \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java index 20c6aa9899..31e6f794c3 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java @@ -224,17 +224,17 @@ public void testSingleRemoteIpForwardedRequestOnly() throws Exception { // Verify metrics final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_ERRORS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.ERRORS).toString()); Assert.assertEquals(1, forwardRequestErrorMeasurements.size()); Assert.assertEquals(0.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_SUCCESS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.REQUESTS).toString()); Assert.assertEquals(1, forwardRequestSuccessMeasurements.size()); Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_LATENCY_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.LATENCY).toString()); Assert.assertEquals(3, forwardRequestLatencyMeasurements.size()); // COUNT Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); @@ -272,17 +272,17 @@ public void testSingleRemoteIpForwardRequestError() { // Verify metrics final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_ERRORS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.ERRORS).toString()); Assert.assertEquals(1, forwardRequestErrorMeasurements.size()); Assert.assertEquals(1.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_SUCCESS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.REQUESTS).toString()); Assert.assertEquals(1, forwardRequestSuccessMeasurements.size()); - Assert.assertEquals(0.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); + Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_LATENCY_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.LATENCY).toString()); Assert.assertEquals(3, forwardRequestLatencyMeasurements.size()); // COUNT Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java new file mode 100644 index 0000000000..9f872b7e5e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java @@ -0,0 +1,82 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3.S3CertificateProvider; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class CertificateProviderFactoryTest { + private CertificateProviderFactory certificateProviderFactory; + + @Test + public void getCertificateProviderAcmProviderSuccess() { + final boolean useAcmCertForSSL = true; + final String awsRegion = "us-east-1"; + final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + final String sslKeyCertChainFile = null; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); + } + + @Test + public void getCertificateProviderS3ProviderSuccess() { + final boolean useAcmCertForSSL = false; + final String awsRegion = "us-east-1"; + final String acmCertificateArn = null; + final String sslKeyCertChainFile = "s3://some_s3_bucket/certificate/test_cert.crt"; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider();; + + assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); + } + + @Test + public void getCertificateProviderFileProviderSuccess() { + final boolean useAcmCertForSSL = false; + final String awsRegion = null; + final String acmCertificateArn = null; + final String sslKeyCertChainFile = "path_to_certificate/test_cert.crt"; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java new file mode 100644 index 0000000000..6150c7925c --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java @@ -0,0 +1,67 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.GetCertificateRequest; +import com.amazonaws.services.certificatemanager.model.GetCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.UUID; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class ACMCertificateProviderTest { + private static final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + private static final long acmCertIssueTimeOutMillis = 2000L; + @Mock + private AWSCertificateManager awsCertificateManager; + + @Mock + private GetCertificateResult getCertificateResult; + + private ACMCertificateProvider acmCertificateProvider; + + @Before + public void beforeEach() { + acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, acmCertificateArn, acmCertIssueTimeOutMillis); + } + + @Test + public void getACMCertificateSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + when(getCertificateResult.getCertificate()).thenReturn(certificateContent); + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenReturn(getCertificateResult); + final Certificate certificate = acmCertificateProvider.getCertificate(); + assertThat(certificate.getCertificate(), is(certificateContent)); + } + + @Test + public void getACMCertificateRequestInProgressException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new RequestInProgressException("Request in progress.")); + assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateResourceNotFoundException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new ResourceNotFoundException("Resource not found.")); + assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateInvalidArnException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new InvalidArnException("Invalid certificate arn.")); + assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java new file mode 100644 index 0000000000..417a48721b --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java @@ -0,0 +1,42 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class FileCertificateProviderTest { + + private FileCertificateProvider fileCertificateProvider; + + @Test + public void getCertificateValidPathSuccess() throws IOException { + final String certificateFilePath = FileCertificateProviderTest.class.getClassLoader().getResource("test-crt.crt").getPath(); + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath); + + final Certificate certificate = fileCertificateProvider.getCertificate(); + + final Path certFilePath = Path.of(certificateFilePath); + final String certAsString = Files.readString(certFilePath); + + assertThat(certificate.getCertificate(), is(certAsString)); + } + + @Test(expected = RuntimeException.class) + public void getCertificateInvalidPathSuccess() { + final String certificateFilePath = "path_does_not_exit/test_cert.crt"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath); + + fileCertificateProvider.getCertificate(); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java new file mode 100644 index 0000000000..0bd0db5656 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java @@ -0,0 +1,65 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class S3CertificateProviderTest { + @Mock + private AmazonS3 amazonS3; + + @Mock + private S3Object certS3Object; + + private S3CertificateProvider s3CertificateProvider; + + @Test + public void getCertificateValidKeyPathSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String certificatePath = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + + final InputStream certObjectStream = IOUtils.toInputStream(certificateContent, StandardCharsets.UTF_8); + + when(certS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(certObjectStream,null)); + + when(amazonS3.getObject(bucketName, certificatePath)).thenReturn(certS3Object); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile); + + final Certificate certificate = s3CertificateProvider.getCertificate(); + + assertThat(certificate.getCertificate(), is(certificateContent)); + } + + @Test(expected = RuntimeException.class) + public void getCertificateValidKeyPathS3Exception() { + final String certificatePath = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile); + when(amazonS3.getObject(anyString(), anyString())).thenThrow(new RuntimeException("S3 exception")); + + s3CertificateProvider.getCertificate(); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java new file mode 100644 index 0000000000..7ba3e16b94 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java @@ -0,0 +1,387 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.linecorp.armeria.client.Endpoint; +import com.linecorp.armeria.client.retry.Backoff; +import org.apache.commons.lang3.RandomStringUtils; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import software.amazon.awssdk.services.servicediscovery.ServiceDiscoveryAsyncClient; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesRequest; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesResponse; +import software.amazon.awssdk.services.servicediscovery.model.HttpInstanceSummary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; + +class AwsCloudMapPeerListProviderTest { + + public static final int WAIT_TIME_MULTIPLIER_MILLIS = 1200; + private ServiceDiscoveryAsyncClient awsServiceDiscovery; + private String namespaceName; + private String serviceName; + private int timeToRefreshSeconds; + private Backoff backoff; + private PluginMetrics pluginMetrics; + private List objectsToClose; + + @BeforeEach + void setUp() { + awsServiceDiscovery = mock(ServiceDiscoveryAsyncClient.class); + namespaceName = RandomStringUtils.randomAlphabetic(10); + serviceName = RandomStringUtils.randomAlphabetic(10); + + timeToRefreshSeconds = 1; + backoff = mock(Backoff.class); + pluginMetrics = mock(PluginMetrics.class); + + objectsToClose = new ArrayList<>(); + } + + @AfterEach + void tearDown() { + objectsToClose.forEach(AwsCloudMapPeerListProvider::close); + } + + private AwsCloudMapPeerListProvider createObjectUnderTest() { + final AwsCloudMapPeerListProvider objectUnderTest = new AwsCloudMapPeerListProvider(awsServiceDiscovery, namespaceName, serviceName, timeToRefreshSeconds, backoff, pluginMetrics); + objectsToClose.add(objectUnderTest); + return objectUnderTest; + } + + @Test + void constructor_throws_with_null_AWSServiceDiscovery() { + awsServiceDiscovery = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_Namespace() { + namespaceName = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_ServiceName() { + serviceName = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_Backoff() { + backoff = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @ParameterizedTest + @ValueSource(ints = {Integer.MIN_VALUE, -10, -1, 0}) + void constructor_throws_with_non_positive_timeToRefreshSeconds(final int badTimeToRefresh) { + timeToRefreshSeconds = badTimeToRefresh; + + assertThrows(IllegalArgumentException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_should_DiscoverInstances_with_correct_request() { + createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final ArgumentCaptor requestArgumentCaptor = + ArgumentCaptor.forClass(DiscoverInstancesRequest.class); + + then(awsServiceDiscovery) + .should() + .discoverInstances(requestArgumentCaptor.capture()); + + final DiscoverInstancesRequest actualRequest = requestArgumentCaptor.getValue(); + + assertThat(actualRequest.namespaceName(), equalTo(namespaceName)); + assertThat(actualRequest.serviceName(), equalTo(serviceName)); + assertThat(actualRequest.healthStatusAsString(), nullValue()); + } + + @Test + void getPeerList_is_empty_before_populated() { + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List peerList = objectUnderTest.getPeerList(); + + assertThat(peerList, notNullValue()); + assertThat(peerList.size(), equalTo(0)); + } + + @Nested + class WithDiscoverInstances { + + private DiscoverInstancesResponse discoverInstancesResponse; + + @BeforeEach + void setUp() { + discoverInstancesResponse = mock(DiscoverInstancesResponse.class); + + final CompletableFuture discoverFuture = + CompletableFuture.completedFuture(discoverInstancesResponse); + + given(awsServiceDiscovery.discoverInstances(any(DiscoverInstancesRequest.class))) + .willReturn(discoverFuture); + } + + @Test + void getPeerList_returns_empty_when_DiscoverInstances_has_no_instances() { + given(discoverInstancesResponse.instances()).willReturn(Collections.emptyList()); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List peerList = objectUnderTest.getPeerList(); + assertThat(peerList, notNullValue()); + assertThat(peerList.size(), equalTo(0)); + } + + @Test + void getPeerList_returns_list_as_found() { + + final List knownIpPeers = IntStream.range(0, 3) + .mapToObj(i -> generateRandomIp()) + .collect(Collectors.toList()); + + final List instances = knownIpPeers + .stream() + .map(ip -> { + final HttpInstanceSummary instanceSummary = mock(HttpInstanceSummary.class); + given(instanceSummary.attributes()).willReturn( + Collections.singletonMap("AWS_INSTANCE_IPV4", ip)); + return instanceSummary; + }) + .collect(Collectors.toList()); + + given(discoverInstancesResponse.instances()).willReturn(instances); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilPeerListPopulated(objectUnderTest); + + final List actualPeers = objectUnderTest.getPeerList(); + assertThat(actualPeers, notNullValue()); + assertThat(actualPeers.size(), equalTo(instances.size())); + + assertThat(new HashSet<>(actualPeers), equalTo(new HashSet<>(knownIpPeers))); + } + + @Test + void constructor_continues_to_discover_instances() { + + createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeast(2); + + final ArgumentCaptor requestArgumentCaptor = + ArgumentCaptor.forClass(DiscoverInstancesRequest.class); + + then(awsServiceDiscovery) + .should(atLeast(2)) + .discoverInstances(requestArgumentCaptor.capture()); + + for (DiscoverInstancesRequest actualRequest : requestArgumentCaptor.getAllValues()) { + assertThat(actualRequest.namespaceName(), equalTo(namespaceName)); + assertThat(actualRequest.serviceName(), equalTo(serviceName)); + assertThat(actualRequest.healthStatusAsString(), nullValue()); + } + } + } + + @Nested + class WithSeveralFailedAttempts { + + private List knownIpPeers; + + @BeforeEach + void setUp() { + final DiscoverInstancesResponse discoverInstancesResponse = mock(DiscoverInstancesResponse.class); + + knownIpPeers = IntStream.range(0, 3) + .mapToObj(i -> generateRandomIp()) + .collect(Collectors.toList()); + + final List instances = knownIpPeers + .stream() + .map(ip -> { + final HttpInstanceSummary instanceSummary = mock(HttpInstanceSummary.class); + given(instanceSummary.attributes()).willReturn( + Collections.singletonMap("AWS_INSTANCE_IPV4", ip)); + return instanceSummary; + }) + .collect(Collectors.toList()); + + given(discoverInstancesResponse.instances()).willReturn(instances); + + final CompletableFuture failedFuture1 = new CompletableFuture<>(); + failedFuture1.completeExceptionally(mock(Throwable.class)); + final CompletableFuture failedFuture2 = new CompletableFuture<>(); + failedFuture2.completeExceptionally(mock(Throwable.class)); + final CompletableFuture successFuture = CompletableFuture.completedFuture(discoverInstancesResponse); + + given(awsServiceDiscovery.discoverInstances(any(DiscoverInstancesRequest.class))) + .willReturn(failedFuture1) + .willReturn(failedFuture2) + .willReturn(successFuture); + + given(backoff.nextDelayMillis(anyInt())) + .willReturn(100L); + + } + + @Test + void getPeerList_returns_value_after_several_failed_attempts() { + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List expectedEmpty = objectUnderTest.getPeerList(); + + assertThat(expectedEmpty, notNullValue()); + assertThat(expectedEmpty.size(), equalTo(0)); + + waitUntilPeerListPopulated(objectUnderTest); + + final List expectedPopulated = objectUnderTest.getPeerList(); + + assertThat(expectedPopulated, notNullValue()); + assertThat(expectedPopulated.size(), equalTo(knownIpPeers.size())); + + assertThat(new HashSet<>(expectedPopulated), equalTo(new HashSet<>(knownIpPeers))); + + final InOrder inOrder = inOrder(backoff); + then(backoff) + .should(inOrder) + .nextDelayMillis(1); + then(backoff) + .should(inOrder) + .nextDelayMillis(2); + then(backoff) + .shouldHaveNoMoreInteractions(); + } + + @Test + void listener_gets_list_after_several_failed_attempts() { + + final List listenerEndpoints = new ArrayList<>(); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + objectUnderTest.addListener(listenerEndpoints::addAll); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + assertThat(listenerEndpoints.size(), equalTo(0)); + + waitUntilPeerListPopulated(objectUnderTest); + + assertThat(listenerEndpoints.size(), equalTo(knownIpPeers.size())); + + final Set observedIps = listenerEndpoints.stream() + .map(Endpoint::ipAddr) + .collect(Collectors.toSet()); + + assertThat(observedIps, equalTo(new HashSet<>(knownIpPeers))); + + final InOrder inOrder = inOrder(backoff); + then(backoff) + .should(inOrder) + .nextDelayMillis(1); + then(backoff) + .should(inOrder) + .nextDelayMillis(2); + then(backoff) + .shouldHaveNoMoreInteractions(); + } + } + + private void waitUntilDiscoverInstancesCalledAtLeastOnce() { + waitUntilDiscoverInstancesCalledAtLeast(1); + } + + /** + * Waits for DiscoverInstances to be called at least a specified number + * of times. This method intentionally does not inspect the request. + * + * @param timesCalled The number of times to wait for it to be called. + */ + private void waitUntilDiscoverInstancesCalledAtLeast(final int timesCalled) { + final long waitTimeMillis = (long) timesCalled * WAIT_TIME_MULTIPLIER_MILLIS; + Awaitility.waitAtMost(waitTimeMillis, TimeUnit.MILLISECONDS) + .pollDelay(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> then(awsServiceDiscovery) + .should(atLeast(timesCalled)) + .discoverInstances(ArgumentMatchers.any(DiscoverInstancesRequest.class))); + } + + /** + * Waits until the give {@link AwsCloudMapPeerListProvider} has a peer list + * with a value greater than 0. + * + * @param objectUnderTest The object to wait for. + */ + private void waitUntilPeerListPopulated(final AwsCloudMapPeerListProvider objectUnderTest) { + Awaitility.waitAtMost(2, TimeUnit.SECONDS) + .pollDelay(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + final List actualPeers = objectUnderTest.getPeerList(); + assertThat(actualPeers, notNullValue()); + assertThat(actualPeers.size(), greaterThan(0)); + }); + } + + private static String generateRandomIp() { + final Random random = new Random(); + + return IntStream.range(0, 4) + .map(i -> random.nextInt(255)) + .mapToObj(Integer::toString) + .collect(Collectors.joining(".")); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..cbe2ed27e1 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java @@ -0,0 +1,86 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.regions.Region; + +import java.util.HashMap; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + +class AwsCloudMapPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String ENDPOINT = "ENDPOINT"; + private static final String PIPELINE_NAME = "pipelineName"; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + @BeforeEach + void setUp() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; + + pluginMetrics = mock(PluginMetrics.class); + + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME, UUID.randomUUID().toString()); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME, UUID.randomUUID().toString()); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_REGION, "us-east-1"); + + } + + @Test + void createPeerListProvider_with_valid_configurations() { + final PeerListProvider result = AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(AwsCloudMapPeerListProvider.class)); + } + + @ParameterizedTest + @ValueSource(strings = { + PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME, + PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME, + PeerForwarderConfig.AWS_REGION + }) + void createPeerListProvider_with_missing_required_property(final String propertyToRemove) { + pluginSetting.getSettings().remove(propertyToRemove); + + assertThrows(NullPointerException.class, + () -> AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + + } + + @ParameterizedTest + @ArgumentsSource(AllRegionsArgumentProvider.class) + void createPeerListProvider_with_all_current_regions(final Region region) { + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_REGION, region.toString()); + + final PeerListProvider result = AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(AwsCloudMapPeerListProvider.class)); + } + + static class AllRegionsArgumentProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Region.regions().stream().map(Arguments::of); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java deleted file mode 100644 index ef36e92982..0000000000 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; - -/** - * Test enum defined to provide an UNSUPPORTED value for use in unit testing. - */ -public enum DiscoveryMode { - STATIC, - DNS, - UNSUPPORTED -} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..739d94a89e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java @@ -0,0 +1,88 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; +import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroupBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DnsPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String VALID_ENDPOINT = "VALID.ENDPOINT"; + private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; + private static final String PIPELINE_NAME = "pipelineName"; + + @Mock + private DnsAddressEndpointGroupBuilder dnsAddressEndpointGroupBuilder; + @Mock + private DnsAddressEndpointGroup dnsAddressEndpointGroup; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + private CompletableFuture completableFuture; + + @BeforeEach + void setup() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; + completableFuture = CompletableFuture.completedFuture(null); + + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); + pluginMetrics = mock(PluginMetrics.class); + } + + @Test + void testCreateProviderDnsInstance() { + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, VALID_ENDPOINT); + + when(dnsAddressEndpointGroupBuilder.build()).thenReturn(dnsAddressEndpointGroup); + when(dnsAddressEndpointGroupBuilder.ttl(anyInt(), anyInt())).thenReturn(dnsAddressEndpointGroupBuilder); + when(dnsAddressEndpointGroup.whenReady()).thenReturn(completableFuture); + + try (MockedStatic armeriaMock = Mockito.mockStatic(DnsAddressEndpointGroup.class)) { + armeriaMock.when(() -> DnsAddressEndpointGroup.builder(anyString())).thenReturn(dnsAddressEndpointGroupBuilder); + + PeerListProvider result = DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(DnsPeerListProvider.class)); + } + } + + @Test + void testCreateProviderDnsInstanceWithNoHostname() { + assertThrows(NullPointerException.class, + () -> DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + + } + + @Test + void testCreateProviderDnsInstanceWithInvalidDomainName() { + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, INVALID_ENDPOINT); + + assertThrows(IllegalStateException.class, + () -> DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java index 0e5acc8c8e..ff778127f0 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java @@ -11,130 +11,78 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; +import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroupBuilder; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.concurrent.CompletableFuture; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) -public class PeerListProviderFactoryTest { +@ExtendWith(MockitoExtension.class) +class PeerListProviderFactoryTest { private static final String PLUGIN_NAME = "PLUGIN_NAME"; - private static final String ENDPOINT = "ENDPOINT"; - private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; private static final String PIPELINE_NAME = "pipelineName"; - @Mock - private DnsAddressEndpointGroupBuilder dnsAddressEndpointGroupBuilder; - @Mock - private DnsAddressEndpointGroup dnsAddressEndpointGroup; - - private CompletableFuture completableFuture; - private PluginSetting pluginSetting; private PeerListProviderFactory factory; - @Before - public void setup() { + @BeforeEach + void setup() { factory = new PeerListProviderFactory(); - pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()){{ setPipelineName(PIPELINE_NAME); }}; - completableFuture = CompletableFuture.completedFuture(null); - } - - @Test(expected = NullPointerException.class) - public void testUnsupportedNoDiscoveryMode() { - factory.createProvider(pluginSetting); - } - - @Test(expected = IllegalArgumentException.class) - public void testUndefinedDiscoveryModeEnum() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, "GARBAGE"); - - factory.createProvider(pluginSetting); - } - - @Test(expected = UnsupportedOperationException.class) - public void testUnsupportedDiscoveryModeEnum() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.UNSUPPORTED.toString()); - - factory.createProvider(pluginSetting); + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; } - @Test(expected = NullPointerException.class) - public void testCreateProviderStaticInstanceNoEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - - factory.createProvider(pluginSetting); + @Test + void createProvider_throws_when_no_DiscoveryMode_is_provided() { + assertThrows(NullPointerException.class, + () -> factory.createProvider(pluginSetting)); } @Test - public void testCreateProviderStaticInstanceWithEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Collections.singletonList(ENDPOINT)); - - PeerListProvider result = factory.createProvider(pluginSetting); + void createProvider_throws_for_undefined_DiscoveryMode() { + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, "GARBAGE"); - assertTrue(result instanceof StaticPeerListProvider); - assertEquals(1, result.getPeerList().size()); - assertTrue(result.getPeerList().contains(ENDPOINT)); + assertThrows(IllegalArgumentException.class, + () -> factory.createProvider(pluginSetting)); } - @Test(expected = IllegalStateException.class) - public void testCreateProviderStaticInstanceWithInvalidEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Arrays.asList(ENDPOINT, INVALID_ENDPOINT)); + @ParameterizedTest + @EnumSource(DiscoveryMode.class) + void createProvider_returns_correct_provider_for_all_DiscoveryModes(final DiscoveryMode discoveryMode) { + final String discoveryModeString = discoveryMode.toString(); + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, discoveryModeString.toLowerCase()); - factory.createProvider(pluginSetting); - } - - @Test - public void testCreateProviderDnsInstance() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + final PeerListProvider expectedProvider = mock(PeerListProvider.class); + final PeerListProvider actualProvider; - when(dnsAddressEndpointGroupBuilder.build()).thenReturn(dnsAddressEndpointGroup); - when(dnsAddressEndpointGroupBuilder.ttl(anyInt(), anyInt())).thenReturn(dnsAddressEndpointGroupBuilder); - when(dnsAddressEndpointGroup.whenReady()).thenReturn(completableFuture); + try (final MockedStatic enumMock = Mockito.mockStatic(DiscoveryMode.class)) { + final DiscoveryMode mockedModeEnum = mock(DiscoveryMode.class); + enumMock.when(() -> DiscoveryMode.valueOf(discoveryModeString)).thenReturn(mockedModeEnum); - try (MockedStatic armeriaMock = Mockito.mockStatic(DnsAddressEndpointGroup.class)) { - armeriaMock.when(() -> DnsAddressEndpointGroup.builder(anyString())).thenReturn(dnsAddressEndpointGroupBuilder); + when(mockedModeEnum.create(eq(pluginSetting), any(PluginMetrics.class))).thenReturn(expectedProvider); - PeerListProvider result = factory.createProvider(pluginSetting); - - assertTrue(result instanceof DnsPeerListProvider); + actualProvider = factory.createProvider(pluginSetting); } - } - - @Test(expected = NullPointerException.class) - public void testCreateProviderDnsInstanceWithNoHostname() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - factory.createProvider(pluginSetting); + assertThat(actualProvider, sameInstance(expectedProvider)); } - @Test(expected = IllegalStateException.class) - public void testCreateProviderDnsInstanceWithInvalidDomainName() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, INVALID_ENDPOINT); - - factory.createProvider(pluginSetting); - } } diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..b4b5f0b4e6 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java @@ -0,0 +1,63 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +class StaticPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String ENDPOINT = "ENDPOINT"; + private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; + private static final String PIPELINE_NAME = "pipelineName"; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + @BeforeEach + void setup() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()){{ setPipelineName(PIPELINE_NAME); }}; + + pluginMetrics = mock(PluginMetrics.class); + } + + @Test + void testCreateProviderStaticInstanceNoEndpoints() { + + assertThrows(NullPointerException.class, + () -> StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + + @Test + void testCreateProviderStaticInstanceWithEndpoints() { + pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Collections.singletonList(ENDPOINT)); + + PeerListProvider result = StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(StaticPeerListProvider.class)); + assertEquals(1, result.getPeerList().size()); + assertTrue(result.getPeerList().contains(ENDPOINT)); + } + + @Test + void testCreateProviderStaticInstanceWithInvalidEndpoints() { + pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Arrays.asList(ENDPOINT, INVALID_ENDPOINT)); + + assertThrows(IllegalStateException.class, + () -> StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties b/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..9462a0739e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties @@ -0,0 +1,10 @@ +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n + +rootLogger.level = warn +rootLogger.appenderRef.stdout.ref = STDOUT + +logger.core.name = com.amazon.dataprepper +logger.core.level = debug diff --git a/data-prepper-plugins/service-map-stateful/README.md b/data-prepper-plugins/service-map-stateful/README.md index ccefa37d5a..c9b75041ee 100644 --- a/data-prepper-plugins/service-map-stateful/README.md +++ b/data-prepper-plugins/service-map-stateful/README.md @@ -1,6 +1,6 @@ # Service-Map Stateful Prepper -This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. +This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. # Usages Example `.yaml` configuration: @@ -11,16 +11,16 @@ prepper: ## Configurations -* window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. +* window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. ## Metrics Besides common metrics in [AbstractPrepper](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/prepper/AbstractPrepper.java), service-map-stateful prepper introduces the following custom metrics. ### Gauge - `spansDbSize`: measures total spans byte sizes in MapDB across the current and previous window durations. -- `traceGroupDbSize`: measures total trace group byte sizes in MapDB across the current and previous trace group window durations. +- `traceGroupDbSize`: measures total trace group byte sizes in MapDB across the current and previous trace group window durations. ## Developer Guide This plugin is compatible with Java 8. See - [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 5da7e9c67f..a12df7d8e7 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -23,15 +23,15 @@ repositories { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') - compile project(':data-prepper-plugins:mapdb-prepper-state') - testCompile project(':data-prepper-api').sourceSets.test.output - implementation "io.micrometer:micrometer-core:1.6.6" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + implementation project(':data-prepper-plugins:mapdb-prepper-state') + testImplementation project(':data-prepper-api').sourceSets.test.output + implementation "io.micrometer:micrometer-core:1.7.2" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation "org.mockito:mockito-inline:3.11.2" } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java index e0f6e51f00..1af17702d0 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java @@ -18,8 +18,8 @@ import com.amazon.dataprepper.model.prepper.AbstractPrepper; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.plugins.prepper.state.MapDbPrepperState; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.google.common.primitives.SignedBytes; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import org.slf4j.Logger; @@ -32,13 +32,11 @@ import java.util.Collections; import java.util.HashSet; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; -import java.util.stream.Collectors; -import java.util.stream.Stream; @SingleThread @DataPrepperPlugin(name = "service_map_stateful", type = PluginType.PREPPER) @@ -57,17 +55,16 @@ public class ServiceMapStatefulPrepper extends AbstractPrepper previousWindow; private volatile static MapDbPrepperState currentWindow; private volatile static MapDbPrepperState previousTraceGroupWindow; private volatile static MapDbPrepperState currentTraceGroupWindow; //TODO: Consider keeping this state in a db - private volatile static HashSet relationshipState = new HashSet<>(); + private static final Set RELATIONSHIP_STATE = Sets.newConcurrentHashSet(); private static File dbPath; private static Clock clock; - private static int processWorkers; private final int thisPrepperId; @@ -88,15 +85,18 @@ public ServiceMapStatefulPrepper(final long windowDurationMillis, ServiceMapStatefulPrepper.clock = clock; this.thisPrepperId = preppersCreated.getAndIncrement(); + if (isMasterInstance()) { previousTimestamp = ServiceMapStatefulPrepper.clock.millis(); ServiceMapStatefulPrepper.windowDurationMillis = windowDurationMillis; ServiceMapStatefulPrepper.dbPath = createPath(databasePath); - ServiceMapStatefulPrepper.processWorkers = processWorkers; + currentWindow = new MapDbPrepperState<>(dbPath, getNewDbName(), processWorkers); previousWindow = new MapDbPrepperState<>(dbPath, getNewDbName() + EMPTY_SUFFIX, processWorkers); currentTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName(), processWorkers); previousTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName() + EMPTY_SUFFIX, processWorkers); + + allThreadsCyclicBarrier = new CyclicBarrier(processWorkers); } pluginMetrics.gauge(SPANS_DB_SIZE, this, serviceMapStateful -> serviceMapStateful.getSpansDbSize()); @@ -176,88 +176,81 @@ public Collection> doExecute(Collection containing json representation of ServiceMapRelationships found */ private Collection> evaluateEdges() { + LOG.info("Evaluating service map edges"); try { - final Stream previousStream = previousWindow.iterate(relationshipIterationFunction, preppersCreated.get(), thisPrepperId).stream().flatMap(serviceMapEdgeStream -> serviceMapEdgeStream); - final Stream currentStream = currentWindow.iterate(relationshipIterationFunction, preppersCreated.get(), thisPrepperId).stream().flatMap(serviceMapEdgeStream -> serviceMapEdgeStream); - - final Collection> serviceDependencyRecords = - Stream.concat(previousStream, currentStream).filter(Objects::nonNull) - .filter(serviceMapRelationship -> !relationshipState.contains(serviceMapRelationship)) - .map(serviceMapRelationship -> { - try { - relationshipState.add(serviceMapRelationship); - return new Record<>(OBJECT_MAPPER.writeValueAsString(serviceMapRelationship)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toSet()); - - if (edgeEvaluationLatch == null) { - initEdgeEvaluationLatch(); - } - doneEvaluatingEdges(); - waitForEvaluationFinish(); + final Collection> serviceDependencyRecords = new HashSet<>(); + + serviceDependencyRecords.addAll(iteratePrepperState(previousWindow)); + serviceDependencyRecords.addAll(iteratePrepperState(currentWindow)); + LOG.info("Done evaluating service map edges"); + + // Wait for all workers before rotating windows + allThreadsCyclicBarrier.await(); if (isMasterInstance()) { rotateWindows(); - resetWorkState(); - } else { - waitForRotationFinish(); } + // Wait for all workers before exiting this method + allThreadsCyclicBarrier.await(); + return serviceDependencyRecords; - } catch (InterruptedException e) { + } catch (InterruptedException | BrokenBarrierException e) { throw new RuntimeException(e); } } - private static synchronized void initEdgeEvaluationLatch() { - if (edgeEvaluationLatch == null) { - edgeEvaluationLatch = new CountDownLatch(preppersCreated.get()); - } - } + private Collection> iteratePrepperState(final MapDbPrepperState prepperState) { + final Collection> serviceDependencyRecords = new HashSet<>(); - /** - * This function is used to iterate over the current window and find parent/child relationships in the current and - * previous windows. - */ - private final BiFunction> relationshipIterationFunction = new BiFunction>() { - @Override - public Stream apply(byte[] s, ServiceMapStateData serviceMapStateData) { - return lookupParentSpan(serviceMapStateData, true); - } - }; - - private Stream lookupParentSpan(final ServiceMapStateData serviceMapStateData, final boolean checkPrev) { - if (serviceMapStateData.parentSpanId != null) { - final ServiceMapStateData parentStateData = getParentStateData(serviceMapStateData.parentSpanId, checkPrev); - final String traceGroupName = getTraceGroupName(serviceMapStateData.traceId); - if (traceGroupName != null && parentStateData != null && !parentStateData.serviceName.equals(serviceMapStateData.serviceName)) { - return Stream.of( - ServiceMapRelationship.newDestinationRelationship(parentStateData.serviceName, parentStateData.spanKind, serviceMapStateData.serviceName, serviceMapStateData.name, traceGroupName), - //This extra edge is added for compatibility of the index for both stateless and stateful preppers - ServiceMapRelationship.newTargetRelationship(serviceMapStateData.serviceName, serviceMapStateData.spanKind, serviceMapStateData.serviceName, serviceMapStateData.name, traceGroupName) - ); - } - } - return Stream.empty(); - } + if (prepperState.getAll() != null && !prepperState.getAll().isEmpty()) { + prepperState.getIterator(preppersCreated.get(), thisPrepperId).forEachRemaining(entry -> { + final ServiceMapStateData child = entry.getValue(); - /** - * Checks both current and previous windows for the given parent span id - * - * @param spanId - * @return ServiceMapStateData for the parent span, if exists. Otherwise null - */ - private ServiceMapStateData getParentStateData(final byte[] spanId, final boolean checkPrev) { - try { - final ServiceMapStateData serviceMapStateData = currentWindow.get(spanId); - return serviceMapStateData != null ? serviceMapStateData : checkPrev ? previousWindow.get(spanId) : null; - } catch (RuntimeException e) { - LOG.error("Caught exception trying to get parent state data", e); - return null; + if (child.parentSpanId == null) { + return; + } + + ServiceMapStateData parent = currentWindow.get(child.parentSpanId); + if (parent == null) { + parent = previousWindow.get(child.parentSpanId); + } + + + final String traceGroupName = getTraceGroupName(child.traceId); + if (traceGroupName == null || parent == null || parent.serviceName.equals(child.serviceName)) { + return; + } + + final ServiceMapRelationship destinationRelationship = + ServiceMapRelationship.newDestinationRelationship(parent.serviceName, + parent.spanKind, child.serviceName, child.name, traceGroupName); + final ServiceMapRelationship targetRelationship = ServiceMapRelationship.newTargetRelationship(child.serviceName, + child.spanKind, child.serviceName, child.name, traceGroupName); + + + // check if relationshipState has it + if (!RELATIONSHIP_STATE.contains(destinationRelationship)) { + try { + serviceDependencyRecords.add(new Record<>(OBJECT_MAPPER.writeValueAsString(destinationRelationship))); + RELATIONSHIP_STATE.add(destinationRelationship); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + if (!RELATIONSHIP_STATE.contains(targetRelationship)) { + try { + serviceDependencyRecords.add(new Record<>(OBJECT_MAPPER.writeValueAsString(targetRelationship))); + RELATIONSHIP_STATE.add(targetRelationship); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); } + + return serviceDependencyRecords; } /** @@ -298,64 +291,30 @@ public void shutdown() { // TODO: Temp code, complex instance creation logic should be moved to a separate class static void resetStaticCounters() { preppersCreated.set(0); - edgeEvaluationLatch = null; } - /** - * Indicate/notify that this instance has finished evaluating edges - */ - private void doneEvaluatingEdges() { - edgeEvaluationLatch.countDown(); - } - - /** - * Wait on all instances to finish evaluating edges - * - * @throws InterruptedException - */ - private void waitForEvaluationFinish() throws InterruptedException { - edgeEvaluationLatch.await(); - } - - /** - * Indicate that window rotation is complete - */ - private void doneRotatingWindows() { - windowRotationLatch.countDown(); - } - - /** - * Wait on window rotation to complete - * - * @throws InterruptedException - */ - private void waitForRotationFinish() throws InterruptedException { - windowRotationLatch.await(); - } - - /** - * Reset state that indicates whether edge evaluation and window rotation is complete - */ - private void resetWorkState() { - windowRotationLatch = new CountDownLatch(1); - edgeEvaluationLatch = new CountDownLatch(preppersCreated.get()); - } /** * Rotate windows for prepper state */ - private void rotateWindows() { - LOG.debug("Rotating windows at " + clock.instant().toString()); - previousWindow.delete(); - previousTraceGroupWindow.delete(); + private void rotateWindows() throws InterruptedException { + LOG.info("Rotating service map windows at " + clock.instant().toString()); + + MapDbPrepperState tempWindow = previousWindow; previousWindow = currentWindow; - currentWindow = new MapDbPrepperState<>(dbPath, getNewDbName(), processWorkers); + currentWindow = tempWindow; + currentWindow.clear(); + + tempWindow = previousTraceGroupWindow; previousTraceGroupWindow = currentTraceGroupWindow; - currentTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName(), processWorkers); + currentTraceGroupWindow = tempWindow; + currentTraceGroupWindow.clear(); + previousTimestamp = clock.millis(); - doneRotatingWindows(); + LOG.info("Done rotating service map windows"); } + /** * @return Spans database size in bytes */ diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java index 6ca54fd37f..a912a9d136 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java @@ -103,8 +103,8 @@ public void testTraceGroups() throws Exception { Mockito.when(clock.instant()).thenReturn(Instant.now()); ExecutorService threadpool = Executors.newCachedThreadPool(); final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 16, PLUGIN_SETTING); - final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 16, PLUGIN_SETTING); + final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); + final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); final byte[] rootSpanId1 = ServiceMapTestUtils.getRandomBytes(8); final byte[] rootSpanId2 = ServiceMapTestUtils.getRandomBytes(8); @@ -205,13 +205,11 @@ public void testTraceGroups() throws Exception { new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") .add(ServiceMapStatefulPrepper.SPANS_DB_SIZE).toString()); Assert.assertEquals(1, spansDbSizeMeasurement.size()); - Assert.assertEquals(2097152, spansDbSizeMeasurement.get(0).getValue(), 0); final List traceGroupDbSizeMeasurement = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") .add(ServiceMapStatefulPrepper.TRACE_GROUP_DB_SIZE).toString()); Assert.assertEquals(1, traceGroupDbSizeMeasurement.size()); - Assert.assertEquals(2097152, traceGroupDbSizeMeasurement.get(0).getValue(), 0); //Make sure that future relationships that are equivalent are caught by cache diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index ca3b59c133..f445e0d033 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -2,30 +2,39 @@ AWSTemplateFormatVersion: "2010-09-09" Description: "Template to install Data Prepper on a single EC2 instance" Parameters: - AWSElasticsearchEndpoint: + AmazonEsEndpoint: Description: Endpoint of the AWS Elasticsearch Service domain (including https://) Type: String AllowedPattern: https:\/\/[a-z0-9-\.]+(es.amazonaws.com) ConstraintDescription: must be a valid Amazon Elasticsearch Service domain endpoint starting with https:// - AWSElasticsearchRegion: + AmazonEsRegion: Description: Region of the AWS Elasticsearch Service domain Type: String AllowedPattern: "[a-z]+-[a-z]+-[0-9]+" Default: us-east-1 ConstraintDescription: must be a valid AWS region (e.g. us-west-2) - AWSElasticsearchSubnetId: + AmazonEsSubnetId: Description: The subnet ID of the AWS Elasticsearch Service domain (Leave blank if the domain is not in a VPC) Type: String + Username: + Description: The username of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role) + Type: String + Default: "" + Password: + Description: The password of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role) + Type: String + Default: "" DataPrepperVersion: Description: Version of Data Prepper to download and run Type: String AllowedPattern: "[0-9]+\\.[0-9]+\\.[0-9]+[a-z-]*" - Default: "0.8.0-beta" + Default: "1.0.0" ConstraintDescription: must be a valid release number IAMRole: Description: Pre-existing IAM Role to associate with the EC2 instance, to be used for authentication when calling Elasticsearch + (Leave blank if the domain is configured with HTTP basic authentication in Fine-grained access control) Type: String - AllowedPattern: "[a-zA-Z0-9+=,\\.@\\-_]+" + AllowedPattern: "^$|[a-zA-Z0-9+=,\\.@\\-_]+" Default: DataPrepperRole ConstraintDescription: must be a valid IAM role name InstanceType: @@ -50,9 +59,39 @@ Parameters: Default: 0.0.0.0/0 AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Common + Parameters: + - IAMRole + - Label: + default: Amazon Elasticsearch Service Domain + Parameters: + - AmazonEsEndpoint + - AmazonEsRegion + - AmazonEsSubnetId + - Username + - Password + - Label: + default: Data-Prepper Configuration + Parameters: + - DataPrepperVersion + - InstanceType + - KeyName + - LatestAmi + - SSHLocation Conditions: - DomainIsPublic: !Equals [!Ref AWSElasticsearchSubnetId, ""] + DomainIsPublic: !Equals [!Ref AmazonEsSubnetId, ""] DomainIsInVPC: !Not [Condition: DomainIsPublic] + NoMasterUser: !And + - !Equals + - !Ref Username + - "" + - !Equals + - !Ref Password + - "" Resources: EC2Instance: Type: AWS::EC2::Instance @@ -70,49 +109,68 @@ Resources: owner: root group: root "/etc/data-prepper/pipelines.yaml": - content: !Sub | - entry-pipeline: - delay: "100" - source: - otel_trace_source: - ssl: false - health_check_service: true - sink: - - pipeline: - name: "raw-pipeline" - - pipeline: - name: "service-map-pipeline" - raw-pipeline: - source: - pipeline: - name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: - sink: - - opensearch: - hosts: [ "${AWSElasticsearchEndpoint}" ] - aws_sigv4: true - aws_region: "${AWSElasticsearchRegion}" - trace_analytics_raw: true - service-map-pipeline: - delay: "100" - source: - pipeline: - name: "entry-pipeline" - prepper: - - service_map_stateful: - sink: - - opensearch: - hosts: [ "${AWSElasticsearchEndpoint}" ] - aws_sigv4: true - aws_region: "${AWSElasticsearchRegion}" - trace_analytics_service_map: true + content: !Sub + - | + entry-pipeline: + delay: "100" + source: + otel_trace_source: + ssl: false + health_check_service: true + sink: + - pipeline: + name: "raw-pipeline" + - pipeline: + name: "service-map-pipeline" + raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - otel_trace_raw_prepper: + sink: + - opensearch: ${rawSpanConfig} + service-map-pipeline: + delay: "100" + source: + pipeline: + name: "entry-pipeline" + prepper: + - service_map_stateful: + sink: + - opensearch: ${serviceMapConfig} + - rawSpanConfig: !If + - NoMasterUser + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: true\n + \ aws_region: \"${AmazonEsRegion}\"\n + \ trace_analytics_raw: true" + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: false\n + \ username: \"${Username}\"\n + \ password: \"${Password}\"\n + \ trace_analytics_raw: true" + serviceMapConfig: !If + - NoMasterUser + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: true\n + \ aws_region: \"${AmazonEsRegion}\"\n + \ trace_analytics_service_map: true" + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: false\n + \ username: \"${Username}\"\n + \ password: \"${Password}\"\n + \ trace_analytics_service_map: true" mode: "000400" owner: root group: root Properties: SubnetId: - !If [DomainIsInVPC, !Ref AWSElasticsearchSubnetId, !Ref "AWS::NoValue"] + !If [DomainIsInVPC, !Ref AmazonEsSubnetId, !Ref "AWS::NoValue"] InstanceType: Ref: InstanceType IamInstanceProfile: @@ -130,10 +188,19 @@ Resources: export RELEASE=opendistroforelasticsearch-data-prepper-${DataPrepperVersion}-linux-x64 yum install java-11-amazon-corretto-headless -y wget https://github.com/opendistro-for-elasticsearch/data-prepper/releases/download/v${DataPrepperVersion}/$RELEASE.tar.gz -O /tmp/$RELEASE.tar.gz - tar -xzf /tmp/$RELEASE.tar.gz --directory /usr/local/bin - /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default - nohup /usr/local/bin/$RELEASE/data-prepper-tar-install.sh /etc/data-prepper/pipelines.yaml /etc/data-prepper/data-prepper-config.yaml > /var/log/data-prepper.out & - /opt/aws/bin/cfn-signal --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + wget_exit_code = $? + if [ $wget_exit_code -ne 0 ] + then + /opt/aws/bin/cfn-signal -e $wget_exit_code --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + else + tar -xzf /tmp/$RELEASE.tar.gz --directory /usr/local/bin + /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default + nohup /usr/local/bin/$RELEASE/data-prepper-tar-install.sh /etc/data-prepper/pipelines.yaml /etc/data-prepper/data-prepper-config.yaml > /var/log/data-prepper.out & + data_prepper_pid=$! + sleep 5s + ps -p $data_prepper_pid + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + fi CreationPolicy: ResourceSignal: Count: 1 diff --git a/docs/readme/configuration.md b/docs/readme/configuration.md index 7664cfff6c..27393815bd 100644 --- a/docs/readme/configuration.md +++ b/docs/readme/configuration.md @@ -59,6 +59,7 @@ Data Prepper allows the following properties to be configured: * `keyStorePassword` string password for keystore. Optional, defaults to empty string * `privateKeyPassword` string password for private key within keystore. Optional, defaults to empty string * `serverPort`: integer port number to use for server APIs. Defaults to `4900` +* `metricRegistries`: list of metrics registries for publishing the generated metrics. Defaults to Prometheus; Prometheus and CloudWatch are currently supported. Example Data Prepper configuration file (data-prepper-config.yaml): ```yaml @@ -67,4 +68,5 @@ keyStoreFilePath: "/usr/share/data-prepper/keystore.jks" keyStorePassword: "password" privateKeyPassword: "other_password" serverPort: 1234 +metricRegistries: [Prometheus] ``` diff --git a/docs/readme/monitoring.md b/docs/readme/monitoring.md index 1c51882118..e5f30fb958 100644 --- a/docs/readme/monitoring.md +++ b/docs/readme/monitoring.md @@ -1,6 +1,6 @@ # Monitoring Metrics in Data Prepper are instrumented using [Micrometer.io](https://micrometer.io/). There are two types of metrics: -(1) JVM and system metrics; (2) Plugin metrics. Prometheus is used as the metrics backend. +(1) JVM and system metrics; (2) Plugin metrics. Prometheus is used as the default metrics backend. ## JVM and system metrics @@ -13,9 +13,10 @@ JVM and system metrics in Data Prepper follows pre-defined names in Micrometer.i ### Serving -Metrics are served from the **/metrics/sys** endpoint on the Data Prepper server. The format +By default, metrics are served from the **/metrics/sys** endpoint on the Data Prepper server. The format is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics, e.g. -[Grafana](https://prometheus.io/docs/visualization/grafana/). +[Grafana](https://prometheus.io/docs/visualization/grafana/). The configuration can be updated to serve metrics to other +registries like CloudWatch which does not require/host the endpoint but publishes the metrics directly to cloudwatch. ## Plugin metrics @@ -51,5 +52,7 @@ Metrics follow a naming convention of **PIPELINE_NAME_PLUGIN_NAME_METRIC_NAME** would have a qualified name of **output-pipeline_elasticsearch_sink_recordsIn**. ### Serving -Metrics are served from the **metrics/prometheus** endpoint on the Data Prepper server. The format -is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics. \ No newline at end of file +By default, metrics are served from the **metrics/prometheus** endpoint on the Data Prepper server. The format +is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics. The configuration +can be updated to serve metrics to other registries like CloudWatch which does not require/host the endpoint but +publishes the metrics directly to cloudwatch. \ No newline at end of file diff --git a/docs/readme/project_setup.md b/docs/readme/project_setup.md index 02108dfd27..2e8bb877ab 100644 --- a/docs/readme/project_setup.md +++ b/docs/readme/project_setup.md @@ -1,14 +1,43 @@ # Project Setup +## Installation Prerequisites + +### JDK Versions + +Running Data Prepper requires JDK 8 and above. + +Running the integration tests requires JDK 14 or 15. + + ## Building from source -To build the project from source, run + +The assemble task will build the Jar files without running the integration +tests. You can use these jar files for running DataPrepper. If you are just +looking to use DataPrepper and modify it, this build +is faster than running the integration test suite and only requires JDK 8+. + +To build the project from source, run ``` -./gradlew build +./gradlew assemble ``` from the project root. +### Full Project Build + +Running the build command will assemble the Jar files needed +for running DataPrepper. It will also run the integration test +suite. + +To build, run + +``` +./gradlew build +``` + +from the project root. + ## Running the project After building, the project can be run from the executable JAR **data-prepper-core-$VERSION** @@ -35,9 +64,13 @@ APIs are available: * /shutdown * starts a graceful shutdown of the Data Prepper * /metrics/prometheus - * returns a scrape of the Data Prepper metrics in Prometheus text format + * returns a scrape of the Data Prepper metrics in Prometheus text format. This API is available provided + `metricsRegistries` parameter in data prepper configuration file `data-prepper-config.yaml` has `Prometheus` as one + of the registry * /metrics/sys - * returns JVM metrics in Prometheus text format + * returns JVM metrics in Prometheus text format. This API is available provided `metricsRegistries` parameter in data + prepper configuration file `data-prepper-config.yaml` has `Prometheus` as one of the registry + ### Running the example app To run the example app against your local changes, use the docker found [here](https://github.com/opensearch-project/data-prepper/tree/master/examples/dev/trace-analytics-sample-app) diff --git a/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md b/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md new file mode 100644 index 0000000000..17812bf7bb --- /dev/null +++ b/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md @@ -0,0 +1,125 @@ +# otel-v1-apm-service-map-index-template + +## Description +Documents in this index correspond to edges in a service map. Edges are created when a request crosses service boundaries. Documents will exclusively contain either a _destination_ or a _target_: +* Destination: corresponds to a client span calling another service. The _destination_ is the other service being called. +* Target: corresponds to a server span. The _target_ is the operation or API being called by the client. + +```json +{ + "version": 0, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "hashId": { + "ignore_above": 1024, + "type": "keyword" + }, + "serviceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "traceGroupName": { + "ignore_above": 1024, + "type": "keyword" + } + } + } +} +``` + +## Fields +* hashId - A deterministic hash of this relationship. +* kind - The span kind, corresponding to the source of the relationship. See [OpenTelemetry - SpanKind](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind). +* serviceName - The name of the service which emitted the span. Currently derived from the `opentelemetry.proto.resource.v1.Resource` associated with the span. +* destination.domain - The serviceName of the service being called by this client. +* destination.resource - The span name (API, operation, etc.) being called by this client. +* target.domain - The serviceName of the service being called by a client. +* target.resource - The span name (API, operation, etc.) being called by a client. +* traceGroupName - The top-level span name which started the request chain. + +## Example Documents +The two example documents below illustrate the "inventory" service calling the "database" service's `updateItem` API. +```json +{ + "_index": "otel-v1-apm-service-map", + "_type": "_doc", + "_id": "7/jRp2VF7544pBN6+mK2vw==", + "_score": 1, + "_source": { + "serviceName": "inventory", + "kind": "SPAN_KIND_CLIENT", + "destination": { + "resource": "updateItem", + "domain": "database" + }, + "target": null, + "traceGroupName": "client_checkout", + "hashId": "7/jRp2VF7544pBN6+mK2vw==" + } +} +``` + +```json +{ + "_index": "otel-v1-apm-service-map", + "_type": "_doc", + "_id": "lZcUyuhGYfnaQqt+r73njA==", + "_version": 3, + "_score": 0, + "_source": { + "serviceName": "database", + "kind": "SPAN_KIND_SERVER", + "destination": null, + "target": { + "resource": "updateItem", + "domain": "database" + }, + "traceGroupName": "client_checkout", + "hashId": "lZcUyuhGYfnaQqt+r73njA==" + } +} +``` + diff --git a/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md new file mode 100644 index 0000000000..5dea6ebcf8 --- /dev/null +++ b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md @@ -0,0 +1,182 @@ +# otel-v1-apm-span-index-template + +## Description +Documents in this index correspond to spans following the [OpenTelemetry tracing specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md). Many fields are directly copied from the span, however some fields are derived and not present in the original span. + +```json +{ + "version": 0, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "resource_attributes_map": { + "mapping": { + "type":"keyword" + }, + "path_match":"resource.attributes.*" + } + }, + { + "attributes_map": { + "mapping": { + "type":"keyword" + }, + "path_match":"attributes.*" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "parentSpanId": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroupFields": { + "properties": { + "endTime": { + "type": "date_nanos" + }, + "durationInNanos": { + "type": "long" + }, + "statusCode": { + "type": "integer" + } + } + }, + "kind": { + "ignore_above": 128, + "type": "keyword" + }, + "startTime": { + "type": "date_nanos" + }, + "endTime": { + "type": "date_nanos" + }, + "status": { + "properties": { + "code": { + "type": "integer" + }, + "message": { + "type": "keyword" + } + } + }, + "serviceName": { + "type": "keyword" + }, + "durationInNanos": { + "type": "long" + }, + "events": { + "type": "nested", + "properties": { + "time": { + "type": "date_nanos" + } + } + }, + "links": { + "type": "nested" + } + } + } +} +``` + +## Fields +Many fields are either copied or derived from the [trace specification protobuf](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) format. + +* traceId - A unique identifier for a trace. All spans from the same trace share the same traceId. +* spanId - A unique identifier for a span within a trace, assigned when the span is created. +* traceState - Conveys information about request position in multiple distributed tracing graphs. +* parentSpanId - The `spanId` of this span's parent span. If this is a root span, then this field must be empty. +* name - A description of the span's operation. +* kind - The type of span. See [OpenTelemetry - SpanKind](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind). +* startTime - The start time of the span. +* endTime - The end time of the span. +* durationInNanos - Difference in nanoseconds between `startTime` and `endTime`. +* serviceName - Currently derived from the `opentelemetry.proto.resource.v1.Resource` associated with the span, the resource from the span originates. +* events - A list of events. See [OpenTelemetry - Events](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#add-events). +* links - A list of linked spans. See [OpenTelemetry - Links](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#specifying-links). +* droppedAttributesCount - The number of attributes that were discarded. +* droppedEventsCount - The number of events that were discarded. +* droppedLinksCount - The number of links that were dropped. +* traceGroup - A derived field, the `name` of the trace's root span. +* traceGroupFields.endTime - A derived field, the `endTime` of the trace's root span. +* traceGroupFields.statusCode - A derived field, the `status.code` of the trace's root span. +* traceGroupFields.durationInNanos - A derived field, the `durationInNanos` of the trace's root span. +* span.attributes.* - All span attributes are split into a list of keywords. +* resource.attributes.* - All resource attributes are split into a list of keywords. +* status.code - The status of the span. See [OpenTelemetry - Status](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status). + + +## Example Documents + +```json +{ + "_index": "otel-v1-apm-span-000006", + "_type": "_doc", + "_id": "fe0e3811627189df", + "_score": 1, + "_source": { + "traceId": "0000000000000000856bfa5aeba5ec77", + "spanId": "fe0e3811627189df", + "traceState": "", + "parentSpanId": "856bfa5aeba5ec77", + "name": "/getcart", + "kind": "SPAN_KIND_UNSPECIFIED", + "startTime": "2021-05-18T18:58:44.695Z", + "endTime": "2021-05-18T18:58:44.760Z", + "durationInNanos": 65000000, + "serviceName": "cartservice", + "events": [], + "links": [], + "droppedAttributesCount": 0, + "droppedEventsCount": 0, + "droppedLinksCount": 0, + "traceGroup": "/cart", + "traceGroupFields.endTime": "2021-05-18T18:58:44.983Z", + "traceGroupFields.statusCode": 0, + "traceGroupFields.durationInNanos": 387000000, + "span.attributes.http@method": "GET", + "span.attributes.http@url": "http://cartservice/GetCart", + "span.attributes.instance": "cartservice-d847fdcf5-j6s2f", + "span.attributes.version": "v5", + "span.attributes.region": "us-east-1", + "resource.attributes.service@name": "cartservice", + "span.attributes.net@host@ip": "172.22.0.8", + "status.code": 0 + }, + "fields": { + "startTime": [ + "2021-05-18T18:58:44.695Z" + ], + "endTime": [ + "2021-05-18T18:58:44.760Z" + ] + } +} +``` + diff --git a/docs/schemas/trace-analytics/readme.md b/docs/schemas/trace-analytics/readme.md new file mode 100644 index 0000000000..1e028e0bd8 --- /dev/null +++ b/docs/schemas/trace-analytics/readme.md @@ -0,0 +1,103 @@ +# Trace Analytics Schema Versioning + +The purpose of this document is to outline the structure and versioning formats followed by Data Prepper (DP) the Trace Analytics (TA) plugin for Kibana and OpenSearch Dashboards. Each individual schema shall have its own supporting document explaining its structure, fields, and purpose. + +## Tenets + +1. Schemas shall follow the [Semantic Versioning 2.0.0 spec](https://semver.org/), excluding patch version numbers. +2. Schema versions shall be detached from Data Prepper and Trace Analytics plugin versions. +3. Index and index template names shall only include the major version (e.g. "otel-v1-apm-span"). The minor version shall be included within the actual schema as a field. +4. Forward and backward-compatibility promises shall only apply to schemas of the same major version. +5. A major version increase shall require Data Prepper (writer) artifacts to be made available before Trace Analytics plugin (reader) updates are made available. +6. A major version increase shall result in a new index template and indexes being created in an Elasticsearch/OpenSearch cluster. + +## Versioning Format + +A schema will be versioned following the [Semantic Versioning 2.0.0 spec](https://semver.org/). A schema version will include a major and minor version number. Patch version numbers will not be used as "patching" is not applicable to a versioned schema; all changes no matter how trivial will have implications. + +1. **Major versions** will be incremented for breaking, backwards-incompatible changes. +2. **Minor versions** will be incremented for backwards-compatible feature additions. This can be thought of an "append-only" change to the schema. + +**Schema versions are detached from Data Prepper and Trace Analytics plugin versions.** A TA plugin version increase does not necessarily affect the version of the schema or Data Prepper. Instead, both DP and TA versions will be *compatible* with a specific schema version. Examples include: + +* Trace Analytics plugin v1.5 includes features built on schema version 1.2.0 +* Data Prepper v1.1 emits documents following schema version 1.2.0 + +### Major version changes + +Major version changes include removing a field, renaming a field, or changing an existing field's datatype. + +* Schema 1.0 to 2.0 *changes the type of a field* from Keyword to Numeric +* Schema 1.0 to 2.0 *removes* *field* "latency" from the schema +* Schema 1.0 to 2.0 *renames field* "end" to "endTime" + * A rename is effectively a field addition and removal in a single operation + +### Minor version changes + +Minor version changes include adding a new field or adding a new nested field. + +* Schema 1.2 to 1.3 adds a new field, "fieldC", as a Keyword +* Schema 2.11 to 2.12 adds a new nested field, "name" to an existing collection, "parentSpan", resulting in "parentSpan.name" + +## Compatibility Promises + +The following compatibility promises are made *only for schemas of the same major version*. + +* ***Backwards compatibility*** - Trace Analytics UI features built on version 1.x of the schema **will not break, but may degrade** if data from a **prior** 1.x schema version is used. +* ***Forwards compatibility*** - Trace Analytics UI features built on version 1.x of the schema **will not break** if data from a **later** 1.x schema version is used. + +### Read-compatibility examples + +1. A plugin built on schema 1.1 but consuming schema 1.2 data will function 100% without issue +2. A plugin built on schema 1.2 but consuming schema 1.1 data will continue to function, however some features might be degraded +3. A plugin built on schema 1.0 but consuming schema 2.0 data is not guaranteed to function +4. A plugin built on schema 2.0 but consuming schema 1.0 data is not guaranteed to function + +### Write-compatibility examples + +1. A writer built on schema 1.2 but writing to a cluster containing schema 1.1 data will succeed +2. A writer built on schema 1.1 but writing to a cluster containing schema 1.2 data will succeed +3. A writer built on schema 1.0 but writing to a cluster containing schema 2.0 data will succeed +4. A writer built on schema 2.0 but writing to a cluster containing schema 1.0 data will succeed + +### Handling minor version updates + +Minor version updates will occur as new fields are needed to support new Trace Analytics features. The steps to update a schema minor version are to: + +1. Ensure both TA and DP owners are aligned with requirements +2. Update the schema in the schema repo + 1. Add new fields to the schema JSON + 2. Increment the `version` field of the schema by 1 + 3. Update the documentation to describe the new fields +3. Add test data to the TA plugin test suite + 1. Don't update existing data in place, instead add new data following the new schema version. The test suite is expected to pass with a range of minor versions being tested at the same time. +4. Update DP to start emitting documents following the new schema version +5. Add new features to the TA plugin utilizing the new data + +### Handling major version updates + +As schema owners, we will do our best to avoid introducing major version changes. However, as our schemas are heavily tied to the OpenTelemetry spec, there is always the risk of an upstream backwards-incompatible change requiring a major version increase. + +The following procedures may be considered while performing a major upgrade across both Data Prepper and the Trace Analytics plugin. + +#### Approach #1: Upgrade both Data Prepper and the Trace Analytics plugin as simultaneously as possible +The simplest approach to perform a major version upgrade is to simultaneously upgrade both Data Prepper and the Trace Analytics plugin. This approach will result in a usability drop while both components are on differing major versions. + +As an example, assume both DP and TA are being upgraded from v1 to v2: +* If DP is upgraded to v2 before TA, then the old TA v1 dashboard will not visualize the new data ingested by DP v2. +* IF TA is upgraded to v2 before DP, then the new TA v2 dashboard will not visualize the old data ingested by DP v1. + +Both component upgrades should happen as close as possible to each other to minimize this window. + +#### Approach #2: Run both old and new Data Prepper versions side-by-side until the Trace Analytics plugin is upgraded +To avoid usability downtime of the Trace Analytics plugin, run two versions of Data Prepper simultaneously before upgrading the plugin. + +* DP v1 writes to the v1 index +* DP v2 writes to the v2 index +* TA v1 reads from the v1 index +* TA v2 reads from the v2 index + +Upgrading the Trace Analytics plugin from v1 to v2 will result in no usability downtime, as both plugin versions will have populated indexes to read from. This approach is more complex in that it requires additional Data Prepper instances and results in duplicate data being written (until the old DP instances are shut down). + +#### Mitigating data loss +Upgrading Data Prepper and the Trace Analytics plugin to a new major version schema will result in data written to old indexes being unusable. If users wish to avoid this data loss, the [reindexing APIs](https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/reindex-data/) must be used. Additionally, transforms might required depending on the differences between the two major schema versions. diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle index 1eab5f5207..66862b10ad 100644 --- a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle +++ b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle @@ -27,9 +27,9 @@ repositories { dependencies { implementation('org.springframework.boot:spring-boot-starter-web') testImplementation('org.springframework.boot:spring-boot-starter-test') - compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.8.1' - compile 'org.elasticsearch.client:elasticsearch-rest-client:7.8.1' - compile 'org.elasticsearch:elasticsearch:7.8.1' + implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.8.1' + implementation 'org.elasticsearch.client:elasticsearch-rest-client:7.8.1' + implementation 'org.elasticsearch:elasticsearch:7.8.1' } bootJar { diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000..aef125e083 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +version=1.0.0 diff --git a/release/archives/build.gradle b/release/archives/build.gradle index a8c7e2ad3b..208c2c777b 100644 --- a/release/archives/build.gradle +++ b/release/archives/build.gradle @@ -97,7 +97,7 @@ subprojects { tasks.withType(Tar) { dependsOn ':release:releasePrerequisites' compression = Compression.GZIP - extension = 'tar.gz' + archiveExtension = 'tar.gz' } tasks.withType(Zip) { diff --git a/release/build.gradle b/release/build.gradle index 236f3812e6..d66f7db3f5 100644 --- a/release/build.gradle +++ b/release/build.gradle @@ -13,7 +13,7 @@ apply from: file("build-resources.gradle") allprojects { dependencies { - compile project(':data-prepper-core') + implementation project(':data-prepper-core') } } diff --git a/release/docker/opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile b/release/docker/Dockerfile similarity index 100% rename from release/docker/opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile rename to release/docker/Dockerfile diff --git a/release/docker/README.md b/release/docker/README.md index 24a52f3cae..1acd44ba39 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -31,5 +31,5 @@ docker run \ --expose 21890 \ -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-pipelines.yaml:/usr/share/data-prepper/pipelines.yaml \ -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml \ - data-prepper/data-prepper:0.8.0-beta + data-prepper/data-prepper:1.0.0 ``` \ No newline at end of file diff --git a/release/docker/build.gradle b/release/docker/build.gradle index c2646500c8..a84c9c5782 100644 --- a/release/docker/build.gradle +++ b/release/docker/build.gradle @@ -20,7 +20,7 @@ docker { buildArgs(['JAR_FILE' : project(':data-prepper-core').jar.archiveName, 'CONFIG_FILEPATH': '/usr/share/data-prepper/data-prepper-config.yaml', 'PIPELINE_FILEPATH': '/usr/share/data-prepper/pipelines.yaml']) - dockerfile file('opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile') + dockerfile file('Dockerfile') } dockerPrepare.dependsOn ':release:releasePrerequisites' diff --git a/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile b/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile deleted file mode 100644 index 0e38211b1b..0000000000 --- a/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM amazoncorretto:15-al2-full -ARG CONFIG_FILEPATH -ARG JAR_FILE -ENV ENV_CONFIG_FILEPATH=$CONFIG_FILEPATH -ENV DATA_PREPPER_PATH /usr/share/data-prepper -RUN mkdir -p $DATA_PREPPER_PATH -COPY $JAR_FILE /usr/share/data-prepper/data-prepper.jar -WORKDIR $DATA_PREPPER_PATH -CMD java $JAVA_OPTS -jar data-prepper.jar ${ENV_CONFIG_FILEPATH} diff --git a/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md b/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md new file mode 100644 index 0000000000..550c23b7ed --- /dev/null +++ b/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md @@ -0,0 +1,8 @@ +# 2021-05-11 Version 1.0.0 + +## Highlights +* Now builds using [version 1.0+](https://github.com/open-telemetry/opentelemetry-specification/pull/1372) of the OpenTelemetry tracing specification +* Additional TraceGroup fields are emitted for enhanced searching and filtering + +## Compatibility +* No compatibility issues with previous versions of Data Prepper. diff --git a/research/zipkin-elastic-to-otel/README.md b/research/zipkin-elastic-to-otel/README.md index 506c9efbad..bff1e04c41 100644 --- a/research/zipkin-elastic-to-otel/README.md +++ b/research/zipkin-elastic-to-otel/README.md @@ -19,4 +19,4 @@ Notice that `FIELD` and `VALUE` are optional arguments. ``` ./gradlew :research:zipkin-elastic-to-otel:run -Dtest=true --args YOUR_INDEX_PATTERN -``` \ No newline at end of file +``` diff --git a/research/zipkin-elastic-to-otel/build.gradle b/research/zipkin-elastic-to-otel/build.gradle index 948f379b1e..0488ec2e0b 100644 --- a/research/zipkin-elastic-to-otel/build.gradle +++ b/research/zipkin-elastic-to-otel/build.gradle @@ -40,9 +40,10 @@ run { } dependencies { - compile project(':data-prepper-plugins:opensearch') - compile project(':data-prepper-plugins:otel-trace-source') - compile project(':data-prepper-plugins:otel-trace-raw-prepper') + implementation project(':data-prepper-plugins:blocking-buffer') + implementation project(':data-prepper-plugins:opensearch') + implementation project(':data-prepper-plugins:otel-trace-source') + implementation project(':data-prepper-plugins:otel-trace-raw-prepper') implementation "org.apache.commons:commons-lang3:3.11" implementation "com.linecorp.armeria:armeria:1.0.0" implementation "com.linecorp.armeria:armeria-grpc:1.0.0"