From f87adf4000d96148b6f07c5045f403fc0cfac0fc Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:49:21 +0000 Subject: [PATCH 001/192] Updating THIRD-PARTY attributions. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- THIRD-PARTY | 62 +++++++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/THIRD-PARTY b/THIRD-PARTY index 6a988b22fa..ab4e9d5984 100644 --- a/THIRD-PARTY +++ b/THIRD-PARTY @@ -2,10 +2,9 @@ ** Amazon ION Java; version 1.0.2 -- https://github.com/amzn/ion-java ** android annotations; version 4.1.1.4 -- https://github.com/androidannotations/androidannotations -** Apache Lucene; version 8.5.1 -- http://lucene.apache.org/ -** armeria; version 1.5.0 -- https://github.com/line/armeria -** armeria-grpc; version 1.5.0 -- https://github.com/line/armeria -** aws-java-sdk-core; version 1.11.965 -- https://aws.amazon.com/sdk-for-java/ +** Apache Lucene; version 8.7.0 -- http://lucene.apache.org/ +** armeria-grpc; version 1.6.0 -- https://github.com/line/armeria +** aws-java-sdk-core; version 1.11.1001 -- https://aws.amazon.com/sdk-for-java/ ** aws-request-signing-apache-interceptor; version b3772780da -- https://github.com/awslabs/aws-request-signing-apache-interceptor ** bval; version 2.0.4 -- https://github.com/apache/bval @@ -13,46 +12,45 @@ https://github.com/awslabs/aws-request-signing-apache-interceptor ** commons-codec; version 1.15 -- https://github.com/apache/commons-codec ** commons-logging; version 1.2 -- https://github.com/apache/commons-logging/tree/LOGGING_1_1_3 -** Elasticsearch; version 7.8.0 -- https://github.com/elastic/elasticsearch -** Elasticsearch Java High Level Rest Client; version 7.8.1 -- +** Elasticsearch; version 7.10.2 -- https://github.com/elastic/elasticsearch +** Elasticsearch Java High Level Rest Client; version 7.10.2 -- https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high.html ** Error Prone; version 2.3.4 -- https://github.com/google/error-prone ** Findbugs Jsr305; version 3.0.2 -- https://search.maven.org/artifact/com.google.code.findbugs/jsr305/3.0.2/jar ** Google Gson; version 2.8.6 -- https://github.com/google/gson -** gRPC; version 1.35.0 -- https://github.com/grpc/grpc -** guava; version 29.0 -- https://github.com/google/guava +** gRPC; version 1.36.1 -- https://github.com/grpc/grpc +** guava; version 30.1.1-jre -- https://github.com/google/guava ** Hppc; version 0.8.1 -- https://github.com/carrotsearch/hppc/blob/master/LICENSE.txt ** httpcomponents-core; version 4.4.12 -- https://github.com/apache/httpcomponents-core/blob/rel/v4.4.1 ** J2ObjC; version 1.3 -- https://developers.google.com/j2objc/ -** Jackson; version 2.12.1 -- http://github.com/FasterXML/jackson +** Jackson; version 2.12.3 -- http://github.com/FasterXML/jackson ** JavaAssist; version 3.26.0-GA -- https://github.com/jboss-javassist/javassist ** JavaxValidationApi; version 2.0.1.Final -- http://beanvalidation.org ** jetbrains-annotations; version 13.0 -- https://github.com/JetBrains/java-annotations/blob/master/LICENSE.txt ** jffi; version 1.2.23 -- https://github.com/jnr/jffi/blob/jffi-1.2.23 -** jna; version 4.5.1 -- https://github.com/java-native-access/jna +** jna; version 5.5.0 -- https://github.com/java-native-access/jna ** jnr a64asm; version 1.0.0 -- https://github.com/jnr/jnr-a64asm/tree/jnr-a64asm-1.0.0 ** jnr constants; version 0.9.15 -- https://github.com/jnr/jnr-constants/tree/jnr-constants-0.9.15 ** joda-time; version 2.10.4 -- http://www.joda.org/joda-time/ -** Kotlin; version 1.2.71 -- https://github.com/JetBrains/kotlin -** lmdbjava; version 0.8.1 -- https://github.com/lmdbjava/lmdbjava +** Kotlin; version 1.4.32 -- https://github.com/JetBrains/kotlin ** log4j-slf4j-impl; version 2.14.0 -- https://logging.apache.org/log4j/2.x/ ** lz4-java; version 1.3.0 -- https://github.com/lz4/lz4-java/tree/1.3.0 ** MapDB; version 3.0.8 -- http://www.mapdb.org/ -** micrometer; version 1.5.5 -- +** micrometer; version 1.6.6 -- https://github.com/micrometer-metrics/micrometer -** micrometer-registry-prometheus; version 1.5.5 -- +** micrometer-registry-prometheus; version 1.6.5 -- https://github.com/micrometer-metrics/micrometer ** MustacheJava; version 0.9.6 -- https://github.com/spullara/mustache.java -** netty 4.1.51; version 4.1.58.Final -- +** netty 4.1.51; version 4.1.63.Final -- https://github.com/netty/netty/tree/netty-4.1.50.Final -** OpenTelemetry 0.10.0; version 0.10.0 -- +** OpenTelemetry 0.10.0; version 1.0.1-alpha -- https://github.com/open-telemetry/opentelemetry-java ** perfmark-api; version 0.23.0 -- https://github.com/perfmark/perfmark ** SnakeYAML; version 1.28 -- http://www.snakeyaml.org @@ -306,13 +304,6 @@ limitations under the License. ICU4J, (under contrib/collation) is licensed under an MIT styles license (contrib/collation/lib/ICU-LICENSE.txt) and Copyright (c) 1995-2008 International Business Machines Corporation and others -* For armeria see also this required NOTICE: - Armeria - ======= - - Please visit the official web site for more information: - - * https://armeria.dev/ * For armeria-grpc see also this required NOTICE: Armeria ======= @@ -540,26 +531,6 @@ limitations under the License. limitations under the License. * For Kotlin see also this required NOTICE: Copyright 2010-2020 JetBrains s.r.o and respective authors and developers -* For lmdbjava see also this required NOTICE: - /*- - * #%L - * LmdbJava - * %% - * Copyright (C) 2016 - 2020 The LmdbJava Open Source Project - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ * For log4j-slf4j-impl see also this required NOTICE: Apache Log4j SLF4J Binding Copyright 1999-2020 The Apache Software Foundation @@ -830,7 +801,7 @@ THE POSSIBILITY OF SUCH DAMAGE. ------ -** google-protobuf; version 3.15.3 -- +** google-protobuf; version 3.15.8 -- https://github.com/protocolbuffers/protobuf Copyright 2008 Google Inc. All rights reserved. @@ -882,7 +853,7 @@ work. If not, see . ------ -** eclipse-collections; version 11.0.0.M1 -- +** eclipse-collections; version 11.0.0.M2 -- https://github.com/eclipse/eclipse-collections Copyright (c) 2016 Shotaro Sano. @@ -892,6 +863,7 @@ Copyright (c) 2017 Goldman Sachs and others. Copyright (c) 2018 Goldman Sachs and others. Copyright (c) 2019 Goldman Sachs and others. Copyright (c) 2020 Goldman Sachs and others. +Copyright (c) 2021 Goldman Sachs and others. Eclipse Public License - v 1.0 From 5e32109c1564360c049403b23b4ec4c3a7832402 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 27 Apr 2021 11:18:43 -0500 Subject: [PATCH 002/192] FIX: ISM API breaking change (#546) * FIX: ISM API breaking change Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .github/workflows/gradle.yml | 11 --- ...k-odfe-before-1_13_0-integration-tests.yml | 34 ++++++++ ...nk-odfe-since-1_13_0-integration-tests.yml | 34 ++++++++ .../elasticsearch/build.gradle | 1 + .../sink/elasticsearch/ElasticsearchSink.java | 10 ++- .../sink/elasticsearch/IndexConstants.java | 3 +- .../elasticsearch/IndexStateManagement.java | 81 ++++++++++++------- ...n => raw-span-policy-no-ism-template.json} | 0 .../raw-span-policy-with-ism-template.json | 22 +++++ .../elasticsearch/ElasticsearchSinkIT.java | 6 +- 10 files changed, 155 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml create mode 100644 .github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml rename data-prepper-plugins/elasticsearch/src/main/resources/{raw-span-policy.json => raw-span-policy-no-ism-template.json} (100%) create mode 100644 data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index f647c9002c..baab044d85 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -23,14 +23,3 @@ jobs: run: chmod +x gradlew - name: Build with Gradle run: ./gradlew build - - name: Run ODFE docker - run: | - export version=1.9.0 - docker pull amazon/opendistro-for-elasticsearch:$version - docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version - sleep 90 - - name: Run ODFE tests - run: | - ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin - ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin - diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml new file mode 100644 index 0000000000..d046b7908d --- /dev/null +++ b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml @@ -0,0 +1,34 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0 + +on: [push, pull_request, workflow_dispatch] + +jobs: + integration_tests: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Checkout Data-Prepper + uses: actions/checkout@v2 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run ODFE docker + run: | + export version=1.12.0 + docker pull amazon/opendistro-for-elasticsearch:$version + docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version + sleep 90 + - name: Run ODFE tests + run: | + ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin + ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin \ No newline at end of file diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml new file mode 100644 index 0000000000..77dfa83ab5 --- /dev/null +++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml @@ -0,0 +1,34 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0 + +on: [push, pull_request, workflow_dispatch] + +jobs: + integration_tests: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Checkout Data-Prepper + uses: actions/checkout@v2 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run ODFE docker + run: | + export version=1.13.2 + docker pull amazon/opendistro-for-elasticsearch:$version + docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version + sleep 90 + - name: Run ODFE tests + run: | + ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin + ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin \ No newline at end of file diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index c7cc16146d..525c5f29cc 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -41,6 +41,7 @@ dependencies { testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } + testImplementation "org.awaitility:awaitility:4.0.3" testImplementation "org.elasticsearch.test:framework:${es_version}" testImplementation "commons-io:commons-io:2.8.0" } diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java index 3f934b6767..2c24ae41a2 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java @@ -35,6 +35,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; @DataPrepperPlugin(name = "elasticsearch", type = PluginType.SINK) @@ -77,9 +78,10 @@ public ElasticsearchSink(final PluginSetting pluginSetting) { public void start() throws IOException { LOG.info("Starting Elasticsearch sink"); restHighLevelClient = esSinkConfig.getConnectionConfiguration().createClient(); - final String ismPolicyId = IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType); + final boolean isISMEnabled = IndexStateManagement.checkISMEnabled(restHighLevelClient); + final Optional policyIdOptional = isISMEnabled? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : Optional.empty(); if (!esSinkConfig.getIndexConfiguration().getIndexTemplate().isEmpty()) { - createIndexTemplate(ismPolicyId); + createIndexTemplate(isISMEnabled, policyIdOptional.orElse(null)); } final String dlqFile = esSinkConfig.getRetryConfiguration().getDlqFile(); if ( dlqFile != null) { @@ -144,7 +146,7 @@ private void flushBatch(final BulkRequest bulkRequest) { }); } - private void createIndexTemplate(final String ismPolicyId) throws IOException { + private void createIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias(); final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexAlias + "-index-template"); final boolean isRaw = indexType.equals(IndexConstants.RAW); @@ -153,7 +155,7 @@ private void createIndexTemplate(final String ismPolicyId) throws IOException { } else { putIndexTemplateRequest.patterns(Collections.singletonList(indexAlias)); } - if (ismPolicyId != null) { + if (isISMEnabled) { IndexStateManagement.attachPolicy(esSinkConfig.getIndexConfiguration(), ismPolicyId, indexAlias); } putIndexTemplateRequest.source(esSinkConfig.getIndexConfiguration().getIndexTemplate()); diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java index 083bd07468..ac2c903343 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java @@ -14,7 +14,8 @@ public class IndexConstants { // TODO: extract out version number into version enum public static final String RAW_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-span-index-template.json"; public static final String RAW_ISM_POLICY = "raw-span-policy"; - public static final String RAW_ISM_FILE = "raw-span-policy.json"; + public static final String RAW_ISM_FILE_NO_ISM_TEMPLATE = "raw-span-policy-no-ism-template.json"; + public static final String RAW_ISM_FILE_WITH_ISM_TEMPLATE = "raw-span-policy-with-ism-template.json"; public static final String ISM_ENABLED_SETTING = "opendistro.index_state_management.enabled"; public static final String ISM_POLICY_ID_SETTING = "opendistro.index_state_management.policy_id"; public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias"; diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java index 096e7a0c79..53382de43f 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java @@ -1,13 +1,12 @@ package com.amazon.dataprepper.plugins.sink.elasticsearch; +import com.google.common.base.Preconditions; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.rest.RestStatus; import javax.ws.rs.HttpMethod; import java.io.BufferedReader; @@ -16,6 +15,8 @@ import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; public class IndexStateManagement { public static boolean checkISMEnabled(final RestHighLevelClient restHighLevelClient) throws IOException { @@ -26,46 +27,66 @@ public static boolean checkISMEnabled(final RestHighLevelClient restHighLevelCli return enabled != null && enabled.equals("true"); } - public static String checkAndCreatePolicy( + /** + * @return ISM policy_id optional that needs to be attached to index settings. + */ + public static Optional checkAndCreatePolicy( final RestHighLevelClient restHighLevelClient, final String indexType) throws IOException { - if (checkISMEnabled(restHighLevelClient) && indexType.equals(IndexConstants.RAW)) { + if (indexType.equals(IndexConstants.RAW)) { final String endPoint = "/_opendistro/_ism/policies/" + IndexConstants.RAW_ISM_POLICY; - Request request = new Request(HttpMethod.HEAD, endPoint); - final Response response = restHighLevelClient.getLowLevelClient().performRequest(request); - if (response.getStatusLine().getStatusCode() != RestStatus.OK.getStatus()) { - final InputStream is = IndexStateManagement.class.getClassLoader().getResourceAsStream(IndexConstants.RAW_ISM_FILE); - assert is != null; - final StringBuilder policyJsonBuffer = new StringBuilder(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { - reader.lines().forEach(line -> policyJsonBuffer.append(line).append("\n")); - } - is.close(); - request = new Request(HttpMethod.PUT, endPoint); - request.setJsonEntity(policyJsonBuffer.toString()); - try { - restHighLevelClient.getLowLevelClient().performRequest(request); - } catch (ResponseException e) { - if (e.getMessage().contains("version_conflict_engine_exception") - || e.getMessage().contains("resource_already_exists_exception")) { - // Do nothing - likely caused by a race condition where the resource was created - // by another host before this host's restClient made its request - } else { - throw e; + Request request = createPolicyRequestFromFile(endPoint, IndexConstants.RAW_ISM_FILE_WITH_ISM_TEMPLATE); + try { + restHighLevelClient.getLowLevelClient().performRequest(request); + } catch (ResponseException e1) { + final String msg = e1.getMessage(); + if (msg.contains("Invalid field: [ism_template]")) { + request = createPolicyRequestFromFile(endPoint, IndexConstants.RAW_ISM_FILE_NO_ISM_TEMPLATE); + try { + restHighLevelClient.getLowLevelClient().performRequest(request); + } catch (ResponseException e2) { + if (e2.getMessage().contains("version_conflict_engine_exception") + || e2.getMessage().contains("resource_already_exists_exception")) { + // Do nothing - likely caused by + // (1) a race condition where the resource was created by another host before this host's + // restClient made its request; + // (2) policy already exists in the cluster + } else { + throw e2; + } } + return Optional.of(IndexConstants.RAW_ISM_POLICY); + } else if (e1.getMessage().contains("version_conflict_engine_exception") + || e1.getMessage().contains("resource_already_exists_exception")) { + // Do nothing - likely caused by + // (1) a race condition where the resource was created by another host before this host's + // restClient made its request; + // (2) policy already exists in the cluster + } else { + throw e1; } } - return IndexConstants.RAW_ISM_POLICY; - } else { - return null; } + return Optional.empty(); } @SuppressWarnings("unchecked") public static void attachPolicy( final IndexConfiguration configuration, final String ismPolicyId, final String rolloverAlias) { configuration.getIndexTemplate().putIfAbsent("settings", new HashMap<>()); - // Attach policy_id and rollover_alias - ((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId); + if (ismPolicyId != null) { + ((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId); + } ((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_ROLLOVER_ALIAS_SETTING, rolloverAlias); } + + private static Request createPolicyRequestFromFile(final String endPoint, final String fileName) throws IOException { + final StringBuilder policyJsonBuffer = new StringBuilder(); + try (final InputStream inputStream = IndexStateManagement.class.getClassLoader().getResourceAsStream(fileName); + final BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(inputStream)))) { + reader.lines().forEach(line -> policyJsonBuffer.append(line).append("\n")); + } + final Request request = new Request(HttpMethod.PUT, endPoint); + request.setJsonEntity(policyJsonBuffer.toString()); + return request; + } } diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy.json b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-no-ism-template.json similarity index 100% rename from data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy.json rename to data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-no-ism-template.json diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json new file mode 100644 index 0000000000..f70ce3f7b8 --- /dev/null +++ b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json @@ -0,0 +1,22 @@ +{ + "policy": { + "description": "Managing raw spans for trace analytics", + "default_state": "current_write_index", + "states": [ + { + "name": "current_write_index", + "actions": [ + { + "rollover": { + "min_size": "50gb", + "min_index_age": "24h" + } + } + ] + } + ], + "ism_template": { + "index_patterns": ["otel-v1-apm-span-*"] + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java index 0e391745df..38056c0fdd 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java @@ -51,9 +51,11 @@ import java.util.Objects; import java.util.Optional; import java.util.StringJoiner; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.http.HttpStatus.SC_OK; +import static org.awaitility.Awaitility.await; public class ElasticsearchSinkIT extends ESRestTestCase { private static final String PLUGIN_NAME = "elasticsearch"; @@ -85,7 +87,9 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { if (isODFE()) { // Check managed index - assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index)); + await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index)); } + ); } // roll over initial index From afd2acae35e32d93a56c68e92b85f4fbe704c6f5 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 29 Apr 2021 17:39:46 -0500 Subject: [PATCH 003/192] MAINT: update cfn template (#548) * MAINT: update cfn template Signed-off-by: qchea --- .../ec2/data-prepper-ec2-deployment-cfn.yaml | 121 ++++++++++++------ 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index 58bee69ad8..b69c750493 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -16,6 +16,14 @@ Parameters: AWSElasticsearchSubnetId: Description: The subnet ID of the AWS Elasticsearch Service domain (Leave blank if the domain is not in a VPC) Type: String + Username: + Description: The username of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role) + Type: String + Default: "" + Password: + Description: The password of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role) + Type: String + Default: "" DataPrepperVersion: Description: Version of Data Prepper to download and run Type: String @@ -24,8 +32,9 @@ Parameters: ConstraintDescription: must be a valid release number IAMRole: Description: Pre-existing IAM Role to associate with the EC2 instance, to be used for authentication when calling Elasticsearch + (Leave blank if the domain is configured with HTTP basic authentication in Fine-grained access control) Type: String - AllowedPattern: "[a-zA-Z0-9+=,\\.@\\-_]+" + AllowedPattern: "^$|[a-zA-Z0-9+=,\\.@\\-_]+" Default: DataPrepperRole ConstraintDescription: must be a valid IAM role name InstanceType: @@ -50,9 +59,39 @@ Parameters: Default: 0.0.0.0/0 AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x. +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Common + Parameters: + - IAMRole + - Label: + default: Amazon Elasticsearch Service Domain + Parameters: + - AWSElasticsearchEndpoint + - AWSElasticsearchRegion + - AWSElasticsearchSubnetId + - Username + - Password + - Label: + default: Data-Prepper Configuration + Parameters: + - DataPrepperVersion + - InstanceType + - KeyName + - LatestAmi + - SSHLocation Conditions: DomainIsPublic: !Equals [!Ref AWSElasticsearchSubnetId, ""] DomainIsInVPC: !Not [Condition: DomainIsPublic] + NoMasterUser: !And + - !Equals + - !Ref Username + - "" + - !Equals + - !Ref Password + - "" Resources: EC2Instance: Type: AWS::EC2::Instance @@ -70,43 +109,49 @@ Resources: owner: root group: root "/etc/data-prepper/pipelines.yaml": - content: !Sub | - entry-pipeline: - delay: "100" - source: - otel_trace_source: - ssl: false - health_check_service: true - sink: - - pipeline: - name: "raw-pipeline" - - pipeline: - name: "service-map-pipeline" - raw-pipeline: - source: - pipeline: - name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: - sink: - - elasticsearch: - hosts: [ "${AWSElasticsearchEndpoint}" ] - aws_sigv4: true - aws_region: "${AWSElasticsearchRegion}" - trace_analytics_raw: true - service-map-pipeline: - delay: "100" - source: - pipeline: - name: "entry-pipeline" - prepper: - - service_map_stateful: - sink: - - elasticsearch: - hosts: [ "${AWSElasticsearchEndpoint}" ] - aws_sigv4: true - aws_region: "${AWSElasticsearchRegion}" - trace_analytics_service_map: true + content: !Sub + - | + entry-pipeline: + delay: "100" + source: + otel_trace_source: + ssl: false + health_check_service: true + sink: + - pipeline: + name: "raw-pipeline" + - pipeline: + name: "service-map-pipeline" + raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - otel_trace_raw_prepper: + sink: + - elasticsearch: ${elasticsearchConfig} + service-map-pipeline: + delay: "100" + source: + pipeline: + name: "entry-pipeline" + prepper: + - service_map_stateful: + sink: + - elasticsearch: ${elasticsearchConfig} + - elasticsearchConfig: !If + - NoMasterUser + - !Sub "\n + \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n + \ aws_sigv4: true\n + \ aws_region: \"${AWSElasticsearchRegion}\"\n + \ trace_analytics_service_map: true" + - !Sub "\n + \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n + \ aws_sigv4: false\n + \ username: \"${Username}\"\n + \ password: \"${Password}\"\n + \ trace_analytics_service_map: true" mode: "000400" owner: root group: root From f35ea6d0e34efe59c1183e6a1dd98c137190383a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 07:30:57 +0000 Subject: [PATCH 004/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-source Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 94b23a97b7..e8fc2b3afb 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.15.8' implementation "com.linecorp.armeria:armeria:1.6.0" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" + implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.19.0' From e3c43ac8fa99ae42bdda4f844cde92bf062792e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 07:32:53 +0000 Subject: [PATCH 005/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 6f883dfc7f..3bb286d074 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.15.8' implementation "com.linecorp.armeria:armeria:1.6.0" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" + implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' From 2c657a334301e0dd7155f9b96d154b966588596b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 07:33:58 +0000 Subject: [PATCH 006/192] Bump armeria from 1.6.0 to 1.7.2 in /data-prepper-plugins/peer-forwarder Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index a8143de01c..df36b8e267 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -13,7 +13,7 @@ dependencies { compile project(':data-prepper-api') testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "com.linecorp.armeria:armeria:1.6.0" + implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.6.0" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" From 2048421cd844f55e910858caa80b1cb941137a4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 07:34:03 +0000 Subject: [PATCH 007/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1001 to 1.11.1009. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1001...1.11.1009) Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 525c5f29cc..d6e5e0faed 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1001" + implementation "com.amazonaws:aws-java-sdk-core:1.11.1009" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.6.5" testImplementation("junit:junit:4.13.2") { @@ -52,7 +52,7 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.11.1000' + force 'com.amazonaws:aws-java-sdk-core:1.11.1009' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' From 10ce391eb66bb26506e5fa9510cdd865a9b36827 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 18:58:19 +0000 Subject: [PATCH 008/192] Bump armeria in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 3bb286d074..801e15abcf 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -9,7 +9,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.15.8' - implementation "com.linecorp.armeria:armeria:1.6.0" + implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From ecb8a176cf7399bfdce145b67cdd80f19ae0fc42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 18:58:22 +0000 Subject: [PATCH 009/192] Bump armeria-grpc in /data-prepper-plugins/peer-forwarder Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index df36b8e267..59c78f335b 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -14,7 +14,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.7.2" - implementation "com.linecorp.armeria:armeria-grpc:1.6.0" + implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" testImplementation "org.mockito:mockito-inline:3.9.0" From 51e4362323cd25a4ea51eb9e35f34ece93b84189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 18:58:40 +0000 Subject: [PATCH 010/192] Bump armeria in /data-prepper-plugins/otel-trace-source Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index e8fc2b3afb..9e0e645cbe 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.15.8' - implementation "com.linecorp.armeria:armeria:1.6.0" + implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From 701c2ea0f3ea48fb2438d32273dd557bb9a6923c Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 4 May 2021 14:01:04 -0500 Subject: [PATCH 011/192] Fixed minor typo in github integ test name --- .../opensearch-sink-odfe-since-1_13_0-integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml index 77dfa83ab5..eb08451867 100644 --- a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml +++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml @@ -1,7 +1,7 @@ # This workflow will build a Java project with Gradle # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle -name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0 +name: Data Prepper elasticsearch sink integration tests with ODFE >= 1.13.0 on: [push, pull_request, workflow_dispatch] @@ -31,4 +31,4 @@ jobs: - name: Run ODFE tests run: | ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin - ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin \ No newline at end of file + ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin From 02502f7cc0131f444a19cf46c42bec81b6b1a17f Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 4 May 2021 14:27:32 -0500 Subject: [PATCH 012/192] Updated workflows to potentially stop duplicate actions from firing. (#563) * Updated workflows to potentially stop duplicate actions from firing. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../workflows/data-prepper-trace-analytics-e2e-tests.yml | 1 - .github/workflows/gradle.yml | 6 +++++- ...opensearch-sink-odfe-before-1_13_0-integration-tests.yml | 6 +++++- .../opensearch-sink-odfe-since-1_13_0-integration-tests.yml | 6 +++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml index bdfab23884..ea39bdc594 100644 --- a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml @@ -7,7 +7,6 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] workflow_dispatch: jobs: diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index baab044d85..3dad348eb5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,7 +3,11 @@ name: Data Prepper Java CI with Gradle -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml index d046b7908d..5295214c2d 100644 --- a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml +++ b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml @@ -3,7 +3,11 @@ name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0 -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: integration_tests: diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml index eb08451867..623672eb2c 100644 --- a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml +++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml @@ -3,7 +3,11 @@ name: Data Prepper elasticsearch sink integration tests with ODFE >= 1.13.0 -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: integration_tests: From 78ad931102e92582edf4fe8f89e4b9296bf5aa2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 19:28:09 +0000 Subject: [PATCH 013/192] Bump kotlin-stdlib in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib](https://github.com/JetBrains/kotlin) from 1.4.32 to 1.5.0. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/commits) Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index 120c3553c2..b469f33676 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -16,7 +16,7 @@ dependencies { compile project(':data-prepper-api') compile project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.4.32' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.4.32' testCompile project(':data-prepper-plugins:common').sourceSets.test.output From 211c7a4f2290ed94c1c18cb89d9251adfee3b833 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 May 2021 19:30:23 +0000 Subject: [PATCH 014/192] Bump kotlin-stdlib-common in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib-common](https://github.com/JetBrains/kotlin) from 1.4.32 to 1.5.0. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/commits) Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index b469f33676..a2184fbad0 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -17,7 +17,7 @@ dependencies { compile project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.4.32' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.0' testCompile project(':data-prepper-plugins:common').sourceSets.test.output testImplementation "org.hamcrest:hamcrest:2.2" From ff834afe08826d8b63c424226725a9de1325a1b3 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 5 May 2021 14:31:22 -0500 Subject: [PATCH 015/192] MAINT: restore traceGroup backward compatibility (#564) * MAINT: restore traceGroup backward compatibility Signed-off-by: qchea --- .../main/resources/otel-v1-apm-span-index-template.json | 8 ++++---- .../plugins/prepper/oteltracegroup/model/TraceGroup.java | 8 ++++---- .../src/test/resources/raw-span-complete-1.json | 8 ++++---- .../src/test/resources/raw-span-complete-2.json | 8 ++++---- .../test/resources/raw-span-missing-trace-group-1.json | 8 ++++---- .../test/resources/raw-span-missing-trace-group-2.json | 8 ++++---- .../plugins/prepper/oteltrace/model/RawSpan.java | 2 +- .../plugins/prepper/oteltrace/model/TraceGroup.java | 4 ++++ .../prepper/oteltrace/OTelTraceRawPrepperTest.java | 8 ++++---- 9 files changed, 33 insertions(+), 29 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json index 252688d207..25df9ff8e7 100644 --- a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json +++ b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json @@ -40,11 +40,11 @@ "type": "keyword" }, "traceGroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroupFields": { "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, "endTime": { "type": "date_nanos" }, diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java index 1af2a9bb93..8695e19ee8 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java +++ b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java @@ -5,10 +5,10 @@ import java.util.Objects; public class TraceGroup { - public static final String TRACE_GROUP_NAME_FIELD = "traceGroup.name"; - public static final String TRACE_GROUP_END_TIME_FIELD = "traceGroup.endTime"; - public static final String TRACE_GROUP_STATUS_CODE_FIELD = "traceGroup.statusCode"; - public static final String TRACE_GROUP_DURATION_IN_NANOS_FIELD = "traceGroup.durationInNanos"; + public static final String TRACE_GROUP_NAME_FIELD = "traceGroup"; + public static final String TRACE_GROUP_END_TIME_FIELD = "traceGroupFields.endTime"; + public static final String TRACE_GROUP_STATUS_CODE_FIELD = "traceGroupFields.statusCode"; + public static final String TRACE_GROUP_DURATION_IN_NANOS_FIELD = "traceGroupFields.durationInNanos"; @JsonProperty(TRACE_GROUP_NAME_FIELD) private final String name; diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json index a03acb51ff..1226514180 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json +++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json @@ -14,10 +14,10 @@ "droppedAttributesCount": 0, "droppedEventsCount": 0, "droppedLinksCount": 0, - "traceGroup.name": "/logs", - "traceGroup.endTime": "2020-08-20T05:40:46.089556800Z", - "traceGroup.statusCode": 1, - "traceGroup.durationInNanos": 48545200, + "traceGroup": "/logs", + "traceGroupFields.endTime": "2020-08-20T05:40:46.089556800Z", + "traceGroupFields.statusCode": 1, + "traceGroupFields.durationInNanos": 48545200, "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", "span.attributes.http@method": "PUT", "resource.attributes.telemetry@sdk@name": "opentelemetry", diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json index a32d96e565..7c119322c4 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json +++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json @@ -14,10 +14,10 @@ "droppedAttributesCount": 0, "droppedEventsCount": 0, "droppedLinksCount": 0, - "traceGroup.name": "/logs", - "traceGroup.endTime": "2020-08-20T05:40:43.217170200Z", - "traceGroup.statusCode": 1, - "traceGroup.durationInNanos": 49160000, + "traceGroup": "/logs", + "traceGroupFields.endTime": "2020-08-20T05:40:43.217170200Z", + "traceGroupFields.statusCode": 1, + "traceGroupFields.durationInNanos": 49160000, "span.attributes.http@url": "http://0.0.0.0:8087/logs", "span.attributes.net@peer@ip": "172.29.0.1", "resource.attributes.telemetry@sdk@language": "java", diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json index 6fe5304b3b..239508132a 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json +++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json @@ -14,10 +14,10 @@ "droppedAttributesCount": 0, "droppedEventsCount": 0, "droppedLinksCount": 0, - "traceGroup.name": null, - "traceGroup.endTime": null, - "traceGroup.statusCode": null, - "traceGroup.durationInNanos": null, + "traceGroup": null, + "traceGroupFields.endTime": null, + "traceGroupFields.statusCode": null, + "traceGroupFields.durationInNanos": null, "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", "span.attributes.http@method": "PUT", "resource.attributes.telemetry@sdk@name": "opentelemetry", diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json index 4348d7340f..cb4ae8c110 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json +++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json @@ -14,10 +14,10 @@ "droppedAttributesCount": 0, "droppedEventsCount": 0, "droppedLinksCount": 0, - "traceGroup.name": null, - "traceGroup.endTime": null, - "traceGroup.statusCode": null, - "traceGroup.durationInNanos": null, + "traceGroup": null, + "traceGroupFields.endTime": null, + "traceGroupFields.statusCode": null, + "traceGroupFields.durationInNanos": null, "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", "span.attributes.http@method": "PUT", "resource.attributes.telemetry@sdk@name": "opentelemetry", diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java index 29d399c410..25d03dd19b 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java @@ -185,7 +185,7 @@ public Map getAttributes() { return attributes; } - @JsonUnwrapped(prefix="traceGroup.") + @JsonUnwrapped public TraceGroup getTraceGroup() { return traceGroup; } diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java index 3059a676e8..d183c51fe5 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java @@ -6,9 +6,13 @@ import java.util.Objects; public class TraceGroup { + @JsonProperty("traceGroup") private final String name; + @JsonProperty("traceGroupFields.endTime") private final String endTime; + @JsonProperty("traceGroupFields.statusCode") private final Integer statusCode; + @JsonProperty("traceGroupFields.durationInNanos") private final Long durationInNanos; public String getName() { diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java index 9f85f59010..cf4b846e8e 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java @@ -236,10 +236,10 @@ private int getMissingTraceGroupFieldsSpanCount(List> records) th for (Record record: records) { final String spanJson = record.getData(); Map spanMap = OBJECT_MAPPER.readValue(spanJson, new TypeReference>() {}); - final String traceGroupName = (String) spanMap.get("traceGroup.name"); - final String traceGroupEndTime = (String) spanMap.get("traceGroup.endTime"); - final Number traceGroupDurationInNanos = (Number) spanMap.get("traceGroup.durationInNanos"); - final Number traceGroupStatusCode = (Number) spanMap.get("traceGroup.statusCode"); + final String traceGroupName = (String) spanMap.get("traceGroup"); + final String traceGroupEndTime = (String) spanMap.get("traceGroupFields.endTime"); + final Number traceGroupDurationInNanos = (Number) spanMap.get("traceGroupFields.durationInNanos"); + final Number traceGroupStatusCode = (Number) spanMap.get("traceGroupFields.statusCode"); if (Stream.of(traceGroupName, traceGroupEndTime, traceGroupDurationInNanos, traceGroupStatusCode).allMatch(Objects::isNull)) { count += 1; } From 9409e0058dc16add541b9a1f7e67c024c596471c Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Thu, 6 May 2021 16:24:32 +0000 Subject: [PATCH 016/192] Added 1.0.0 release notes. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../odfe-data-prepper.release-notes-1.0.0.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 release/release-notes/odfe-data-prepper.release-notes-1.0.0.md diff --git a/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md b/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md new file mode 100644 index 0000000000..550c23b7ed --- /dev/null +++ b/release/release-notes/odfe-data-prepper.release-notes-1.0.0.md @@ -0,0 +1,8 @@ +# 2021-05-11 Version 1.0.0 + +## Highlights +* Now builds using [version 1.0+](https://github.com/open-telemetry/opentelemetry-specification/pull/1372) of the OpenTelemetry tracing specification +* Additional TraceGroup fields are emitted for enhanced searching and filtering + +## Compatibility +* No compatibility issues with previous versions of Data Prepper. From 1d587bf1eab9b129ca2b5112feb3f6ef5c17b22a Mon Sep 17 00:00:00 2001 From: Bala Yadav <55510769+yadavcbala@users.noreply.github.com> Date: Fri, 7 May 2021 11:18:54 -0500 Subject: [PATCH 017/192] Add CloudWatchMeterRegistry (#566) * Add CloudWatchMeterRegistry * Revert code coverage to existing % * Fix logger class * Update CloudWatchMeterRegistryProvider for :: and parameter name --- data-prepper-core/build.gradle | 2 + .../CloudWatchMeterRegistryProvider.java | 71 +++++++++++++++++++ .../src/main/resources/cloudwatch.properties | 4 ++ .../CloudWatchMeterRegistryProviderTest.java | 57 +++++++++++++++ .../test/resources/cloudwatch_test.properties | 4 ++ 5 files changed, 138 insertions(+) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java create mode 100644 data-prepper-core/src/main/resources/cloudwatch.properties create mode 100644 data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java create mode 100644 data-prepper-core/src/test/resources/cloudwatch_test.properties diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 2ddfa3c412..7339af5305 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation "org.reflections:reflections:0.9.12" implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" + implementation "io.micrometer:micrometer-registry-cloudwatch2:1.6.6" + implementation "software.amazon.awssdk:cloudwatch:2.16.54" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.9.0" } diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java new file mode 100644 index 0000000000..9705369316 --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/CloudWatchMeterRegistryProvider.java @@ -0,0 +1,71 @@ +package com.amazon.dataprepper.pipeline.server; + +import io.micrometer.cloudwatch2.CloudWatchConfig; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import io.micrometer.core.instrument.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import static java.util.Objects.requireNonNull; + +/** + * Provides {@link CloudWatchMeterRegistry} that enables publishing metrics to AWS Cloudwatch. Registry + * uses the default aws credentials (i.e. credentials from .aws directory; + * refer https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html#credentials-file-format). + * {@link CloudWatchMeterRegistryProvider} also has a constructor with {@link CloudWatchAsyncClient} that will be used + * for communication with Cloudwatch. + */ +public class CloudWatchMeterRegistryProvider { + private static final String CLOUDWATCH_PROPERTIES = "cloudwatch.properties"; + private static final Logger LOG = LoggerFactory.getLogger(CloudWatchMeterRegistryProvider.class); + + private final CloudWatchMeterRegistry cloudWatchMeterRegistry; + + public CloudWatchMeterRegistryProvider() { + this(CLOUDWATCH_PROPERTIES, CloudWatchAsyncClient.create()); + } + + public CloudWatchMeterRegistryProvider( + final String cloudWatchPropertiesFilePath, + final CloudWatchAsyncClient cloudWatchAsyncClient) { + final CloudWatchConfig cloudWatchConfig = createCloudWatchConfig( + requireNonNull(cloudWatchPropertiesFilePath, "cloudWatchPropertiesFilePath must not be null")); + this.cloudWatchMeterRegistry = new CloudWatchMeterRegistry(cloudWatchConfig, Clock.SYSTEM, + requireNonNull(cloudWatchAsyncClient, "cloudWatchAsyncClient must not be null")); + } + + /** + * Returns the CloudWatchMeterRegistry created using the default aws credentials + */ + public CloudWatchMeterRegistry getCloudWatchMeterRegistry() { + return this.cloudWatchMeterRegistry; + } + + /** + * Returns CloudWatchConfig using the properties from {@link #CLOUDWATCH_PROPERTIES} + */ + private CloudWatchConfig createCloudWatchConfig(final String cloudWatchPropertiesFilePath) { + CloudWatchConfig cloudWatchConfig = null; + try (final InputStream inputStream = requireNonNull(getClass().getClassLoader() + .getResourceAsStream(cloudWatchPropertiesFilePath))) { + final Properties cloudwatchProperties = new Properties(); + cloudwatchProperties.load(inputStream); + cloudWatchConfig = new CloudWatchConfig() { + @Override + public String get(final String key) { + return cloudwatchProperties.getProperty(key); + } + }; + } catch (IOException ex) { + LOG.error("Encountered exception in creating CloudWatchConfig for CloudWatchMeterRegistry, " + + "Proceeding without metrics", ex); + //If there is no registry attached, micrometer will make NoopMeters which are discarded. + } + return cloudWatchConfig; + } +} diff --git a/data-prepper-core/src/main/resources/cloudwatch.properties b/data-prepper-core/src/main/resources/cloudwatch.properties new file mode 100644 index 0000000000..c916d81f77 --- /dev/null +++ b/data-prepper-core/src/main/resources/cloudwatch.properties @@ -0,0 +1,4 @@ +cloudwatch.enabled=true +cloudwatch.namespace=dataprepper +cloudwatch.batchSize=20 +cloudwatch.step=PT1M \ No newline at end of file diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java new file mode 100644 index 0000000000..24f8c0bab0 --- /dev/null +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java @@ -0,0 +1,57 @@ +package com.amazon.dataprepper.server; + +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CloudWatchMeterRegistryProviderTest { + private static final String TEST_CLOUDWATCH_PROPERTIES = "cloudwatch_test.properties"; + + @Mock + CloudWatchAsyncClient cloudWatchAsyncClient; + + @Mock + CompletableFuture putMetricDataResponseCompletableFuture; + + + @Before + public void setup() throws Exception { + when(cloudWatchAsyncClient.putMetricData(any(PutMetricDataRequest.class))).thenReturn(putMetricDataResponseCompletableFuture); + doAnswer(ans -> null).when(putMetricDataResponseCompletableFuture).whenCompleteAsync(any()); + } + + @Test(expected = NullPointerException.class) + public void testCreateWithInvalidPropertiesFile() { + new CloudWatchMeterRegistryProvider("does not exist", cloudWatchAsyncClient); + } + + @Test + public void testCreateCloudWatchMeterRegistry() { + final CloudWatchMeterRegistryProvider cloudWatchMeterRegistryProvider = new CloudWatchMeterRegistryProvider( + TEST_CLOUDWATCH_PROPERTIES, cloudWatchAsyncClient); + final CloudWatchMeterRegistry cloudWatchMeterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); + assertThat(cloudWatchMeterRegistry, notNullValue()); + cloudWatchMeterRegistry.gauge("test-gauge", 1d); + cloudWatchMeterRegistry.close(); //This will trigger publishing of metrics + verify(cloudWatchAsyncClient, atLeast(1)).putMetricData(any(PutMetricDataRequest.class)); + } + +} diff --git a/data-prepper-core/src/test/resources/cloudwatch_test.properties b/data-prepper-core/src/test/resources/cloudwatch_test.properties new file mode 100644 index 0000000000..9023c91de3 --- /dev/null +++ b/data-prepper-core/src/test/resources/cloudwatch_test.properties @@ -0,0 +1,4 @@ +cloudwatch.enabled=true +cloudwatch.namespace=dataprepper-test +cloudwatch.batchSize=20 +cloudwatch.step=PT1S \ No newline at end of file From 668cb47d82b42eb8f6e9c94e6c7ebe1c5520781e Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Fri, 7 May 2021 16:22:28 +0000 Subject: [PATCH 018/192] Consolidated dockerfiles in the release directory. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- release/docker/README.md | 2 +- release/docker/build.gradle | 2 +- ...oforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile | 9 --------- ...> opendistroforelasticsearch-data-prepper.Dockerfile} | 0 4 files changed, 2 insertions(+), 11 deletions(-) delete mode 100644 release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile rename release/docker/{opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile => opendistroforelasticsearch-data-prepper.Dockerfile} (100%) diff --git a/release/docker/README.md b/release/docker/README.md index 24a52f3cae..1acd44ba39 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -31,5 +31,5 @@ docker run \ --expose 21890 \ -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-pipelines.yaml:/usr/share/data-prepper/pipelines.yaml \ -v /workplace/github/simple-ingest-transformation-utility-pipeline/examples/config/example-data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml \ - data-prepper/data-prepper:0.8.0-beta + data-prepper/data-prepper:1.0.0 ``` \ No newline at end of file diff --git a/release/docker/build.gradle b/release/docker/build.gradle index 5e2c217f5c..ed29a38f37 100644 --- a/release/docker/build.gradle +++ b/release/docker/build.gradle @@ -9,7 +9,7 @@ docker { buildArgs(['JAR_FILE' : project(':data-prepper-core').jar.archiveName, 'CONFIG_FILEPATH': '/usr/share/data-prepper/data-prepper-config.yaml', 'PIPELINE_FILEPATH': '/usr/share/data-prepper/pipelines.yaml']) - dockerfile file('opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile') + dockerfile file('opendistroforelasticsearch-data-prepper.Dockerfile') } dockerPrepare.dependsOn ':release:releasePrerequisites' diff --git a/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile b/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile deleted file mode 100644 index 0e38211b1b..0000000000 --- a/release/docker/opendistroforelasticsearch-data-prepper-0.7.1-alpha.Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM amazoncorretto:15-al2-full -ARG CONFIG_FILEPATH -ARG JAR_FILE -ENV ENV_CONFIG_FILEPATH=$CONFIG_FILEPATH -ENV DATA_PREPPER_PATH /usr/share/data-prepper -RUN mkdir -p $DATA_PREPPER_PATH -COPY $JAR_FILE /usr/share/data-prepper/data-prepper.jar -WORKDIR $DATA_PREPPER_PATH -CMD java $JAVA_OPTS -jar data-prepper.jar ${ENV_CONFIG_FILEPATH} diff --git a/release/docker/opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile b/release/docker/opendistroforelasticsearch-data-prepper.Dockerfile similarity index 100% rename from release/docker/opendistroforelasticsearch-data-prepper-0.8.0-beta.Dockerfile rename to release/docker/opendistroforelasticsearch-data-prepper.Dockerfile From 6a1e287a7f67b2ed815297ad7ee8f6ff6cbb5e60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:05:36 +0000 Subject: [PATCH 019/192] Bump cloudwatch from 2.16.54 to 2.16.59 in /data-prepper-core Bumps cloudwatch from 2.16.54 to 2.16.59. Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 7339af5305..2f02d05aab 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.6.6" - implementation "software.amazon.awssdk:cloudwatch:2.16.54" + implementation "software.amazon.awssdk:cloudwatch:2.16.59" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.9.0" } From 59398dda0d641993896f8222cc82f3da216625ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:20:57 +0000 Subject: [PATCH 020/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-source Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.15.8 to 3.16.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.15.8...v3.16.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 9e0e645cbe..c63aa536b9 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,7 +8,7 @@ dependencies { compile 'commons-codec:commons-codec:1.15' testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.15.8' + implementation 'com.google.protobuf:protobuf-java-util:3.16.0' implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From 6693d6133c18a6c3b15777fe4b9667721649b7bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:21:01 +0000 Subject: [PATCH 021/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1009 to 1.11.1015. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1009...1.11.1015) Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index d6e5e0faed..fb1621d573 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1009" + implementation "com.amazonaws:aws-java-sdk-core:1.11.1015" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.6.5" testImplementation("junit:junit:4.13.2") { From 51ca0babc3fc83586a4dea528268691d618cc176 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:21:05 +0000 Subject: [PATCH 022/192] Bump awaitility in /data-prepper-plugins/elasticsearch Bumps [awaitility](https://github.com/awaitility/awaitility) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/awaitility/awaitility/releases) - [Changelog](https://github.com/awaitility/awaitility/blob/master/changelog.txt) - [Commits](https://github.com/awaitility/awaitility/commits/awaitility-4.1.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index d6e5e0faed..c845833166 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -41,7 +41,7 @@ dependencies { testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } - testImplementation "org.awaitility:awaitility:4.0.3" + testImplementation "org.awaitility:awaitility:4.1.0" testImplementation "org.elasticsearch.test:framework:${es_version}" testImplementation "commons-io:commons-io:2.8.0" } From 7b5aafde5a7cfd61451a5ff16f3bb568c77e6172 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:22:10 +0000 Subject: [PATCH 023/192] Bump awaitility in /data-prepper-plugins/otel-trace-raw-prepper Bumps [awaitility](https://github.com/awaitility/awaitility) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/awaitility/awaitility/releases) - [Changelog](https://github.com/awaitility/awaitility/blob/master/changelog.txt) - [Commits](https://github.com/awaitility/awaitility/commits/awaitility-4.1.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 801e15abcf..426539efc5 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -17,7 +17,7 @@ dependencies { testImplementation 'org.assertj:assertj-core:3.19.0' testImplementation "org.mockito:mockito-inline:3.9.0" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.awaitility:awaitility:4.0.3" + testImplementation "org.awaitility:awaitility:4.1.0" } jacocoTestCoverageVerification { From 330ba9b0a870736b3eb5b534e93f3845565bc874 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 May 2021 07:22:16 +0000 Subject: [PATCH 024/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-raw-prepper Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.15.8 to 3.16.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.15.8...v3.16.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 801e15abcf..36026311b8 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -8,7 +8,7 @@ dependencies { compile 'commons-codec:commons-codec:1.15' testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.15.8' + implementation 'com.google.protobuf:protobuf-java-util:3.16.0' implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From 11c488268249fc07cac8a3066054cc1223f910fc Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 11 May 2021 10:40:51 -0500 Subject: [PATCH 025/192] MAINT: merge v1.0.0 CFN (#583) Signed-off-by: qchea --- .../ec2/data-prepper-ec2-deployment-cfn.yaml | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index b69c750493..e764d957a6 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -2,18 +2,18 @@ AWSTemplateFormatVersion: "2010-09-09" Description: "Template to install Data Prepper on a single EC2 instance" Parameters: - AWSElasticsearchEndpoint: + AmazonEsEndpoint: Description: Endpoint of the AWS Elasticsearch Service domain (including https://) Type: String AllowedPattern: https:\/\/[a-z0-9-\.]+(es.amazonaws.com) ConstraintDescription: must be a valid Amazon Elasticsearch Service domain endpoint starting with https:// - AWSElasticsearchRegion: + AmazonEsRegion: Description: Region of the AWS Elasticsearch Service domain Type: String AllowedPattern: "[a-z]+-[a-z]+-[0-9]+" Default: us-east-1 ConstraintDescription: must be a valid AWS region (e.g. us-west-2) - AWSElasticsearchSubnetId: + AmazonEsSubnetId: Description: The subnet ID of the AWS Elasticsearch Service domain (Leave blank if the domain is not in a VPC) Type: String Username: @@ -69,9 +69,9 @@ Metadata: - Label: default: Amazon Elasticsearch Service Domain Parameters: - - AWSElasticsearchEndpoint - - AWSElasticsearchRegion - - AWSElasticsearchSubnetId + - AmazonEsEndpoint + - AmazonEsRegion + - AmazonEsSubnetId - Username - Password - Label: @@ -83,7 +83,7 @@ Metadata: - LatestAmi - SSHLocation Conditions: - DomainIsPublic: !Equals [!Ref AWSElasticsearchSubnetId, ""] + DomainIsPublic: !Equals [!Ref AmazonEsSubnetId, ""] DomainIsInVPC: !Not [Condition: DomainIsPublic] NoMasterUser: !And - !Equals @@ -129,7 +129,7 @@ Resources: prepper: - otel_trace_raw_prepper: sink: - - elasticsearch: ${elasticsearchConfig} + - elasticsearch: ${rawSpanConfig} service-map-pipeline: delay: "100" source: @@ -138,16 +138,29 @@ Resources: prepper: - service_map_stateful: sink: - - elasticsearch: ${elasticsearchConfig} - - elasticsearchConfig: !If + - elasticsearch: ${serviceMapConfig} + - rawSpanConfig: !If - NoMasterUser - !Sub "\n - \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n \ aws_sigv4: true\n - \ aws_region: \"${AWSElasticsearchRegion}\"\n + \ aws_region: \"${AmazonEsRegion}\"\n + \ trace_analytics_raw: true" + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: false\n + \ username: \"${Username}\"\n + \ password: \"${Password}\"\n + \ trace_analytics_raw: true" + serviceMapConfig: !If + - NoMasterUser + - !Sub "\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n + \ aws_sigv4: true\n + \ aws_region: \"${AmazonEsRegion}\"\n \ trace_analytics_service_map: true" - !Sub "\n - \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n + \ hosts: [ \"${AmazonEsEndpoint}\" ]\n \ aws_sigv4: false\n \ username: \"${Username}\"\n \ password: \"${Password}\"\n @@ -157,7 +170,7 @@ Resources: group: root Properties: SubnetId: - !If [DomainIsInVPC, !Ref AWSElasticsearchSubnetId, !Ref "AWS::NoValue"] + !If [DomainIsInVPC, !Ref AmazonEsSubnetId, !Ref "AWS::NoValue"] InstanceType: Ref: InstanceType IamInstanceProfile: @@ -193,4 +206,4 @@ Resources: FromPort: "22" ToPort: "22" CidrIp: - Ref: SSHLocation + Ref: SSHLocation \ No newline at end of file From a4be797123a661ef5a4dbc2cc56c42a637caa9e5 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 11 May 2021 16:43:24 -0500 Subject: [PATCH 026/192] Updated CFN template default Data Prepper version. --- deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index e764d957a6..f44fd91e12 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -28,7 +28,7 @@ Parameters: Description: Version of Data Prepper to download and run Type: String AllowedPattern: "[0-9]+\\.[0-9]+\\.[0-9]+[a-z-]*" - Default: "0.8.0-beta" + Default: "1.0.0" ConstraintDescription: must be a valid release number IAMRole: Description: Pre-existing IAM Role to associate with the EC2 instance, to be used for authentication when calling Elasticsearch @@ -206,4 +206,4 @@ Resources: FromPort: "22" ToPort: "22" CidrIp: - Ref: SSHLocation \ No newline at end of file + Ref: SSHLocation From 02e474c2619ec93da0c7f1aaa2e3deba6aa5d8eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 May 2021 17:17:22 -0500 Subject: [PATCH 027/192] Maintenance: Bump com.bmuschko.docker-remote-api from 6.6.1 to 7.0.0 in /data-prepper-core (#551) * Bump com.bmuschko.docker-remote-api in /data-prepper-core Bumps com.bmuschko.docker-remote-api from 6.6.1 to 7.0.0. Signed-off-by: dependabot[bot] * Bump com.bmuschko.docker-remote-api in /data-prepper-core Bumps com.bmuschko.docker-remote-api from 6.6.1 to 7.0.0. Signed-off-by: dependabot[bot] * MAINT: update version in buildScript Signed-off-by: qchea Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: qchea --- data-prepper-core/build.gradle | 2 +- data-prepper-core/integrationTest.gradle | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 2f02d05aab..5b58b3eafc 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.bmuschko.docker-remote-api' version '6.6.1' + id 'com.bmuschko.docker-remote-api' version '7.0.0' } sourceSets { diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 7dfb517c5d..1ea096ad9a 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -1,9 +1,11 @@ buildscript { repositories { - jcenter() + maven { + url "https://plugins.gradle.org/m2/" + } } dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:6.6.1' + classpath 'com.bmuschko:gradle-docker-plugin:7.0.0' } } From 6f079bd300935cd2350db007d326b799a458ba6b Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Thu, 13 May 2021 13:50:14 -0500 Subject: [PATCH 028/192] Infrastructure: add spotless code formatting -- first pass (#584) * MAINT: apply spotless in build.gradle Signed-off-by: qchea * FIX: *.yaml Signed-off-by: qchea * MAINT: introduce simple java formatting rule Signed-off-by: qchea * MAINT: not reformatting *.gradle yet Signed-off-by: qchea * MAINT: reformat build.gradle Signed-off-by: qchea * FIX: apply gradle docker plugin in external script Signed-off-by: qchea * MAINT: separate our markdown format Signed-off-by: qchea --- README.md | 2 +- build.gradle | 26 +++++++++++++++++++ .../configuration/PluginSettingsTests.java | 1 - .../mapdb-benchmarks/README.md | 8 +++--- .../READEME.md | 8 +++--- data-prepper-core/build.gradle | 4 --- data-prepper-core/integrationTest.gradle | 2 +- .../blocking-buffer/README.md | 8 +++--- data-prepper-plugins/common/README.md | 2 +- data-prepper-plugins/elasticsearch/README.md | 18 ++++++------- .../elasticsearch/odfe_security.md | 4 +-- .../elasticsearch/security.md | 18 ++++++------- .../ConnectionConfiguration.java | 1 - .../elasticsearch/IndexStateManagement.java | 1 - .../otel-trace-group-prepper/README.md | 14 +++++----- .../otel-trace-raw-prepper/README.md | 8 +++--- .../prepper/oteltrace/model/RawSpan.java | 1 - .../oteltrace/model/OTelProtoHelperTest.java | 1 - .../oteltrace/model/RawBuilderTest.java | 2 -- .../otel-trace-source/README.md | 10 +++---- .../health/HealthGrpcServiceTest.java | 1 - data-prepper-plugins/peer-forwarder/README.md | 10 +++---- .../peerforwarder/PeerForwarderUtils.java | 4 --- .../service-map-stateful/README.md | 12 ++++----- research/zipkin-elastic-to-otel/README.md | 2 +- 25 files changed, 89 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 9f307082a9..7f90184371 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ - [Code of Conduct](#Code-of-Conduct) - [Security Issue Notifications](#Security-Issue-Notifications) - [License](#License) - + ## Contribute diff --git a/build.gradle b/build.gradle index 695c217cb9..f4e31e520b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,17 +1,43 @@ +plugins { + id "com.diffplug.spotless" version "5.12.4" +} + apply from: file("${rootDir}/build-resources.gradle") allprojects { + apply plugin: 'com.diffplug.spotless' + group = 'com.amazon' version = '0.8.0-beta' repositories { mavenCentral() maven { url "https://jitpack.io" } } + + spotless { + format 'markdown', { + target '*.md' + // TODO: enrich format rules + endWithNewline() + } + format 'misc', { + target '.gitignore', '*.yml', '*.yaml' + // TODO: enrich format rules + trimTrailingWhitespace() + endWithNewline() + } + } } subprojects { apply plugin: 'java' apply plugin: 'jacoco' sourceCompatibility = '1.8' + spotless { + java { + // TODO: enrich format rules + removeUnusedImports() + } + } dependencies { implementation "com.google.guava:guava:29.0-jre" implementation "org.apache.logging.log4j:log4j-core:2.14.0" diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java index f2d1eadd5e..723011c012 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/model/configuration/PluginSettingsTests.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/data-prepper-benchmarks/mapdb-benchmarks/README.md b/data-prepper-benchmarks/mapdb-benchmarks/README.md index 324308fb59..98c3e1d9cd 100644 --- a/data-prepper-benchmarks/mapdb-benchmarks/README.md +++ b/data-prepper-benchmarks/mapdb-benchmarks/README.md @@ -1,8 +1,8 @@ # MapDb Benchmarks -This package uses JMH (https://openjdk.java.net/projects/code-tools/jmh/) to benchmark the MapDb Prepper State plugin. -To use jmh benchmarking easily with gradle, this package uses a jmh gradle plugin (https://github.com/melix/jmh-gradle-plugin/) . -Details on configuration and other options can be found there. +This package uses JMH (https://openjdk.java.net/projects/code-tools/jmh/) to benchmark the MapDb Prepper State plugin. +To use jmh benchmarking easily with gradle, this package uses a jmh gradle plugin (https://github.com/melix/jmh-gradle-plugin/) . +Details on configuration and other options can be found there. To run the benchmarks from this directory, run the following command: @@ -14,4 +14,4 @@ To build an executable standalone jar of these benchmarks, run: ``` ../../gradlew jmhJar -``` \ No newline at end of file +``` diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md b/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md index 4a1e721ba9..3e5e113da6 100644 --- a/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md +++ b/data-prepper-benchmarks/service-map-stateful-benchmarks/READEME.md @@ -1,6 +1,6 @@ # Service Map Stateful Benchmarks -This package contains benchmarks for the service map stateful prepper using JMH: https://openjdk.java.net/projects/code-tools/jmh/ . +This package contains benchmarks for the service map stateful prepper using JMH: https://openjdk.java.net/projects/code-tools/jmh/ . Integration with gradle is done with the following gradle plugin for JMH: https://github.com/melix/jmh-gradle-plugin. @@ -9,11 +9,11 @@ The plugin creates a source set for the JMH benchmarks, and provides a few gradl ## Running the tests via gradle task Tests can be run via the "jmh" gradle task provided by the plugin. The README for the plugin provides the various parameters that -can be provided to the plugin. +can be provided to the plugin. ## Running the tests via JAR -To run the tests via JAR, you can build the benchmark jar using the gradle task "jmhJar". This jar is an executable jar +To run the tests via JAR, you can build the benchmark jar using the gradle task "jmhJar". This jar is an executable jar that runs the benchmark tests. Example command: ``` @@ -21,4 +21,4 @@ java -jar service-map-stateful-benchmarks-0.1-beta-jmh.jar -r 600 -i 2 -p batchS ``` The above command will run the benchmarks for 600 seconds (10 minutes) per iteration, 2 iterations. It also -sets the batchSize and windowDurationSeconds benchmark parameters. +sets the batchSize and windowDurationSeconds benchmark parameters. diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 5b58b3eafc..a5e03a5500 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -1,7 +1,3 @@ -plugins { - id 'com.bmuschko.docker-remote-api' version '7.0.0' -} - sourceSets { main { resources { diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 1ea096ad9a..ca407a7604 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -9,7 +9,7 @@ buildscript { } } -apply plugin: 'com.bmuschko.docker-remote-api' +apply plugin: com.bmuschko.gradle.docker.DockerRemoteApiPlugin import com.bmuschko.gradle.docker.tasks.container.* import com.bmuschko.gradle.docker.tasks.image.* diff --git a/data-prepper-plugins/blocking-buffer/README.md b/data-prepper-plugins/blocking-buffer/README.md index 740ece7d75..783e3796a0 100644 --- a/data-prepper-plugins/blocking-buffer/README.md +++ b/data-prepper-plugins/blocking-buffer/README.md @@ -8,7 +8,7 @@ Example `.yaml` configuration buffer: - bounded_blocking: ``` -*Note*: *By default, Data Prepper uses only one buffer. the `bounded_blocking` buffer, so this section in the `.yaml` need not be defined unless one wants to mention a custom buffer or tune the buffer settings* +*Note*: *By default, Data Prepper uses only one buffer. the `bounded_blocking` buffer, so this section in the `.yaml` need not be defined unless one wants to mention a custom buffer or tune the buffer settings* ## Configuration - buffer_size => An `int` representing max number of unchecked records the buffer accepts (num of unchecked records = num of records written into the buffer + num of in-flight records not yet checked by the Checkpointing API). Default is `512`. @@ -18,6 +18,6 @@ buffer: This plugin inherits the common metrics defined in [AbstractBuffer](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java) ## Developer Guide -This plugin is compatible with Java 14. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +This plugin is compatible with Java 14. See +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/common/README.md b/data-prepper-plugins/common/README.md index 6e1beddd26..86da8ea331 100644 --- a/data-prepper-plugins/common/README.md +++ b/data-prepper-plugins/common/README.md @@ -24,4 +24,4 @@ A source plugin to read input data from console. ## `stdout` -A sink plugin to write output data to console. \ No newline at end of file +A sink plugin to write output data to console. diff --git a/data-prepper-plugins/elasticsearch/README.md b/data-prepper-plugins/elasticsearch/README.md index 4b3d470e3b..7f0bc798ef 100644 --- a/data-prepper-plugins/elasticsearch/README.md +++ b/data-prepper-plugins/elasticsearch/README.md @@ -52,7 +52,7 @@ pipeline: sink: elasticsearch: hosts: ["https://your-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true cert: path/to/cert insecure: false trace_analytics_service_map: true @@ -64,9 +64,9 @@ pipeline: - `hosts`: A list of IP addresses of elasticsearch nodes. - `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that ODFE is using. -Default is null. +Default is null. -- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. +- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. @@ -115,14 +115,14 @@ Default value is false. Set it to true for [Service map trace analytics](#servic - `index`: A String used as index name for custom data type. Applicable and required only If both `trace_analytics_raw` and `trace_analytics_service_map` are set to false. - `template_file`(optional): A json file path to be read as index template for custom data ingestion. The json file content should be the json value of -`"template"` key in the json content of elasticsearch [Index templates API](https://www.elastic.co/guide/en/elasticsearch/reference/7.8/index-templates.html), +`"template"` key in the json content of elasticsearch [Index templates API](https://www.elastic.co/guide/en/elasticsearch/reference/7.8/index-templates.html), e.g. [otel-v1-apm-span-index-template.json](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json) - `dlq_file`(optional): A String of absolute file path for DLQ failed output records. Defaults to null. If not provided, failed records will be written into the default data-prepper log file (`logs/Data-Prepper.log`). -- `bulk_size` (optional): A long of bulk size in bulk requests in MB. Default to 5 MB. If set to be less than 0, -all the records received from the upstream prepper at a time will be sent as a single bulk request. +- `bulk_size` (optional): A long of bulk size in bulk requests in MB. Default to 5 MB. If set to be less than 0, +all the records received from the upstream prepper at a time will be sent as a single bulk request. If a single record turns out to be larger than the set bulk size, it will be sent as a bulk request of a single document. ## Metrics @@ -138,11 +138,11 @@ Besides common metrics in [AbstractSink](https://github.com/opendistro-for-elast - `bulkRequestErrors`: measures number of errors encountered in sending bulk requests. - `documentsSuccess`: measures number of documents successfully sent to ES by bulk requests including retries. - `documentsSuccessFirstAttempt`: measures number of documents successfully sent to ES by bulk requests on first attempt. -- `documentErrors`: measures number of documents failed to be sent by bulk requests. +- `documentErrors`: measures number of documents failed to be sent by bulk requests. ## Developer Guide -This plugin is compatible with Java 8. See +This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) - [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/elasticsearch/odfe_security.md b/data-prepper-plugins/elasticsearch/odfe_security.md index 047686980c..676d49cc0a 100644 --- a/data-prepper-plugins/elasticsearch/odfe_security.md +++ b/data-prepper-plugins/elasticsearch/odfe_security.md @@ -11,7 +11,7 @@ sink: password: "admin" ``` -or by using user credential assigned with a role that has the below required permissions. +or by using user credential assigned with a role that has the below required permissions. ### Cluster permissions @@ -31,4 +31,4 @@ Note that `indices:admin/template/*` need to be in cluster permissions. --------------- -With administrative privilege, one can create an internal user, a role and map the user to the role by following the ODFE [instructions](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/). \ No newline at end of file +With administrative privilege, one can create an internal user, a role and map the user to the role by following the ODFE [instructions](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles/). diff --git a/data-prepper-plugins/elasticsearch/security.md b/data-prepper-plugins/elasticsearch/security.md index 40440714bd..544c67f8c3 100644 --- a/data-prepper-plugins/elasticsearch/security.md +++ b/data-prepper-plugins/elasticsearch/security.md @@ -4,7 +4,7 @@ This document provides more details about the security settings of the sink. ## AWS Elasticsearch Service -Elasticsearch sink is capable of sending data to Amazon Elasticsearch domain which use Identity and Access Management. The plugin uses the default credential chain. Run `aws configure` using the AWS CLI to set your credentials. +Elasticsearch sink is capable of sending data to Amazon Elasticsearch domain which use Identity and Access Management. The plugin uses the default credential chain. Run `aws configure` using the AWS CLI to set your credentials. You should ensure that the credentials you configure have the required permissions. Below is an example Resource based policy, with required set of permissions that is required for the sink to work, @@ -36,33 +36,33 @@ You should ensure that the credentials you configure have the required permissio } ] } -``` +``` Please check this [doc](https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-ac.html) to know how to set IAM to your Elasticsearch domain, ### Fine-Grained Access Control (FGAC) in Amazon Elasticsearch Service The Elasticsearch sink creates an [Index State Management (ISM)](https://opendistro.github.io/for-elasticsearch-docs/docs/ism/) policy for Trace Analytics indices but Amazon Elasticsearch Service allows only the `master user` to create an ISM policy. So, - + * If you use IAM for your master user in FGAC domain, configure the sink as below, - + ``` sink: elasticsearch: hosts: ["https://your-fgac-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true ``` -Run `aws configure` using the AWS CLI to set your credentials to the master IAM user. - +Run `aws configure` using the AWS CLI to set your credentials to the master IAM user. + * If you use internal database for your master user in FGAC domain, configure the sink as below, - + ``` sink: elasticsearch: hosts: ["https://your-fgac-amazon-elasticssearch-service-endpoint"] aws_sigv4: false username: "master-username" - password: "master-password" + password: "master-password" ``` Note: You can create a new IAM/internal user with `all_access` and use instead of the master IAM/internal user. diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index 25262b61be..a83ba66329 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -14,7 +14,6 @@ import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java index 53382de43f..99bbac95f8 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java @@ -1,6 +1,5 @@ package com.amazon.dataprepper.plugins.sink.elasticsearch; -import com.google.common.base.Preconditions; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.elasticsearch.client.Request; diff --git a/data-prepper-plugins/otel-trace-group-prepper/README.md b/data-prepper-plugins/otel-trace-group-prepper/README.md index 32e28525ed..323c199770 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/README.md +++ b/data-prepper-plugins/otel-trace-group-prepper/README.md @@ -16,7 +16,7 @@ pipeline: cert: path/to/cert username: YOUR_USERNAME_HERE password: YOUR_PASSWORD_HERE -``` +``` See [odfe_security.md](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-plugins/elasticsearch/odfe_security.md) for detailed explanation. @@ -28,7 +28,7 @@ pipeline: prepper: - otel-trace-group-prepper: hosts: ["https://your-amazon-elasticssearch-service-endpoint"] - aws_sigv4: true + aws_sigv4: true cert: path/to/cert insecure: false ``` @@ -40,9 +40,9 @@ See [security.md](https://github.com/opendistro-for-elasticsearch/data-prepper/b - `hosts`: A list of IP addresses of elasticsearch nodes. - `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that ODFE is using. -Default is null. +Default is null. -- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. +- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon Elasticsearch Service. See [security](security.md) for details. Default to `false`. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. @@ -61,7 +61,7 @@ Default is null. ## Developer Guide -This plugin is compatible with Java 8. See +This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/README.md b/data-prepper-plugins/otel-trace-raw-prepper/README.md index 76762c0503..0b6a0716ba 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/README.md +++ b/data-prepper-plugins/otel-trace-raw-prepper/README.md @@ -1,6 +1,6 @@ # OTel Trace Raw Prepper -This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. +This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. ## Usages Example `.yaml` configuration @@ -23,6 +23,6 @@ Apart from common metrics in [AbstractPrepper](https://github.com/opendistro-for - `totalProcessingErrors`: records the total number of processing errors for spans and resource spans. ## Developer Guide -This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +This plugin is compatible with Java 8. See +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java index 25d03dd19b..00d7b83851 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java @@ -1,7 +1,6 @@ package com.amazon.dataprepper.plugins.prepper.oteltrace.model; import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java index d62e381ddb..8cd77feaeb 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java @@ -17,7 +17,6 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java index 7a006bd7b2..8abe972a0c 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java @@ -10,8 +10,6 @@ import org.junit.Test; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index ed48615dc1..b09de5ac79 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -1,4 +1,4 @@ -# OTel Trace Source +# OTel Trace Source This is a source which follows the [OTLP Protocol](https://github.com/open-telemetry/oteps/blob/master/text/0035-opentelemetry-protocol.md). This source supports ```OTLP/grpc``` and ```OTLP/HTTP```. Support for ```OTLP/HTTP+JSON``` is not complete due as the traceId and spanId will be ```base64``` and not ```HexString```. @@ -12,7 +12,7 @@ source: ## Configurations -* port(Optional) => An `int` represents the port Otel trace source is running on. Default is ```21890```. +* port(Optional) => An `int` represents the port Otel trace source is running on. Default is ```21890```. * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables a gRPC health check service under ```grpc.health.v1 / Health / Check```. Default is ```false```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. @@ -26,9 +26,9 @@ source: ### Counter - `requestTimeouts`: measures total number of requests that time out. -- `requestsReceived`: measures total number of requests received by otel trace source. +- `requestsReceived`: measures total number of requests received by otel trace source. ## Developer Guide -This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +This plugin is compatible with Java 8. See +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) - [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java index 48cb0eeeaf..81809ba7d6 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com.amazon.situp.plugins/health/HealthGrpcServiceTest.java @@ -14,7 +14,6 @@ import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; public class HealthGrpcServiceTest { diff --git a/data-prepper-plugins/peer-forwarder/README.md b/data-prepper-plugins/peer-forwarder/README.md index a87009d222..64437a1417 100644 --- a/data-prepper-plugins/peer-forwarder/README.md +++ b/data-prepper-plugins/peer-forwarder/README.md @@ -1,5 +1,5 @@ # Peer Forwarder -This prepper forwards `ExportTraceServiceRequests` via gRPC to other Data Prepper instances. +This prepper forwards `ExportTraceServiceRequests` via gRPC to other Data Prepper instances. ## Usage The primary usecase of this prepper is to ensure that groups of traces are aggregated by trace ID and processed by the same Prepper instance. @@ -67,11 +67,11 @@ Besides common metrics in [AbstractPrepper](https://github.com/opendistro-for-el ### Gauge -- `peerEndpoints`: measures number of dynamically discovered peer data-prepper endpoints. For `static` mode, the size is fixed. +- `peerEndpoints`: measures number of dynamically discovered peer data-prepper endpoints. For `static` mode, the size is fixed. ## Developer Guide -This plugin is compatible with Java 14. See +This plugin is compatible with Java 14. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java index 473be1f9d9..a619b80584 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java @@ -5,10 +5,6 @@ import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.net.UnknownHostException; import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; diff --git a/data-prepper-plugins/service-map-stateful/README.md b/data-prepper-plugins/service-map-stateful/README.md index 8f271a4bbd..bc18c88efe 100644 --- a/data-prepper-plugins/service-map-stateful/README.md +++ b/data-prepper-plugins/service-map-stateful/README.md @@ -1,6 +1,6 @@ # Service-Map Stateful Prepper -This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. +This is a special prepper that consumes Opentelemetry traces, stores them in a MapDB data store and evaluate relationships at fixed ```window_duration```. # Usages Example `.yaml` configuration: @@ -11,16 +11,16 @@ prepper: ## Configurations -* window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. +* window_duration(Optional) => An `int` represents the fixed time window in seconds to evaluate service-map relationships. Default is ```180```. ## Metrics Besides common metrics in [AbstractPrepper](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/prepper/AbstractPrepper.java), service-map-stateful prepper introduces the following custom metrics. ### Gauge - `spansDbSize`: measures total spans byte sizes in MapDB across the current and previous window durations. -- `traceGroupDbSize`: measures total trace group byte sizes in MapDB across the current and previous trace group window durations. +- `traceGroupDbSize`: measures total trace group byte sizes in MapDB across the current and previous trace group window durations. ## Developer Guide -This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) \ No newline at end of file +This plugin is compatible with Java 8. See +- [CONTRIBUTING](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opendistro-for-elasticsearch/data-prepper/blob/main/docs/readme/monitoring.md) diff --git a/research/zipkin-elastic-to-otel/README.md b/research/zipkin-elastic-to-otel/README.md index 506c9efbad..bff1e04c41 100644 --- a/research/zipkin-elastic-to-otel/README.md +++ b/research/zipkin-elastic-to-otel/README.md @@ -19,4 +19,4 @@ Notice that `FIELD` and `VALUE` are optional arguments. ``` ./gradlew :research:zipkin-elastic-to-otel:run -Dtest=true --args YOUR_INDEX_PATTERN -``` \ No newline at end of file +``` From aafbc2d8ca049c4a279a160c30cde3df72c7f5e4 Mon Sep 17 00:00:00 2001 From: Bala Yadav <55510769+yadavcbala@users.noreply.github.com> Date: Thu, 13 May 2021 14:02:42 -0500 Subject: [PATCH 029/192] Add capability to configure MeterRegistry via server configuration file (#585) * Add capability to configure metrics registry using server configuration file * Reformat & Rename Signed-off-by: Bala Yadav * Fix Test Signed-off-by: Bala Yadav * Remove unused imports Signed-off-by: Bala Yadav * Use plural names for List Signed-off-by: Bala Yadav * update project-setup documentation Signed-off-by: Bala Yadav * update Test data for plural * Refactor - for naming and switch cases Signed-off-by: Bala Yadav * Remove unused imports Signed-off-by: Bala Yadav * Remove redundant code Signed-off-by: Bala Yadav --- .../com/amazon/dataprepper/DataPrepper.java | 71 ++++++++------ .../dataprepper/DataPrepperExecute.java | 2 + .../model/DataPrepperConfiguration.java | 17 +++- .../parser/model/MetricRegistryType.java | 25 +++++ .../pipeline/server/DataPrepperServer.java | 34 +++++-- .../server/PrometheusMetricsHandler.java | 10 +- .../amazon/dataprepper/DataPrepperTests.java | 16 ++-- .../amazon/dataprepper/TestDataProvider.java | 2 + .../model/DataPrepperConfigurationTests.java | 28 ++++++ .../CloudWatchMeterRegistryProviderTest.java | 23 ----- .../server/DataPrepperServerTest.java | 93 +++++++++++++++++-- ...data_prepper_cloudwatch_metrics_config.yml | 2 + ...d_data_prepper_multiple_metrics_config.yml | 2 + docs/readme/configuration.md | 2 + docs/readme/monitoring.md | 13 ++- docs/readme/project_setup.md | 8 +- 16 files changed, 256 insertions(+), 92 deletions(-) create mode 100644 data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java create mode 100644 data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml create mode 100644 data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 2743d2bbba..3698d55a7c 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -2,6 +2,7 @@ import com.amazon.dataprepper.parser.PipelineParser; import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; +import com.amazon.dataprepper.parser.model.MetricRegistryType; import com.amazon.dataprepper.pipeline.Pipeline; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; import io.micrometer.core.instrument.Metrics; @@ -10,12 +11,12 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; -import io.micrometer.prometheus.PrometheusConfig; -import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.List; import java.util.Map; /** @@ -27,32 +28,31 @@ public class DataPrepper { private static final Logger LOG = LoggerFactory.getLogger(DataPrepper.class); - private static final PrometheusMeterRegistry sysJVMMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + private static final CompositeMeterRegistry systemMeterRegistry = new CompositeMeterRegistry(); private Map transformationPipelines; private static volatile DataPrepper dataPrepper; private static DataPrepperServer dataPrepperServer; - private static DataPrepperConfiguration configuration = DataPrepperConfiguration.DEFAULT_CONFIG; - - static { - new ClassLoaderMetrics().bindTo(sysJVMMeterRegistry); - new JvmMemoryMetrics().bindTo(sysJVMMeterRegistry); - new JvmGcMetrics().bindTo(sysJVMMeterRegistry); - new ProcessorMetrics().bindTo(sysJVMMeterRegistry); - new JvmThreadMetrics().bindTo(sysJVMMeterRegistry); - } + private static DataPrepperConfiguration configuration; /** - * Set the DataPrepperConfiguration from a file + * Set the DataPrepperConfiguration from file + * * @param configurationFile File containing DataPrepperConfiguration yaml */ public static void configure(final String configurationFile) { - final DataPrepperConfiguration dataPrepperConfiguration = - DataPrepperConfiguration.fromFile(new File(configurationFile)); + configuration = DataPrepperConfiguration.fromFile(new File(configurationFile)); + configureMeterRegistry(); + } - configuration = dataPrepperConfiguration; + /** + * Set the DataPrepperConfiguration with defaults + */ + public static void configureWithDefaults() { + configuration = DataPrepperConfiguration.DEFAULT_CONFIG; + configureMeterRegistry(); } public static DataPrepper getInstance() { @@ -69,20 +69,32 @@ private DataPrepper() { if (dataPrepper != null) { throw new RuntimeException("Please use getInstance() for an instance of this Data Prepper"); } - startPrometheusBackend(); + startMeterRegistryForDataPrepper(); dataPrepperServer = new DataPrepperServer(this); } - public static PrometheusMeterRegistry getSysJVMMeterRegistry() { - return sysJVMMeterRegistry; - } - /** - * Create a PrometheusMeterRegistry for this DataPrepper and register it with the global registry + * Creates instances of configured MeterRegistry and registers to {@link Metrics} globalRegistry to be used by + * Meters. */ - private static void startPrometheusBackend() { - final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); - Metrics.addRegistry(prometheusMeterRegistry); + private static void startMeterRegistryForDataPrepper() { + final List configuredMetricRegistryTypes = configuration.getMetricRegistryTypes(); + configuredMetricRegistryTypes.forEach(metricRegistryType -> Metrics.addRegistry(MetricRegistryType + .getDefaultMeterRegistryForType(metricRegistryType))); + } + + private static void configureMeterRegistry() { + configuration.getMetricRegistryTypes().forEach(metricRegistryType -> + systemMeterRegistry.add(MetricRegistryType.getDefaultMeterRegistryForType(metricRegistryType))); + new ClassLoaderMetrics().bindTo(systemMeterRegistry); + new JvmMemoryMetrics().bindTo(systemMeterRegistry); + new JvmGcMetrics().bindTo(systemMeterRegistry); + new ProcessorMetrics().bindTo(systemMeterRegistry); + new JvmThreadMetrics().bindTo(systemMeterRegistry); + } + + public static CompositeMeterRegistry getSystemMeterRegistry() { + return systemMeterRegistry; } /** @@ -95,7 +107,7 @@ public boolean execute(final String configurationFileLocation) { LOG.info("Using {} configuration file", configurationFileLocation); final PipelineParser pipelineParser = new PipelineParser(configurationFileLocation); transformationPipelines = pipelineParser.parseConfiguration(); - if (transformationPipelines.size() == 0){ + if (transformationPipelines.size() == 0) { LOG.error("No valid pipeline is available for execution, exiting"); System.exit(1); } @@ -115,22 +127,23 @@ public void shutdown() { /** * Triggers shutdown of the Data Prepper server. */ - public void shutdownDataPrepperServer(){ + public void shutdownDataPrepperServer() { dataPrepperServer.stop(); } /** * Triggers shutdown of the provided pipeline, no-op if the pipeline does not exist. + * * @param pipeline name of the pipeline */ public void shutdown(final String pipeline) { - if(transformationPipelines.containsKey(pipeline)) { + if (transformationPipelines.containsKey(pipeline)) { transformationPipelines.get(pipeline).shutdown(); } } public Map getTransformationPipelines() { - return transformationPipelines; + return transformationPipelines; } public static DataPrepperConfiguration getConfiguration() { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java index a6acc2bff5..9d71b9717d 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java @@ -14,6 +14,8 @@ public static void main(String[] args) { if(args.length > 1) { DataPrepper.configure(args[1]); + } else { + DataPrepper.configureWithDefaults(); } final DataPrepper dataPrepper = DataPrepper.getInstance(); if (args.length > 0) { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java index 34186ead0f..4530f3e321 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java @@ -1,21 +1,26 @@ package com.amazon.dataprepper.parser.model; -import java.io.File; -import java.io.IOException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + /** * Class to hold configuration for DataPrepper, including server port and Log4j settings */ public class DataPrepperConfiguration { + private static final List DEFAULT_METRIC_REGISTRY_TYPE = Collections.singletonList(MetricRegistryType.Prometheus); private int serverPort = 4900; private boolean ssl = true; private String keyStoreFilePath = ""; private String keyStorePassword = ""; private String privateKeyPassword = ""; + private List metricRegistries = DEFAULT_METRIC_REGISTRY_TYPE; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); @@ -42,12 +47,14 @@ public DataPrepperConfiguration( @JsonProperty("keyStoreFilePath") final String keyStoreFilePath, @JsonProperty("keyStorePassword") final String keyStorePassword, @JsonProperty("privateKeyPassword") final String privateKeyPassword, - @JsonProperty("serverPort") final String serverPort + @JsonProperty("serverPort") final String serverPort, + @JsonProperty("metricRegistries") final List metricRegistries ) { setSsl(ssl); this.keyStoreFilePath = keyStoreFilePath != null ? keyStoreFilePath : ""; this.keyStorePassword = keyStorePassword != null ? keyStorePassword : ""; this.privateKeyPassword = privateKeyPassword != null ? privateKeyPassword : ""; + this.metricRegistries = metricRegistries != null && !metricRegistries.isEmpty() ? metricRegistries : DEFAULT_METRIC_REGISTRY_TYPE; setServerPort(serverPort); } @@ -71,6 +78,10 @@ public String getPrivateKeyPassword() { return privateKeyPassword; } + public List getMetricRegistryTypes() { + return metricRegistries; + } + private void setSsl(final Boolean ssl) { if (ssl != null) { this.ssl = ssl; diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java new file mode 100644 index 0000000000..29417f990e --- /dev/null +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java @@ -0,0 +1,25 @@ +package com.amazon.dataprepper.parser.model; + +import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; + +import static java.lang.String.format; + +public enum MetricRegistryType { + Prometheus, + CloudWatch; + + public static MeterRegistry getDefaultMeterRegistryForType(final MetricRegistryType metricRegistryType) { + switch (metricRegistryType) { + case Prometheus: + return new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + case CloudWatch: + return new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); + default: + throw new IllegalArgumentException(format("Invalid metricRegistryType %s", metricRegistryType)); + + } + } +} \ No newline at end of file diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java index fb42b1af4a..e4c70da47f 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/DataPrepperServer.java @@ -1,10 +1,14 @@ package com.amazon.dataprepper.pipeline.server; import com.amazon.dataprepper.DataPrepper; +import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.sun.net.httpserver.HttpServer; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsParameters; import com.sun.net.httpserver.HttpsServer; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,6 +16,8 @@ import javax.net.ssl.SSLParameters; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.Optional; +import java.util.Set; /** * Class to handle any serving that the data prepper instance needs to do. @@ -22,11 +28,12 @@ public class DataPrepperServer { private HttpServer server; public DataPrepperServer(final DataPrepper dataPrepper) { - final int port = DataPrepper.getConfiguration().getServerPort(); - final boolean ssl = DataPrepper.getConfiguration().ssl(); - final String keyStoreFilePath = DataPrepper.getConfiguration().getKeyStoreFilePath(); - final String keyStorePassword = DataPrepper.getConfiguration().getKeyStorePassword(); - final String privateKeyPassword = DataPrepper.getConfiguration().getPrivateKeyPassword(); + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); + final int port = dataPrepperConfiguration.getServerPort(); + final boolean ssl = dataPrepperConfiguration.ssl(); + final String keyStoreFilePath = dataPrepperConfiguration.getKeyStoreFilePath(); + final String keyStorePassword = dataPrepperConfiguration.getKeyStorePassword(); + final String privateKeyPassword = dataPrepperConfiguration.getPrivateKeyPassword(); try { if (ssl) { @@ -40,12 +47,25 @@ public DataPrepperServer(final DataPrepper dataPrepper) { throw new RuntimeException("Failed to create server", e); } - server.createContext("/metrics/prometheus", new PrometheusMetricsHandler()); - server.createContext("/metrics/sys", new PrometheusMetricsHandler(DataPrepper.getSysJVMMeterRegistry())); + getPrometheusMeterRegistryFromRegistries(Metrics.globalRegistry.getRegistries()).ifPresent(meterRegistry -> { + final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = (PrometheusMeterRegistry) meterRegistry; + server.createContext("/metrics/prometheus", new PrometheusMetricsHandler(prometheusMeterRegistryForDataPrepper)); + }); + + getPrometheusMeterRegistryFromRegistries(DataPrepper.getSystemMeterRegistry().getRegistries()).ifPresent( + meterRegistry -> { + final PrometheusMeterRegistry prometheusMeterRegistryForSystem = (PrometheusMeterRegistry) meterRegistry; + server.createContext("/metrics/sys", new PrometheusMetricsHandler(prometheusMeterRegistryForSystem)); + }); server.createContext("/list", new ListPipelinesHandler(dataPrepper)); server.createContext("/shutdown", new ShutdownHandler(dataPrepper)); } + private Optional getPrometheusMeterRegistryFromRegistries(final Set meterRegistries) { + return meterRegistries.stream().filter(meterRegistry -> + meterRegistry instanceof PrometheusMeterRegistry).findFirst(); + } + /** * Start the DataPrepperServer */ diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java index 948b9280bf..62f2a9b907 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/server/PrometheusMetricsHandler.java @@ -1,14 +1,14 @@ package com.amazon.dataprepper.pipeline.server; -import java.io.IOException; -import java.net.HttpURLConnection; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; -import io.micrometer.core.instrument.Metrics; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.HttpURLConnection; + /** * HttpHandler to handle requests for Prometheus metrics */ @@ -17,10 +17,6 @@ public class PrometheusMetricsHandler implements HttpHandler { private PrometheusMeterRegistry prometheusMeterRegistry; private final Logger LOG = LoggerFactory.getLogger(PrometheusMetricsHandler.class); - public PrometheusMetricsHandler() { - prometheusMeterRegistry = (PrometheusMeterRegistry) Metrics.globalRegistry.getRegistries().iterator().next(); - } - public PrometheusMetricsHandler(final PrometheusMeterRegistry prometheusMeterRegistry) { this.prometheusMeterRegistry = prometheusMeterRegistry; } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java index 69e6e47786..ba5b2cf842 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/DataPrepperTests.java @@ -43,21 +43,21 @@ public void testInstanceCreation() { } @Test - public void testDataPrepperSysMetrics() { + public void testDataPrepperSystemMetrics() { // Test retrieve gauge in ClassLoaderMetrics - final List classesLoaded = getSysMeasurementList("jvm.classes.loaded"); + final List classesLoaded = getSystemMeasurementList("jvm.classes.loaded"); Assert.assertEquals(1, classesLoaded.size()); // Test retrieve gauge in JvmMemoryMetrics - final List jvmBufferCount = getSysMeasurementList("jvm.buffer.count"); + final List jvmBufferCount = getSystemMeasurementList("jvm.buffer.count"); Assert.assertEquals(1, jvmBufferCount.size()); // Test retrieve gauge in JvmGcMetrics - final List jvmGcMaxDataSize = getSysMeasurementList("jvm.gc.max.data.size"); + final List jvmGcMaxDataSize = getSystemMeasurementList("jvm.gc.max.data.size"); Assert.assertEquals(1, jvmGcMaxDataSize.size()); // Test retrieve gauge in ProcessorMetrics - final List sysCPUCount = getSysMeasurementList("system.cpu.count"); + final List sysCPUCount = getSystemMeasurementList("system.cpu.count"); Assert.assertEquals(1, sysCPUCount.size()); // Test retrieve gauge in JvmThreadMetrics - final List jvmThreadsPeak = getSysMeasurementList("jvm.threads.peak"); + final List jvmThreadsPeak = getSystemMeasurementList("jvm.threads.peak"); Assert.assertEquals(1, jvmThreadsPeak.size()); } @@ -122,8 +122,8 @@ public int getExitStatus() { } } - private static List getSysMeasurementList(final String meterName) { - return StreamSupport.stream(DataPrepper.getSysJVMMeterRegistry().find(meterName).meter().measure().spliterator(), false) + private static List getSystemMeasurementList(final String meterName) { + return StreamSupport.stream(DataPrepper.getSystemMeterRegistry().find(meterName).meter().measure().spliterator(), false) .collect(Collectors.toList()); } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java index c0d42218cd..29d1086399 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/TestDataProvider.java @@ -40,6 +40,8 @@ public class TestDataProvider { public static final String VALID_DATA_PREPPER_CONFIG_FILE_WITH_TLS = "src/test/resources/valid_data_prepper_config_with_tls.yml"; public static final String VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE = "src/test/resources/valid_data_prepper_config_default_log4j.yml"; public static final String VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE = "src/test/resources/valid_data_prepper_some_default_config.yml"; + public static final String VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE = "src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml"; + public static final String VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE = "src/test/resources/valid_data_prepper_multiple_metrics_config.yml"; public static final String INVALID_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_data_prepper_config.yml"; public static final String INVALID_PORT_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_port_data_prepper_config.yml"; public static final String INVALID_KEYSTORE_PASSWORD_DATA_PREPPER_CONFIG_FILE = "src/test/resources/invalid_data_prepper_config_with_bad_keystore_password.yml"; diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java index 26b282bc21..695f752bb5 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/parser/model/DataPrepperConfigurationTests.java @@ -1,5 +1,6 @@ package com.amazon.dataprepper.parser.model; +import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; @@ -7,8 +8,11 @@ import static com.amazon.dataprepper.TestDataProvider.INVALID_DATA_PREPPER_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.INVALID_PORT_DATA_PREPPER_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_SOME_DEFAULT_CONFIG_FILE; +import static org.hamcrest.MatcherAssert.assertThat; public class DataPrepperConfigurationTests { @@ -26,6 +30,30 @@ public void testSomeDefaultConfig() { Assert.assertEquals(DataPrepperConfiguration.DEFAULT_CONFIG.getServerPort(), dataPrepperConfiguration.getServerPort()); } + @Test + public void testDefaultMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.DEFAULT_CONFIG; + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + } + + @Test + public void testCloudWatchMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = + DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(1)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + + @Test + public void testMultipleMetricsRegistry() { + final DataPrepperConfiguration dataPrepperConfiguration = + DataPrepperConfiguration.fromFile(new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes().size(), Matchers.equalTo(2)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.Prometheus)); + assertThat(dataPrepperConfiguration.getMetricRegistryTypes(), Matchers.hasItem(MetricRegistryType.CloudWatch)); + } + @Test(expected = IllegalArgumentException.class) public void testInvalidConfig() { final DataPrepperConfiguration dataPrepperConfiguration = diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java index 24f8c0bab0..84fbcdad50 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/CloudWatchMeterRegistryProviderTest.java @@ -2,24 +2,14 @@ import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; -import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataResponse; - -import java.util.concurrent.CompletableFuture; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class CloudWatchMeterRegistryProviderTest { @@ -28,16 +18,6 @@ public class CloudWatchMeterRegistryProviderTest { @Mock CloudWatchAsyncClient cloudWatchAsyncClient; - @Mock - CompletableFuture putMetricDataResponseCompletableFuture; - - - @Before - public void setup() throws Exception { - when(cloudWatchAsyncClient.putMetricData(any(PutMetricDataRequest.class))).thenReturn(putMetricDataResponseCompletableFuture); - doAnswer(ans -> null).when(putMetricDataResponseCompletableFuture).whenCompleteAsync(any()); - } - @Test(expected = NullPointerException.class) public void testCreateWithInvalidPropertiesFile() { new CloudWatchMeterRegistryProvider("does not exist", cloudWatchAsyncClient); @@ -49,9 +29,6 @@ public void testCreateCloudWatchMeterRegistry() { TEST_CLOUDWATCH_PROPERTIES, cloudWatchAsyncClient); final CloudWatchMeterRegistry cloudWatchMeterRegistry = cloudWatchMeterRegistryProvider.getCloudWatchMeterRegistry(); assertThat(cloudWatchMeterRegistry, notNullValue()); - cloudWatchMeterRegistry.gauge("test-gauge", 1d); - cloudWatchMeterRegistry.close(); //This will trigger publishing of metrics - verify(cloudWatchAsyncClient, atLeast(1)).putMetricData(any(PutMetricDataRequest.class)); } } diff --git a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java index d2020933bd..d3bad94fd9 100644 --- a/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java +++ b/data-prepper-core/src/test/java/com/amazon/dataprepper/server/DataPrepperServerTest.java @@ -5,9 +5,13 @@ import com.amazon.dataprepper.parser.model.DataPrepperConfiguration; import com.amazon.dataprepper.pipeline.server.DataPrepperServer; import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.cloudwatch2.CloudWatchMeterRegistry; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -18,7 +22,9 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import java.io.File; import java.io.IOException; +import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -29,8 +35,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.Properties; +import java.util.Set; import java.util.UUID; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE; +import static com.amazon.dataprepper.TestDataProvider.VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.Mockito.mock; + public class DataPrepperServerTest { @@ -40,12 +52,17 @@ public class DataPrepperServerTest { private final int port = 1234; private void setRegistry(PrometheusMeterRegistry prometheusMeterRegistry) { - Metrics.globalRegistry.getRegistries().iterator().forEachRemaining(meterRegistry -> Metrics.globalRegistry.remove(meterRegistry)); + final Set meterRegistries = Metrics.globalRegistry.getRegistries(); + //to avoid ConcurrentModificationException + final Object[] registeredMeterRegistries = meterRegistries.toArray(); + for (final Object meterRegistry : registeredMeterRegistries) { + Metrics.removeRegistry((MeterRegistry) meterRegistry); + } Metrics.addRegistry(prometheusMeterRegistry); } private void setupDataPrepper() { - dataPrepper = Mockito.mock(DataPrepper.class); + dataPrepper = mock(DataPrepper.class); DataPrepper.configure(TestDataProvider.VALID_DATA_PREPPER_DEFAULT_LOG4J_CONFIG_FILE); } @@ -97,18 +114,20 @@ public void testScrapeGlobalFailure() throws IOException, InterruptedException, } @Test - public void testGetSysMetrics() throws IOException, InterruptedException, URISyntaxException { + public void testGetSystemMetrics() throws IOException, InterruptedException, URISyntaxException { final String scrape = UUID.randomUUID().toString(); final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, scrape); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(prometheusMeterRegistry); setupDataPrepper(); final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperMockedStatic.when(DataPrepper::getSysJVMMeterRegistry).thenReturn(prometheusMeterRegistry); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); dataPrepperServer = new DataPrepperServer(dataPrepper); dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:"+ port + "/metrics/sys")) + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) .GET().build(); HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); @@ -117,17 +136,19 @@ public void testGetSysMetrics() throws IOException, InterruptedException, URISyn } @Test - public void testScrapeSysMetricsFailure() throws IOException, InterruptedException, URISyntaxException { + public void testScrapeSystemMetricsFailure() throws IOException, InterruptedException, URISyntaxException { final PrometheusMeterRegistry prometheusMeterRegistry = new PrometheusRegistryThrowingScrape(PrometheusConfig.DEFAULT); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(prometheusMeterRegistry); setupDataPrepper(); final DataPrepperConfiguration dataPrepperConfiguration = DataPrepper.getConfiguration(); try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { - dataPrepperMockedStatic.when(DataPrepper::getSysJVMMeterRegistry).thenReturn(prometheusMeterRegistry); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); dataPrepperServer = new DataPrepperServer(dataPrepper); dataPrepperServer.start(); - HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:"+ port + "/metrics/sys")) + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) .GET().build(); HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); Assert.assertEquals(HttpURLConnection.HTTP_INTERNAL_ERROR, response.statusCode()); @@ -135,6 +156,62 @@ public void testScrapeSysMetricsFailure() throws IOException, InterruptedExcepti } } + @Test + public void testNonPrometheusMeterRegistry() throws Exception { + final CloudWatchMeterRegistry cloudWatchMeterRegistry = mock(CloudWatchMeterRegistry.class); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(cloudWatchMeterRegistry); + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( + new File(VALID_DATA_PREPPER_CLOUDWATCH_METRICS_CONFIG_FILE)); + setupDataPrepper(); + try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { + dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); + dataPrepperServer = new DataPrepperServer(dataPrepper); + dataPrepperServer.start(); + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + port + "/metrics/sys")) + .GET().build(); + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + } catch (ConnectException ex) { + dataPrepperServer.stop(); + //there should not be any Prometheus endpoints available + assertThat(ex.getMessage(), Matchers.is("Connection refused")); + } + } + + @Test + public void testMultipleMeterRegistries() throws Exception { + //for system metrics + final CloudWatchMeterRegistry cloudWatchMeterRegistryForSystem = mock(CloudWatchMeterRegistry.class); + final PrometheusMeterRegistry prometheusMeterRegistryForSystem = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); + compositeMeterRegistry.add(cloudWatchMeterRegistryForSystem); + compositeMeterRegistry.add(prometheusMeterRegistryForSystem); + + //for data prepper metrics + final PrometheusMeterRegistry prometheusMeterRegistryForDataPrepper = + new PrometheusRegistryMockScrape(PrometheusConfig.DEFAULT, UUID.randomUUID().toString()); + setRegistry(prometheusMeterRegistryForDataPrepper); + + final DataPrepperConfiguration dataPrepperConfiguration = DataPrepperConfiguration.fromFile( + new File(VALID_DATA_PREPPER_MULTIPLE_METRICS_CONFIG_FILE)); + setupDataPrepper(); + try (final MockedStatic dataPrepperMockedStatic = Mockito.mockStatic(DataPrepper.class)) { + dataPrepperMockedStatic.when(DataPrepper::getConfiguration).thenReturn(dataPrepperConfiguration); + dataPrepperMockedStatic.when(DataPrepper::getSystemMeterRegistry).thenReturn(compositeMeterRegistry); + dataPrepperServer = new DataPrepperServer(dataPrepper); + dataPrepperServer.start(); + + //test prometheus registry + HttpRequest request = HttpRequest.newBuilder(new URI("http://127.0.0.1:" + dataPrepperConfiguration.getServerPort() + + "/metrics/sys")).GET().build(); + HttpResponse response = HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + dataPrepperServer.stop(); + } + } + @Test public void testListPipelines() throws URISyntaxException, IOException, InterruptedException { setupDataPrepper(); diff --git a/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml b/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml new file mode 100644 index 0000000000..91a416eaaa --- /dev/null +++ b/data-prepper-core/src/test/resources/valid_data_prepper_cloudwatch_metrics_config.yml @@ -0,0 +1,2 @@ +ssl: false +metricRegistries: [CloudWatch] \ No newline at end of file diff --git a/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml b/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml new file mode 100644 index 0000000000..cea3343be8 --- /dev/null +++ b/data-prepper-core/src/test/resources/valid_data_prepper_multiple_metrics_config.yml @@ -0,0 +1,2 @@ +ssl: false +metricRegistries: [Prometheus, CloudWatch] \ No newline at end of file diff --git a/docs/readme/configuration.md b/docs/readme/configuration.md index 7664cfff6c..27393815bd 100644 --- a/docs/readme/configuration.md +++ b/docs/readme/configuration.md @@ -59,6 +59,7 @@ Data Prepper allows the following properties to be configured: * `keyStorePassword` string password for keystore. Optional, defaults to empty string * `privateKeyPassword` string password for private key within keystore. Optional, defaults to empty string * `serverPort`: integer port number to use for server APIs. Defaults to `4900` +* `metricRegistries`: list of metrics registries for publishing the generated metrics. Defaults to Prometheus; Prometheus and CloudWatch are currently supported. Example Data Prepper configuration file (data-prepper-config.yaml): ```yaml @@ -67,4 +68,5 @@ keyStoreFilePath: "/usr/share/data-prepper/keystore.jks" keyStorePassword: "password" privateKeyPassword: "other_password" serverPort: 1234 +metricRegistries: [Prometheus] ``` diff --git a/docs/readme/monitoring.md b/docs/readme/monitoring.md index 1c51882118..e5f30fb958 100644 --- a/docs/readme/monitoring.md +++ b/docs/readme/monitoring.md @@ -1,6 +1,6 @@ # Monitoring Metrics in Data Prepper are instrumented using [Micrometer.io](https://micrometer.io/). There are two types of metrics: -(1) JVM and system metrics; (2) Plugin metrics. Prometheus is used as the metrics backend. +(1) JVM and system metrics; (2) Plugin metrics. Prometheus is used as the default metrics backend. ## JVM and system metrics @@ -13,9 +13,10 @@ JVM and system metrics in Data Prepper follows pre-defined names in Micrometer.i ### Serving -Metrics are served from the **/metrics/sys** endpoint on the Data Prepper server. The format +By default, metrics are served from the **/metrics/sys** endpoint on the Data Prepper server. The format is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics, e.g. -[Grafana](https://prometheus.io/docs/visualization/grafana/). +[Grafana](https://prometheus.io/docs/visualization/grafana/). The configuration can be updated to serve metrics to other +registries like CloudWatch which does not require/host the endpoint but publishes the metrics directly to cloudwatch. ## Plugin metrics @@ -51,5 +52,7 @@ Metrics follow a naming convention of **PIPELINE_NAME_PLUGIN_NAME_METRIC_NAME** would have a qualified name of **output-pipeline_elasticsearch_sink_recordsIn**. ### Serving -Metrics are served from the **metrics/prometheus** endpoint on the Data Prepper server. The format -is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics. \ No newline at end of file +By default, metrics are served from the **metrics/prometheus** endpoint on the Data Prepper server. The format +is a text Prometheus scrape. This port can be used for any frontend which accepts Prometheus metrics. The configuration +can be updated to serve metrics to other registries like CloudWatch which does not require/host the endpoint but +publishes the metrics directly to cloudwatch. \ No newline at end of file diff --git a/docs/readme/project_setup.md b/docs/readme/project_setup.md index 4faeb18ded..d7de5bf99c 100644 --- a/docs/readme/project_setup.md +++ b/docs/readme/project_setup.md @@ -35,9 +35,13 @@ APIs are available: * /shutdown * starts a graceful shutdown of the Data Prepper * /metrics/prometheus - * returns a scrape of the Data Prepper metrics in Prometheus text format + * returns a scrape of the Data Prepper metrics in Prometheus text format. This API is available provided + `metricsRegistries` parameter in data prepper configuration file `data-prepper-config.yaml` has `Prometheus` as one + of the registry * /metrics/sys - * returns JVM metrics in Prometheus text format + * returns JVM metrics in Prometheus text format. This API is available provided `metricsRegistries` parameter in data + prepper configuration file `data-prepper-config.yaml` has `Prometheus` as one of the registry + ### Running the example app To run the example app against your local changes, use the docker found [here](https://github.com/opendistro-for-elasticsearch/data-prepper/tree/master/examples/dev/trace-analytics-sample-app) From 67ff400295a33564dec437b63ea4e848553f880f Mon Sep 17 00:00:00 2001 From: Bala Yadav Date: Fri, 14 May 2021 13:40:51 -0500 Subject: [PATCH 030/192] update version Signed-off-by: Bala Yadav --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f4e31e520b..40c2e91452 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ allprojects { apply plugin: 'com.diffplug.spotless' group = 'com.amazon' - version = '0.8.0-beta' + version = '1.0.0' repositories { mavenCentral() maven { url "https://jitpack.io" } From 03bc819fb28a5f5d28859c1df64772ced17f945b Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Fri, 14 May 2021 15:05:58 -0500 Subject: [PATCH 031/192] Maintenance: report back failure in CFN if exit code found (#587) * MAINT: report back failure if exception found in data-prepper.log Signed-off-by: qchea * MAINT: remove unnecessary conversion Signed-off-by: qchea * ENH: use exit code capture Signed-off-by: qchea --- .../ec2/data-prepper-ec2-deployment-cfn.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index f44fd91e12..cd69183c34 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -188,10 +188,19 @@ Resources: export RELEASE=opendistroforelasticsearch-data-prepper-${DataPrepperVersion}-linux-x64 yum install java-11-amazon-corretto-headless -y wget https://github.com/opendistro-for-elasticsearch/data-prepper/releases/download/v${DataPrepperVersion}/$RELEASE.tar.gz -O /tmp/$RELEASE.tar.gz - tar -xzf /tmp/$RELEASE.tar.gz --directory /usr/local/bin - /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default - nohup /usr/local/bin/$RELEASE/data-prepper-tar-install.sh /etc/data-prepper/pipelines.yaml /etc/data-prepper/data-prepper-config.yaml > /var/log/data-prepper.out & - /opt/aws/bin/cfn-signal --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + wget_exit_code = $? + if [ $wget_exit_code -ne 0 ] + then + /opt/aws/bin/cfn-signal -e $wget_exit_code --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + else + tar -xzf /tmp/$RELEASE.tar.gz --directory /usr/local/bin + /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default + nohup /usr/local/bin/$RELEASE/data-prepper-tar-install.sh /etc/data-prepper/pipelines.yaml /etc/data-prepper/data-prepper-config.yaml > /var/log/data-prepper.out & + data_prepper_pid=$! + sleep 5s + ps -p $data_prepper_pid + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} + fi CreationPolicy: ResourceSignal: Count: 1 From c55c757c997e204030e750fdc568313c0aa00bda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:15:14 +0000 Subject: [PATCH 032/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-raw-prepper Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 37d28b8209..a5defd75c8 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation "org.mockito:mockito-inline:3.10.0" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.awaitility:awaitility:4.1.0" } From b58f48161b6a8334a6c7eb3ed6ea79837cc5fe0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:16:11 +0000 Subject: [PATCH 033/192] Bump mockito-inline in /data-prepper-plugins/peer-forwarder Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 59c78f335b..6b316b4f09 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation "org.mockito:mockito-inline:3.10.0" } jacocoTestCoverageVerification { From 6b3d0dc1c0a23d2e1faa08cbd78c9b433f0557ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:16:28 +0000 Subject: [PATCH 034/192] Bump mockito-core in /data-prepper-plugins/otel-trace-source Bumps [mockito-core](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index c63aa536b9..00a45fe3b2 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -19,7 +19,7 @@ dependencies { testImplementation(platform('org.junit:junit-bom:5.7.1')) testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects - testImplementation("org.mockito:mockito-core:3.9.0") + testImplementation("org.mockito:mockito-core:3.10.0") testImplementation("org.mockito:mockito-junit-jupiter:3.9.0") } From 65a4996f9a19345540a4bc002e754e827d15fdbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:16:46 +0000 Subject: [PATCH 035/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-source Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index c63aa536b9..d60fc3dbc6 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation "org.mockito:mockito-inline:3.10.0" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation(platform('org.junit:junit-bom:5.7.1')) testImplementation('org.junit.jupiter:junit-jupiter') From 3c3fe92a4280f4dd298ea6f782614d94419f1cc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:17:08 +0000 Subject: [PATCH 036/192] Bump mockito-inline in /data-prepper-plugins/service-map-stateful Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index f6e8ebcbe9..058bb19809 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-inline:3.9.0" + testImplementation "org.mockito:mockito-inline:3.10.0" } jacocoTestCoverageVerification { From 45ec65d7e08a2279e2c79786db613255aa06700c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 May 2021 06:31:31 +0000 Subject: [PATCH 037/192] Bump mockito-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [mockito-core](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index 59e0681348..7639d8cfbc 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -14,6 +14,6 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "io.micrometer:micrometer-core:1.6.6" - testImplementation "org.mockito:mockito-core:3.9.0" + testImplementation "org.mockito:mockito-core:3.10.0" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From ba270f5c26064641c672c764d4b66289001572d8 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Mon, 17 May 2021 18:23:53 +0000 Subject: [PATCH 038/192] Added version to index template schemas. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../model/DataPrepperConfiguration.java | 2 +- .../sink/elasticsearch/ElasticsearchSink.java | 79 ++++++++-- ...tel-v1-apm-service-map-index-template.json | 1 + .../otel-v1-apm-span-index-template.json | 1 + .../elasticsearch/ElasticsearchSinkIT.java | 136 +++++++++++++----- .../resources/test-index-template-v2.json | 26 ++++ .../test/resources/test-index-template.json | 1 + 7 files changed, 198 insertions(+), 48 deletions(-) create mode 100644 data-prepper-plugins/elasticsearch/src/test/resources/test-index-template-v2.json diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java index 4530f3e321..8242619618 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/DataPrepperConfiguration.java @@ -35,7 +35,7 @@ public static DataPrepperConfiguration fromFile(File file) { try { return OBJECT_MAPPER.readValue(file, DataPrepperConfiguration.class); } catch (IOException e) { - throw new IllegalArgumentException("Invalid DataPrepper configuration file."); + throw new IllegalArgumentException("Invalid DataPrepper configuration file.", e); } } diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java index 2c24ae41a2..2e1f2f1ac9 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java @@ -17,6 +17,10 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; +import org.elasticsearch.client.indices.IndexTemplateMetadata; +import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -79,12 +83,13 @@ public void start() throws IOException { LOG.info("Starting Elasticsearch sink"); restHighLevelClient = esSinkConfig.getConnectionConfiguration().createClient(); final boolean isISMEnabled = IndexStateManagement.checkISMEnabled(restHighLevelClient); - final Optional policyIdOptional = isISMEnabled? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : Optional.empty(); + final Optional policyIdOptional = isISMEnabled ? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : + Optional.empty(); if (!esSinkConfig.getIndexConfiguration().getIndexTemplate().isEmpty()) { - createIndexTemplate(isISMEnabled, policyIdOptional.orElse(null)); + checkAndCreateIndexTemplate(isISMEnabled, policyIdOptional.orElse(null)); } final String dlqFile = esSinkConfig.getRetryConfiguration().getDlqFile(); - if ( dlqFile != null) { + if (dlqFile != null) { dlqWriter = Files.newBufferedWriter(Paths.get(dlqFile), StandardOpenOption.CREATE, StandardOpenOption.APPEND); } checkAndCreateIndex(); @@ -103,7 +108,7 @@ public void doOutput(final Collection> records) { return; } BulkRequest bulkRequest = bulkRequestSupplier.get(); - for (final Record record: records) { + for (final Record record : records) { final String document = record.getData(); final IndexRequest indexRequest = new IndexRequest().source(document, XContentType.JSON); try { @@ -146,9 +151,16 @@ private void flushBatch(final BulkRequest bulkRequest) { }); } - private void createIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { + private void checkAndCreateIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException { final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias(); - final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexAlias + "-index-template"); + final String indexTemplateName = indexAlias + "-index-template"; + + // Check existing index template version - only overwrite if version is less than or does not exist + if (!shouldCreateIndexTemplate(indexTemplateName)) { + return; + } + + final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexTemplateName); final boolean isRaw = indexType.equals(IndexConstants.RAW); if (isRaw) { putIndexTemplateRequest.patterns(Collections.singletonList(indexAlias + "-*")); @@ -158,16 +170,65 @@ private void createIndexTemplate(final boolean isISMEnabled, final String ismPol if (isISMEnabled) { IndexStateManagement.attachPolicy(esSinkConfig.getIndexConfiguration(), ismPolicyId, indexAlias); } + putIndexTemplateRequest.source(esSinkConfig.getIndexConfiguration().getIndexTemplate()); restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); } + // TODO: Unit tests for this (and for the rest of the class) + private boolean shouldCreateIndexTemplate(final String indexTemplateName) throws IOException { + final Optional indexTemplateMetadataOptional = getIndexTemplateMetadata(indexTemplateName); + if (indexTemplateMetadataOptional.isPresent()) { + final Integer existingTemplateVersion = indexTemplateMetadataOptional.get().version(); + LOG.info("Found version {} for existing index template {}", existingTemplateVersion, indexTemplateName); + + final int newTemplateVersion = (int) esSinkConfig.getIndexConfiguration().getIndexTemplate().getOrDefault("version", 0); + + if (existingTemplateVersion != null && existingTemplateVersion >= newTemplateVersion) { + LOG.info("Index template {} should not be updated, current version {} >= existing version {}", + indexTemplateName, + existingTemplateVersion, + newTemplateVersion); + return false; + + } else { + LOG.info("Index template {} should be updated from version {} to version {}", + indexTemplateName, + existingTemplateVersion, + newTemplateVersion); + return true; + } + } else { + LOG.info("Index template {} does not exist and should be created", indexTemplateName); + return true; + } + } + + private Optional getIndexTemplateMetadata(final String indexTemplateName) throws IOException { + final IndexTemplatesExistRequest existsRequest = new IndexTemplatesExistRequest(indexTemplateName); + final boolean exists = restHighLevelClient.indices().existsTemplate(existsRequest, RequestOptions.DEFAULT); + if (!exists) { + return Optional.empty(); + } + + final GetIndexTemplatesRequest request = new GetIndexTemplatesRequest(indexTemplateName); + final GetIndexTemplatesResponse response = restHighLevelClient.indices().getIndexTemplate(request, RequestOptions.DEFAULT); + + if (response.getIndexTemplates().size() == 1) { + return Optional.of(response.getIndexTemplates().get(0)); + } else { + throw new RuntimeException(String.format("Found multiple index templates (%s) result when querying for %s", + response.getIndexTemplates().size(), + indexTemplateName)); + } + } + private void checkAndCreateIndex() throws IOException { // Check alias exists final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias(); final boolean isRaw = indexType.equals(IndexConstants.RAW); - final boolean exists = isRaw? - restHighLevelClient.indices().existsAlias(new GetAliasesRequest().aliases(indexAlias), RequestOptions.DEFAULT): + final boolean exists = isRaw ? + restHighLevelClient.indices().existsAlias(new GetAliasesRequest().aliases(indexAlias), RequestOptions.DEFAULT) : restHighLevelClient.indices().exists(new GetIndexRequest(indexAlias), RequestOptions.DEFAULT); if (!exists) { // TODO: use date as suffix? @@ -210,7 +271,7 @@ private void logFailure(final DocWriteRequest docWriteRequest, final Throwabl } } else { LOG.warn("Document [{}] has failure: {}", docWriteRequest.toString(), failure); - }; + } } @Override diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-service-map-index-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-service-map-index-template.json index 399263ce28..99aa8ea796 100644 --- a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-service-map-index-template.json +++ b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-service-map-index-template.json @@ -1,4 +1,5 @@ { + "version": 0, "mappings": { "date_detection": false, "dynamic_templates": [ diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json index 25df9ff8e7..f7bc7b8727 100644 --- a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json +++ b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json @@ -1,4 +1,5 @@ { + "version": 0, "mappings": { "date_detection": false, "dynamic_templates": [ diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java index 38056c0fdd..4666e42b50 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java @@ -61,8 +61,9 @@ public class ElasticsearchSinkIT extends ESRestTestCase { private static final String PLUGIN_NAME = "elasticsearch"; private static final String PIPELINE_NAME = "integTestPipeline"; public List HOSTS = Arrays.stream(System.getProperty("tests.rest.cluster").split(",")) - .map(ip -> String.format("%s://%s", getProtocol(), ip)).collect(Collectors.toList()); - private static final String DEFAULT_TEMPLATE_FILE = "test-index-template.json"; + .map(ip -> String.format("%s://%s", getProtocol(), ip)).collect(Collectors.toList()); + private static final String TEST_TEMPLATE_V1_FILE = "test-index-template.json"; + private static final String TEST_TEMPLATE_V2_FILE = "test-index-template-v2.json"; private static final String DEFAULT_RAW_SPAN_FILE_1 = "raw-span-1.json"; private static final String DEFAULT_RAW_SPAN_FILE_2 = "raw-span-2.json"; private static final String DEFAULT_SERVICE_MAP_FILE = "service-map-1.json"; @@ -82,13 +83,14 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { final String index = String.format("%s-000001", indexAlias); final Map mappings = getIndexMappings(index); assertNotNull(mappings); - assertFalse((boolean)mappings.get("date_detection")); + assertFalse((boolean) mappings.get("date_detection")); sink.shutdown(); if (isODFE()) { // Check managed index await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { - assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index)); } + assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index)); + } ); } @@ -129,7 +131,7 @@ public void testOutputRawSpanDefault() throws IOException, InterruptedException final List> retSources = getSearchResponseDocSources(expIndexAlias); assertEquals(2, retSources.size()); assertTrue(retSources.containsAll(Arrays.asList(expData1, expData2))); - assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String)expData1.get("spanId"))); + assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String) expData1.get("spanId"))); sink.shutdown(); // Verify metrics @@ -216,7 +218,7 @@ public void testInstantiateSinkServiceMapDefault() throws IOException { assertEquals(SC_OK, response.getStatusLine().getStatusCode()); final Map mappings = getIndexMappings(indexAlias); assertNotNull(mappings); - assertFalse((boolean)mappings.get("date_detection")); + assertFalse((boolean) mappings.get("date_detection")); sink.shutdown(); if (isODFE()) { @@ -228,8 +230,7 @@ public void testInstantiateSinkServiceMapDefault() throws IOException { public void testOutputServiceMapDefault() throws IOException, InterruptedException { final String testDoc = readDocFromFile(DEFAULT_SERVICE_MAP_FILE); final ObjectMapper mapper = new ObjectMapper(); - @SuppressWarnings("unchecked") - final Map expData = mapper.readValue(testDoc, Map.class); + @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc, Map.class); final List> testRecords = Collections.singletonList(new Record<>(testDoc)); final PluginSetting pluginSetting = generatePluginSetting(false, true, null, null); @@ -239,7 +240,7 @@ public void testOutputServiceMapDefault() throws IOException, InterruptedExcepti final List> retSources = getSearchResponseDocSources(expIndexAlias); assertEquals(1, retSources.size()); assertEquals(expData, retSources.get(0)); - assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String)expData.get("hashId"))); + assertEquals(Integer.valueOf(1), getDocumentCount(expIndexAlias, "_id", (String) expData.get("hashId"))); sink.shutdown(); // verify metrics @@ -258,7 +259,7 @@ public void testOutputServiceMapDefault() throws IOException, InterruptedExcepti public void testInstantiateSinkCustomIndex() throws IOException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( - getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); + getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile); ElasticsearchSink sink = new ElasticsearchSink(pluginSetting); final Request request = new Request(HttpMethod.HEAD, testIndexAlias); @@ -271,10 +272,67 @@ public void testInstantiateSinkCustomIndex() throws IOException { sink.shutdown(); } + public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates() throws IOException { + final String testIndexAlias = "test-alias"; + final String expectedIndexTemplateName = testIndexAlias + "-index-template"; + final String testTemplateFileV1 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE).getFile(); + final String testTemplateFileV2 = getClass().getClassLoader().getResource(TEST_TEMPLATE_V2_FILE).getFile(); + + // Create sink with template version 1 + PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1); + ElasticsearchSink sink = new ElasticsearchSink(pluginSetting); + + Request getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + Response getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + String responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer firstResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + assertEquals(Integer.valueOf(1), firstResponseVersion); + sink.shutdown(); + + // Create sink with template version 2 + pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV2); + sink = new ElasticsearchSink(pluginSetting); + + getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer secondResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + assertEquals(Integer.valueOf(2), secondResponseVersion); + sink.shutdown(); + + // Create sink with template version 1 again + pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFileV1); + sink = new ElasticsearchSink(pluginSetting); + + getTemplateRequest = new Request(HttpMethod.GET, "/_template/" + expectedIndexTemplateName); + getTemplateResponse = client().performRequest(getTemplateRequest); + assertEquals(SC_OK, getTemplateResponse.getStatusLine().getStatusCode()); + + responseBody = EntityUtils.toString(getTemplateResponse.getEntity()); + @SuppressWarnings("unchecked") final Integer thirdResponseVersion = + (Integer) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(expectedIndexTemplateName)).get("version"); + + // Assert version 2 was not overwritten by version 1 + assertEquals(Integer.valueOf(2), thirdResponseVersion); + sink.shutdown(); + + } + public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( - getClass().getClassLoader().getResource(DEFAULT_TEMPLATE_FILE)).getFile(); + getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; final String testId = "foo"; final List> testRecords = Collections.singletonList(generateCustomRecord(testIdField, testId)); @@ -296,7 +354,8 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0); } - private PluginSetting generatePluginSetting(final boolean isRaw, final boolean isServiceMap, final String indexAlias, final String templateFilePath) { + private PluginSetting generatePluginSetting(final boolean isRaw, final boolean isServiceMap, final String indexAlias, + final String templateFilePath) { final Map metadata = new HashMap<>(); metadata.put(IndexConfiguration.TRACE_ANALYTICS_RAW_FLAG, isRaw); metadata.put(IndexConfiguration.TRACE_ANALYTICS_SERVICE_MAP_FLAG, isServiceMap); @@ -317,19 +376,19 @@ private PluginSetting generatePluginSetting(final boolean isRaw, final boolean i private Record generateCustomRecord(final String idField, final String documentId) throws IOException { return new Record<>( - Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .field(idField, documentId) - .endObject() - ) + Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .field(idField, documentId) + .endObject() + ) ); } private String readDocFromFile(final String filename) throws IOException { final StringBuilder jsonBuilder = new StringBuilder(); try (final InputStream inputStream = Objects.requireNonNull( - getClass().getClassLoader().getResourceAsStream(filename))){ + getClass().getClassLoader().getResourceAsStream(filename))) { final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); bufferedReader.lines().forEach(jsonBuilder::append); } @@ -337,9 +396,10 @@ private String readDocFromFile(final String filename) throws IOException { } private Boolean checkIsWriteIndex(final String responseBody, final String aliasName, final String indexName) throws IOException { - @SuppressWarnings("unchecked") final Map indexBlob = (Map)createParser(XContentType.JSON.xContent(), responseBody).map().get(indexName); - @SuppressWarnings("unchecked") final Map aliasesBlob = (Map)indexBlob.get("aliases"); - @SuppressWarnings("unchecked") final Map aliasBlob = (Map)aliasesBlob.get(aliasName); + @SuppressWarnings("unchecked") final Map indexBlob = (Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(indexName); + @SuppressWarnings("unchecked") final Map aliasesBlob = (Map) indexBlob.get("aliases"); + @SuppressWarnings("unchecked") final Map aliasBlob = (Map) aliasesBlob.get(aliasName); return (Boolean) aliasBlob.get("is_write_index"); } @@ -347,19 +407,19 @@ private Integer getDocumentCount(final String index, final String field, final S final Request request = new Request(HttpMethod.GET, index + "/_count"); if (field != null && value != null) { final String jsonEntity = Strings.toString( - XContentFactory.jsonBuilder().startObject() - .startObject("query") - .startObject("match") - .field(field, value) - .endObject() - .endObject() - .endObject() + XContentFactory.jsonBuilder().startObject() + .startObject("query") + .startObject("match") + .field(field, value) + .endObject() + .endObject() + .endObject() ); request.setJsonEntity(jsonEntity); } final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - return (Integer)createParser(XContentType.JSON.xContent(), responseBody).map().get("count"); + return (Integer) createParser(XContentType.JSON.xContent(), responseBody).map().get("count"); } private List> getSearchResponseDocSources(final String index) throws IOException { @@ -369,10 +429,11 @@ private List> getSearchResponseDocSources(final String index final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") final List hits = (List) ((Map)createParser(XContentType.JSON.xContent(), - responseBody).map().get("hits")).get("hits"); + @SuppressWarnings("unchecked") final List hits = + (List) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get("hits")).get("hits"); @SuppressWarnings("unchecked") final List> sources = hits.stream() - .map(hit -> (Map)((Map) hit).get("_source")) + .map(hit -> (Map) ((Map) hit).get("_source")) .collect(Collectors.toList()); return sources; } @@ -382,9 +443,9 @@ private Map getIndexMappings(final String index) throws IOExcept final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") - final Map mappings = (Map) ((Map)createParser(XContentType.JSON.xContent(), - responseBody).map().get(index)).get("mappings"); + @SuppressWarnings("unchecked") final Map mappings = + (Map) ((Map) createParser(XContentType.JSON.xContent(), + responseBody).map().get(index)).get("mappings"); return mappings; } @@ -393,8 +454,7 @@ private String getIndexPolicyId(final String index) throws IOException { final Response response = client().performRequest(request); final String responseBody = EntityUtils.toString(response.getEntity()); - @SuppressWarnings("unchecked") - final String policyId = (String) ((Map)createParser(XContentType.JSON.xContent(), + @SuppressWarnings("unchecked") final String policyId = (String) ((Map) createParser(XContentType.JSON.xContent(), responseBody).map().get(index)).get("index.opendistro.index_state_management.policy_id"); return policyId; } diff --git a/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template-v2.json b/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template-v2.json new file mode 100644 index 0000000000..7282a7cead --- /dev/null +++ b/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template-v2.json @@ -0,0 +1,26 @@ +{ + "version": 2, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "_source": { + "enabled": false + }, + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template.json b/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template.json index 16bb72d011..6b942a8ec2 100644 --- a/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template.json +++ b/data-prepper-plugins/elasticsearch/src/test/resources/test-index-template.json @@ -1,4 +1,5 @@ { + "version": 1, "mappings": { "date_detection": false, "dynamic_templates": [ From 71dbe987187b3496ec0daac8a4902de74b87a3cd Mon Sep 17 00:00:00 2001 From: Bala Yadav <55510769+yadavcbala@users.noreply.github.com> Date: Mon, 17 May 2021 17:24:01 -0500 Subject: [PATCH 039/192] Add capability to configure unframed requets from otel trace source configuration (#607) * Add capability to configure unframed requests from source configuration * use underscore over dash Signed-off-by: Bala Yadav * update README Signed-off-by: Bala Yadav --- data-prepper-plugins/otel-trace-source/README.md | 1 + .../plugins/source/oteltrace/OTelTraceSource.java | 2 ++ .../plugins/source/oteltrace/OTelTraceSourceConfig.java | 9 +++++++++ .../source/oteltrace/OtelTraceSourceConfigTests.java | 7 +++++++ 4 files changed, 19 insertions(+) diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index b09de5ac79..d73b0155f1 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -16,6 +16,7 @@ source: * request_timeout(Optional) => An `int` represents request timeout in millis. Default is ```10_000```. * health_check_service(Optional) => A boolean enables a gRPC health check service under ```grpc.health.v1 / Health / Check```. Default is ```false```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. +* unframed_requests(Optional) => A boolean to enable requests not framed using the gRPC wire protocol. * ssl(Optional) => A boolean enables TLS/SSL. Default is ```true```. * sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path. Required if ```ssl``` is set to ```true``` * sslKeyFile(Optional) => A `String` represents the SSL key file path. Required if ```ssl``` is set to ```true``` diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 81b3d8cd46..9199ee06cf 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -59,6 +59,8 @@ public void start(Buffer> buffer) { LOG.info("Proto reflection service is enabled"); grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } + + grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); final ServerBuilder sb = Server.builder(); sb.service(grpcServiceBuilder.build()); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index 1c7a126916..1e0b5ad6b4 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -12,6 +12,7 @@ public class OTelTraceSourceConfig { static final String SSL_KEY_FILE = "sslKeyFile"; static final String THREAD_COUNT = "thread_count"; static final String MAX_CONNECTION_COUNT = "max_connection_count"; + static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21890; static final int DEFAULT_THREAD_COUNT = 200; @@ -21,6 +22,7 @@ public class OTelTraceSourceConfig { private final int port; private final boolean healthCheck; private final boolean protoReflectionService; + private final boolean enableUnframedRequests; private final boolean ssl; private final String sslKeyCertChainFile; private final String sslKeyFile; @@ -31,6 +33,7 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, final int port, final boolean healthCheck, final boolean protoReflectionService, + final boolean enableUnframedRequests, final boolean isSSL, final String sslKeyCertChainFile, final String sslKeyFile, @@ -40,6 +43,7 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, this.port = port; this.healthCheck = healthCheck; this.protoReflectionService = protoReflectionService; + this.enableUnframedRequests = enableUnframedRequests; this.ssl = isSSL; this.sslKeyCertChainFile = sslKeyCertChainFile; this.sslKeyFile = sslKeyFile; @@ -58,6 +62,7 @@ public static OTelTraceSourceConfig buildConfig(final PluginSetting pluginSettin pluginSetting.getIntegerOrDefault(PORT, DEFAULT_PORT), pluginSetting.getBooleanOrDefault(HEALTH_CHECK_SERVICE, false), pluginSetting.getBooleanOrDefault(PROTO_REFLECTION_SERVICE, false), + pluginSetting.getBooleanOrDefault(ENABLE_UNFRAMED_REQUESTS, false), pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL), pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null), pluginSetting.getStringOrDefault(SSL_KEY_FILE, null), @@ -81,6 +86,10 @@ public boolean hasProtoReflectionService() { return protoReflectionService; } + public boolean enableUnframedRequests() { + return enableUnframedRequests; + } + public boolean isSsl() { return ssl; } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index a974a0eda5..206625a086 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -53,6 +53,7 @@ public void testValidConfig() { TEST_PORT, true, true, + false, true, TEST_KEY_CERT, TEST_KEY, @@ -81,6 +82,7 @@ public void testInvalidConfig() { DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_PORT, false, false, + false, true, null, TEST_KEY, DEFAULT_THREAD_COUNT, @@ -94,6 +96,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, "", TEST_KEY, @@ -108,6 +111,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, TEST_KEY_CERT, null, @@ -122,6 +126,7 @@ public void testInvalidConfig() { DEFAULT_PORT, false, false, + false, true, TEST_KEY_CERT, "", @@ -135,6 +140,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT final int port, final boolean healthCheck, final boolean protoReflectionService, + final boolean enableUnframedRequests, final boolean isSSL, final String sslKeyCertChainFile, final String sslKeyFile, @@ -145,6 +151,7 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.PORT, port); settings.put(OTelTraceSourceConfig.HEALTH_CHECK_SERVICE, healthCheck); settings.put(OTelTraceSourceConfig.PROTO_REFLECTION_SERVICE, protoReflectionService); + settings.put(OTelTraceSourceConfig.ENABLE_UNFRAMED_REQUESTS, enableUnframedRequests); settings.put(OTelTraceSourceConfig.SSL, isSSL); settings.put(OTelTraceSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); From 636c5cc0f23c6dcb316c0c491dc13e8e971e0fd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 14:52:51 +0000 Subject: [PATCH 040/192] Bump mockito-core from 3.9.0 to 3.10.0 in /data-prepper-core Bumps [mockito-core](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index a5e03a5500..08226d4987 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation "io.micrometer:micrometer-registry-cloudwatch2:1.6.6" implementation "software.amazon.awssdk:cloudwatch:2.16.59" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-core:3.9.0" + testImplementation "org.mockito:mockito-core:3.10.0" } jar { From 1261b634c0326ae35bb466ccc0230f5bafe62662 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:04:26 +0000 Subject: [PATCH 041/192] Bump micrometer-core in /data-prepper-plugins/service-map-stateful Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.6.5 to 1.7.0. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.6.5...v1.7.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index f6e8ebcbe9..82e95a163d 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -16,7 +16,7 @@ dependencies { compile project(':data-prepper-plugins:common') compile project(':data-prepper-plugins:mapdb-prepper-state') testCompile project(':data-prepper-api').sourceSets.test.output - implementation "io.micrometer:micrometer-core:1.6.5" + implementation "io.micrometer:micrometer-core:1.7.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" From 0204399ff313dd5837faf1c21db49c9f2e53ec60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:04:28 +0000 Subject: [PATCH 042/192] Bump micrometer-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.6.6 to 1.7.0. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.6.6...v1.7.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index 7639d8cfbc..ff11cf6b94 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - implementation "io.micrometer:micrometer-core:1.6.6" + implementation "io.micrometer:micrometer-core:1.7.0" testImplementation "org.mockito:mockito-core:3.10.0" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From 51c5b9426a8fc659955cb55c78183b03df6b54b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:04:28 +0000 Subject: [PATCH 043/192] Bump micrometer-registry-cloudwatch2 in /data-prepper-core Bumps [micrometer-registry-cloudwatch2](https://github.com/micrometer-metrics/micrometer) from 1.6.6 to 1.7.0. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.6.6...v1.7.0) Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 08226d4987..9f144ce50e 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "org.reflections:reflections:0.9.12" implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" - implementation "io.micrometer:micrometer-registry-cloudwatch2:1.6.6" + implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" implementation "software.amazon.awssdk:cloudwatch:2.16.59" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.10.0" From a963c7d91e5ad09e37de70b3bd805685794fcdaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:04:29 +0000 Subject: [PATCH 044/192] Bump mockito-junit-jupiter in /data-prepper-plugins/otel-trace-source Bumps [mockito-junit-jupiter](https://github.com/mockito/mockito) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.9.0...v3.10.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 00a45fe3b2..9dc3b5992e 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -20,7 +20,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects testImplementation("org.mockito:mockito-core:3.10.0") - testImplementation("org.mockito:mockito-junit-jupiter:3.9.0") + testImplementation("org.mockito:mockito-junit-jupiter:3.10.0") } test { From 442d3388c6aa342e1d8fbe7fda10a5ecceffd448 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:13:32 +0000 Subject: [PATCH 045/192] Bump micrometer-core from 1.6.5 to 1.7.0 in /data-prepper-api Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.6.5 to 1.7.0. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.6.5...v1.7.0) Signed-off-by: dependabot[bot] --- data-prepper-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 3086822f43..20b3c6b42d 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -2,7 +2,7 @@ plugins { } dependencies { - implementation "io.micrometer:micrometer-core:1.6.5" + implementation "io.micrometer:micrometer-core:1.7.0" testImplementation "org.hamcrest:hamcrest:2.2" } From 5080665e803ae235e76a4cd063aed9f3274f0984 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 18 May 2021 11:24:14 -0500 Subject: [PATCH 046/192] Manually update to micrometer-core:1.7.0 --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index ef20cd6177..576e9f8985 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation "com.amazonaws:aws-java-sdk-core:1.11.1015" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" - implementation "io.micrometer:micrometer-core:1.6.5" + implementation "io.micrometer:micrometer-core:1.7.0" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } From 1a70e4f1dce8e8d1778e6889ec220c47ee58d718 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:25:05 +0000 Subject: [PATCH 047/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1015 to 1.11.1020. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1015...1.11.1020) Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 576e9f8985..7d6455a6a9 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1015" + implementation "com.amazonaws:aws-java-sdk-core:1.11.1020" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.0" testImplementation("junit:junit:4.13.2") { @@ -52,7 +52,7 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.11.1009' + force 'com.amazonaws:aws-java-sdk-core:1.11.1020' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' From 7e222103cd7b6ef87d99e01a5df326bb38d7b1df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:25:07 +0000 Subject: [PATCH 048/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-raw-prepper Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.16.0 to 3.17.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.16.0...v3.17.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index a5defd75c8..9b6f52d40d 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -8,7 +8,7 @@ dependencies { compile 'commons-codec:commons-codec:1.15' testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.16.0' + implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From 4a9f5d14edfa0537321eeaed5be959aa3cf3cc82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:25:10 +0000 Subject: [PATCH 049/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-source Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.16.0 to 3.17.0. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.16.0...v3.17.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index c83d231a4d..6a8c2f9a3c 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,7 +8,7 @@ dependencies { compile 'commons-codec:commons-codec:1.15' testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.16.0' + implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.7.2" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From 98075684942d8919601527bac985a6d974071a40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:25:14 +0000 Subject: [PATCH 050/192] Bump junit-bom in /data-prepper-plugins/otel-trace-source Bumps [junit-bom](https://github.com/junit-team/junit5) from 5.7.1 to 5.7.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.1...r5.7.2) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index c83d231a4d..0b2a740c90 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -16,7 +16,7 @@ dependencies { testImplementation 'org.assertj:assertj-core:3.19.0' testImplementation "org.mockito:mockito-inline:3.10.0" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation(platform('org.junit:junit-bom:5.7.1')) + testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects testImplementation("org.mockito:mockito-core:3.10.0") From fbc2c5c80a422d94d8c8d4d73bb51cf2799100f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 16:25:16 +0000 Subject: [PATCH 051/192] Bump cloudwatch from 2.16.59 to 2.16.64 in /data-prepper-core Bumps cloudwatch from 2.16.59 to 2.16.64. Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 9f144ce50e..6ba0071092 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" - implementation "software.amazon.awssdk:cloudwatch:2.16.59" + implementation "software.amazon.awssdk:cloudwatch:2.16.64" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.10.0" } From 6fbf401ad71a12d74333ebcb7af96c2d44350ac1 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 18 May 2021 11:40:32 -0500 Subject: [PATCH 052/192] Bug Fixes: capture invalid index alias exception and display instructive msg (#588) * MAINT: capture invalid raw span index alias due to index with the same name already exists Signed-off-by: qchea * FIX: place shutdown properly Signed-off-by: qchea * STY: better shutdown placement Signed-off-by: qchea * FIX: spotless Signed-off-by: qchea * MAINT: throw checked exception in start API Signed-off-by: qchea * MAINT: add TODO Signed-off-by: qchea * STY: rename and simplify Signed-off-by: qchea --- .../sink/elasticsearch/ElasticsearchSink.java | 17 ++++++++++++----- .../sink/elasticsearch/ElasticsearchSinkIT.java | 9 +++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java index 2e1f2f1ac9..67eed556e2 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java @@ -50,6 +50,7 @@ public class ElasticsearchSink extends AbstractSink> { private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchSink.class); // Pulled from BulkRequest to make estimation of bytes consistent private static final int REQUEST_OVERHEAD = 50; + protected static final String INDEX_ALIAS_USED_AS_INDEX_ERROR = "Invalid alias name [%s], an index exists with the same name as the alias"; private BufferedWriter dlqWriter; private final ElasticsearchSinkConfiguration esSinkConfig; @@ -73,14 +74,15 @@ public ElasticsearchSink(final PluginSetting pluginSetting) { this.indexType = esSinkConfig.getIndexConfiguration().getIndexType(); this.documentIdField = esSinkConfig.getIndexConfiguration().getDocumentIdField(); try { - start(); + initialize(); } catch (final IOException e) { + this.shutdown(); throw new RuntimeException(e.getMessage(), e); } } - public void start() throws IOException { - LOG.info("Starting Elasticsearch sink"); + public void initialize() throws IOException { + LOG.info("Initializing Elasticsearch sink"); restHighLevelClient = esSinkConfig.getConnectionConfiguration().createClient(); final boolean isISMEnabled = IndexStateManagement.checkISMEnabled(restHighLevelClient); final Optional policyIdOptional = isISMEnabled ? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : @@ -99,7 +101,7 @@ public void start() throws IOException { this::logFailure, pluginMetrics, bulkRequestSupplier); - LOG.info("Started Elasticsearch sink"); + LOG.info("Initialized Elasticsearch sink"); } @Override @@ -248,8 +250,13 @@ private void checkAndCreateIndex() throws IOException { if (e.getMessage().contains("resource_already_exists_exception")) { // Do nothing - likely caused by a race condition where the resource was created // by another host before this host's restClient made its request + } else if (e.getMessage().contains(String.format(INDEX_ALIAS_USED_AS_INDEX_ERROR, indexAlias))) { + // TODO: replace IOException with custom data-prepper exception + throw new IOException( + String.format("An index exists with the same name as the reserved index alias name [%s], please delete or migrate the existing index", + indexAlias)); } else { - throw e; + throw new IOException(e); } } } diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java index 4666e42b50..b188d42308 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java @@ -115,6 +115,15 @@ public void testInstantiateSinkRawSpanDefault() throws IOException { } } + public void testInstantiateSinkRawSpanReservedAliasAlreadyUsedAsIndex() throws IOException { + final String reservedIndexAlias = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexConstants.RAW); + final Request request = new Request(HttpMethod.PUT, reservedIndexAlias); + client().performRequest(request); + final PluginSetting pluginSetting = generatePluginSetting(true, false, null, null); + assertThrows(String.format(ElasticsearchSink.INDEX_ALIAS_USED_AS_INDEX_ERROR, reservedIndexAlias), + RuntimeException.class, () -> new ElasticsearchSink(pluginSetting)); + } + public void testOutputRawSpanDefault() throws IOException, InterruptedException { final String testDoc1 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1); final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_2); From 578cadf952b99e5907765faf7d4d3dc78ce12ba4 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 18 May 2021 15:09:56 -0500 Subject: [PATCH 053/192] Infrastructure: add compatibility e2e tests and separate github workflow (#609) * INFRA: add compatibility e2e tests and separate github workflow Signed-off-by: qchea * MAINT: missing changes Signed-off-by: qchea * MAINT: fix mounting volume; separate compatibility test infrastructure Signed-off-by: qchea * FIX: expose ports Signed-off-by: qchea * STY: better step name Signed-off-by: qchea * STY: better port name Signed-off-by: qchea * MAINT: change the test infra and logic for compatibility Signed-off-by: qchea * MAINT: extend root span flush delay so that peer forwarded requests arrive in time Signed-off-by: qchea * FIX: use same config file for compatibility Signed-off-by: qchea --- ...tics-raw-span-compatibility-e2e-tests.yml} | 8 +-- ...per-trace-analytics-raw-span-e2e-tests.yml | 29 +++++++++ ...-trace-analytics-service-map-e2e-tests.yml | 29 +++++++++ data-prepper-core/integrationTest.gradle | 62 ++++++++++++++++++- .../raw-span-e2e-pipeline-latest-release.yml | 26 ++++++++ 5 files changed, 147 insertions(+), 7 deletions(-) rename .github/workflows/{data-prepper-trace-analytics-e2e-tests.yml => data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml} (66%) create mode 100644 .github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml create mode 100644 .github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml create mode 100644 data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml diff --git a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml similarity index 66% rename from .github/workflows/data-prepper-trace-analytics-e2e-tests.yml rename to .github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index ea39bdc594..eada827ce2 100644 --- a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -1,7 +1,7 @@ # This workflow will build a Java project with Gradle # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle -name: Data Prepper Trace Analytics End-to-end test with Gradle +name: Data Prepper Trace Analytics Raw Span Compatibility End-to-end test with Gradle on: push: @@ -25,7 +25,5 @@ jobs: java-version: ${{ matrix.java }} - name: Grant execute permission for gradlew run: chmod +x gradlew - - name: Run raw-span end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:rawSpanEndToEndTest - - name: Run service-map end-to-end tests with Gradle - run: ./gradlew :data-prepper-core:serviceMapEndToEndTest + - name: Run raw-span compatibility end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:rawSpanCompatibilityEndToEndTest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml new file mode 100644 index 0000000000..1d80d7be23 --- /dev/null +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -0,0 +1,29 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper Trace Analytics Raw Span End-to-end test with Gradle + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run raw-span end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:rawSpanEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml new file mode 100644 index 0000000000..7868746e36 --- /dev/null +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -0,0 +1,29 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper Trace Analytics Service Map End-to-end test with Gradle + +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + java: [14] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Run service-map end-to-end tests with Gradle + run: ./gradlew :data-prepper-core:serviceMapEndToEndTest \ No newline at end of file diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index ca407a7604..40bfdbaebc 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -52,6 +52,7 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) { } def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" +def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" /** @@ -66,6 +67,7 @@ task createDataPrepperDockerFile(type: Dockerfile) { workingDir("/app") copyFile("build/libs/${jar.archiveName}", "/app/data-prepper.jar") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") + copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml") defaultCommand("java", "-jar", "data-prepper.jar", "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml") @@ -110,11 +112,30 @@ def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDo } } +task pullDataPrepperDockerImage(type: DockerPullImage) { + image = 'amazon/opendistro-for-elasticsearch-data-prepper:latest' +} + +def createDataPrepperDockerContainerFromPullImage(final String taskBaseName, final String dataPrepperName, final int grpcPort, + final int serverPort, final String pipelineConfigYAML) { + return tasks.create("create${taskBaseName}", DockerCreateContainer) { + dependsOn createDataPrepperNetwork + dependsOn pullDataPrepperDockerImage + containerName = dataPrepperName + hostConfig.portBindings = [String.format('%d:21890', grpcPort), String.format('%d:4900', serverPort)] + exposePorts('tcp', [21890, 4900]) + hostConfig.network = createDataPrepperNetwork.getNetworkName() + hostConfig.binds = [(project.file(pipelineConfigYAML).toString()):"/usr/share/data-prepper/pipelines.yaml", + (project.file("src/integrationTest/resources/data_prepper.yml").toString()):"/usr/share/data-prepper/data-prepper-config.yaml"] + targetImageId pullDataPrepperDockerImage.image + } +} + /** * ODFE Docker tasks */ task pullOdfeDockerImage(type: DockerPullImage) { - image = 'amazon/opendistro-for-elasticsearch:1.9.0' + image = 'amazon/opendistro-for-elasticsearch:1.13.2' } task createOdfeDockerContainer(type: DockerCreateContainer) { @@ -169,7 +190,44 @@ task rawSpanEndToEndTest(type: Test) { classpath = sourceSets.integrationTest.runtimeClasspath filter { - includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest*" + includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" + } + + finalizedBy stopOdfeDockerContainer + def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer) + def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer) + finalizedBy stopDataPrepper1Task + finalizedBy stopDataPrepper2Task + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper1Task as DockerStopContainer) + finalizedBy removeDataPrepperDockerContainer(stopDataPrepper2Task as DockerStopContainer) + finalizedBy removeDataPrepperNetwork +} + +task rawSpanCompatibilityEndToEndTest(type: Test) { + dependsOn build + dependsOn startOdfeDockerContainer + def createDataPrepper1Task = createDataPrepperDockerContainer( + "rawSpanDataPrepperFromBuild", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage( + "rawSpanDataPrepperFromPull", "data-prepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) + def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) + dependsOn startDataPrepper1Task + dependsOn startDataPrepper2Task + startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer' + startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer' + // wait for data-preppers to be ready + doFirst { + sleep(10*1000) + } + + description = 'Runs the raw span compatibility integration tests.' + group = 'verification' + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath + + filter { + includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest.testPipelineEndToEnd*" } finalizedBy stopOdfeDockerContainer diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml new file mode 100644 index 0000000000..0d7df44b0e --- /dev/null +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -0,0 +1,26 @@ +entry-pipeline: + source: + otel_trace_source: + ssl: false + prepper: + - peer_forwarder: + discovery_mode: "static" + static_endpoints: ["data-prepper1", "data-prepper2"] + ssl: false + sink: + - pipeline: + name: "raw-pipeline" +raw-pipeline: + source: + pipeline: + name: "entry-pipeline" + prepper: + - otel_trace_raw_prepper: + root_span_flush_delay: 3 + trace_flush_interval: 5 + sink: + - elasticsearch: + hosts: [ "https://node-0.example.com:9200" ] + username: "admin" + password: "admin" + trace_analytics_raw: true \ No newline at end of file From 4e5f83814c4a0eed2a1ca9bab0693b9e32240c97 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Wed, 19 May 2021 15:41:16 +0000 Subject: [PATCH 054/192] Added trace-analytics schema documentation. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../otel-v1-apm-service-map-index-template.md | 125 ++++++++++++ .../otel-v1-apm-span-index-template.md | 182 ++++++++++++++++++ docs/schemas/trace-analytics/readme.md | 91 +++++++++ 3 files changed, 398 insertions(+) create mode 100644 docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md create mode 100644 docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md create mode 100644 docs/schemas/trace-analytics/readme.md diff --git a/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md b/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md new file mode 100644 index 0000000000..17812bf7bb --- /dev/null +++ b/docs/schemas/trace-analytics/otel-v1-apm-service-map-index-template.md @@ -0,0 +1,125 @@ +# otel-v1-apm-service-map-index-template + +## Description +Documents in this index correspond to edges in a service map. Edges are created when a request crosses service boundaries. Documents will exclusively contain either a _destination_ or a _target_: +* Destination: corresponds to a client span calling another service. The _destination_ is the other service being called. +* Target: corresponds to a server span. The _target_ is the operation or API being called by the client. + +```json +{ + "version": 0, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "hashId": { + "ignore_above": 1024, + "type": "keyword" + }, + "serviceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "target": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "traceGroupName": { + "ignore_above": 1024, + "type": "keyword" + } + } + } +} +``` + +## Fields +* hashId - A deterministic hash of this relationship. +* kind - The span kind, corresponding to the source of the relationship. See [OpenTelemetry - SpanKind](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind). +* serviceName - The name of the service which emitted the span. Currently derived from the `opentelemetry.proto.resource.v1.Resource` associated with the span. +* destination.domain - The serviceName of the service being called by this client. +* destination.resource - The span name (API, operation, etc.) being called by this client. +* target.domain - The serviceName of the service being called by a client. +* target.resource - The span name (API, operation, etc.) being called by a client. +* traceGroupName - The top-level span name which started the request chain. + +## Example Documents +The two example documents below illustrate the "inventory" service calling the "database" service's `updateItem` API. +```json +{ + "_index": "otel-v1-apm-service-map", + "_type": "_doc", + "_id": "7/jRp2VF7544pBN6+mK2vw==", + "_score": 1, + "_source": { + "serviceName": "inventory", + "kind": "SPAN_KIND_CLIENT", + "destination": { + "resource": "updateItem", + "domain": "database" + }, + "target": null, + "traceGroupName": "client_checkout", + "hashId": "7/jRp2VF7544pBN6+mK2vw==" + } +} +``` + +```json +{ + "_index": "otel-v1-apm-service-map", + "_type": "_doc", + "_id": "lZcUyuhGYfnaQqt+r73njA==", + "_version": 3, + "_score": 0, + "_source": { + "serviceName": "database", + "kind": "SPAN_KIND_SERVER", + "destination": null, + "target": { + "resource": "updateItem", + "domain": "database" + }, + "traceGroupName": "client_checkout", + "hashId": "lZcUyuhGYfnaQqt+r73njA==" + } +} +``` + diff --git a/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md new file mode 100644 index 0000000000..0e775927dd --- /dev/null +++ b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md @@ -0,0 +1,182 @@ +# otel-v1-apm-span-index-template + +## Description +Documents in this index correspond to spans following the [OpenTelemetry tracing specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md). Many fields are directly copied from the span, however some fields are derived and not present in the original span. + +```json +{ + "version": 0, + "mappings": { + "date_detection": false, + "dynamic_templates": [ + { + "resource_attributes_map": { + "mapping": { + "type":"keyword" + }, + "path_match":"resource.attributes.*" + } + }, + { + "attributes_map": { + "mapping": { + "type":"keyword" + }, + "path_match":"attributes.*" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "parentSpanId": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "traceGroupFields": { + "properties": { + "endTime": { + "type": "date_nanos" + }, + "durationInNanos": { + "type": "long" + }, + "statusCode": { + "type": "integer" + } + } + }, + "kind": { + "ignore_above": 128, + "type": "keyword" + }, + "startTime": { + "type": "date_nanos" + }, + "endTime": { + "type": "date_nanos" + }, + "status": { + "properties": { + "code": { + "type": "integer" + }, + "message": { + "type": "keyword" + } + } + }, + "serviceName": { + "type": "keyword" + }, + "durationInNanos": { + "type": "long" + }, + "events": { + "type": "nested", + "properties": { + "time": { + "type": "date_nanos" + } + } + }, + "links": { + "type": "nested" + } + } + } +} +``` + +## Fields +Many fields are either copied or derived from the [trace specification protobuff](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) format. + +* traceId - A unique identifier for a trace. All spans from the same trace share the same traceId. +* spanId - A unique identifier for a span within a trace, assigned when the span is created. +* traceState - Conveys information about request position in multiple distributed tracing graphs. +* parentSpanId - The `spanId` of this span's parent span. If this is a root span, then this field must be empty. +* name - A description of the span's operation. +* kind - The type of span. See [OpenTelemetry - SpanKind](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind). +* startTime - The start time of the span. +* endTime - The end time of the span. +* durationInNanos - Difference in nanoseconds between `startTime` and `endTime`. +* serviceName - Currently derived from the `opentelemetry.proto.resource.v1.Resource` associated with the span, the resource from the span originates. +* events - A list of events. See [OpenTelemetry - Events](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#add-events). +* links - A list of linked spans. See [OpenTelemetry - Links](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#specifying-links). +* droppedAttributesCount - The number of attributes that were discarded. +* droppedEventsCount - The number of events that were discarded. +* droppedLinksCount - The number of links that were dropped. +* traceGroup - A derived field, the `name` of the trace's root span. +* traceGroupFields.endTime - A derived field, the `endTime` of the trace's root span. +* traceGroupFields.statusCode - A derived field, the `status.code` of the trace's root span. +* traceGroupFields.durationInNanos - A derived field, the `durationInNanos` of the trace's root span. +* span.attributes.* - All span attributes are split into a list of keywords. +* resource.attributes.* - All resource attributes are split into a list of keywords. +* status.code - The status of the span. See [OpenTelemetry - Status](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status). + + +## Example Documents + +```json +{ + "_index": "otel-v1-apm-span-000006", + "_type": "_doc", + "_id": "fe0e3811627189df", + "_score": 1, + "_source": { + "traceId": "0000000000000000856bfa5aeba5ec77", + "spanId": "fe0e3811627189df", + "traceState": "", + "parentSpanId": "856bfa5aeba5ec77", + "name": "/getcart", + "kind": "SPAN_KIND_UNSPECIFIED", + "startTime": "2021-05-18T18:58:44.695Z", + "endTime": "2021-05-18T18:58:44.760Z", + "durationInNanos": 65000000, + "serviceName": "cartservice", + "events": [], + "links": [], + "droppedAttributesCount": 0, + "droppedEventsCount": 0, + "droppedLinksCount": 0, + "traceGroup": "/cart", + "traceGroupFields.endTime": "2021-05-18T18:58:44.983Z", + "traceGroupFields.statusCode": 0, + "traceGroupFields.durationInNanos": 387000000, + "span.attributes.http@method": "GET", + "span.attributes.http@url": "http://cartservice/GetCart", + "span.attributes.instance": "cartservice-d847fdcf5-j6s2f", + "span.attributes.version": "v5", + "span.attributes.region": "us-east-1", + "resource.attributes.service@name": "cartservice", + "span.attributes.net@host@ip": "172.22.0.8", + "status.code": 0 + }, + "fields": { + "startTime": [ + "2021-05-18T18:58:44.695Z" + ], + "endTime": [ + "2021-05-18T18:58:44.760Z" + ] + } +} +``` + diff --git a/docs/schemas/trace-analytics/readme.md b/docs/schemas/trace-analytics/readme.md new file mode 100644 index 0000000000..4d7ed27586 --- /dev/null +++ b/docs/schemas/trace-analytics/readme.md @@ -0,0 +1,91 @@ +# Trace Analytics Schema Versioning + +The purpose of this document is to outline the structure and versioning formats followed by Data Prepper (DP) the Trace Analytics (TA) plugin for Kibana and OpenSearch Dashboards. Each individual schema shall have its own supporting document explaining its structure, fields, and purpose. + +## Tenets + +1. Schemas shall follow [semantic versioning](https://semver.org/), excluding patch version numbers. +2. Schema versions shall be detached from Data Prepper and Trace Analytics plugin versions. +3. Index and index template names shall only include the major version (e.g. "otel-v1-apm-span"). The minor version shall be included within the actual schema as a field. +4. Forward and backward-compatibility promises shall only apply to schemas of the same major version. +5. A major version increase shall require Data Prepper (writer) artifacts to be made available before Trace Analytics plugin (reader) updates are made available. +6. A major version increase shall result in a new index template and indexes being created in an Elasticsearch/OpenSearch cluster. + +## Versioning Format + +A schema will be versioned following the [Semantic Versioning 2.0.0 spec](https://semver.org/). A schema version will include a major and minor version number. Patch version numbers will not be used as "patching" is not applicable to a versioned schema; all changes no matter how trivial will have implications. + +1. **Major versions** will be incremented for breaking, backwards-incompatible changes. +2. **Minor versions** will be incremented for backwards-compatible feature additions. This can be thought of an "append-only" change to the schema. + +**Schema versions are detached from Data Prepper and Trace Analytics plugin versions.** A TA plugin version increase does not necessarily affect the version of the schema or Data Prepper. Instead, both DP and TA versions will be *compatible* with a specific schema version. Examples include: + +* Trace Analytics plugin v1.5 includes features built on schema version 1.2.0 +* Data Prepper v1.1 emits documents following schema version 1.2.0 + +### Major version changes + +Major version changes include removing a field, renaming a field, or changing an existing field's datatype. + +* Schema 1.0 to 2.0 *changes the type of a field* from Keyword to Numeric +* Schema 1.0 to 2.0 *removes* *field* "latency" from the schema +* Schema 1.0 to 2.0 *renames field* "end" to "endTime" + * A rename is effectively a field addition and removal in a single operation + +### Minor version changes + +Minor version changes include adding a new field or adding a new nested field. + +* Schema 1.2 to 1.3 adds a new field, "fieldC", as a Keyword +* Schema 2.11 to 2.12 adds a new nested field, "name" to an existing collection, "parentSpan", resulting in "parentSpan.name" + +## Compatibility Promises + +The following compatibility promises are made *only for schemas of the same major version*. + +* ***Backwards compatibility*** - features built on version 1.x of the schema **will not break, but may degrade** if data from a **prior** 1.x schema version is used. +* ***Forwards compatibility*** - features built on version 1.x of the schema **will not break** if data from a **later** 1.x schema version is used. + +### Read-compatibility examples + +1. A plugin built on schema 1.1 but consuming schema 1.2 data will function 100% without issue +2. A plugin built on schema 1.2 but consuming schema 1.1 data will continue to function, however some features might be degraded +3. A plugin built on schema 1.0 but consuming schema 2.0 data is not guaranteed to function +4. A plugin built on schema 2.0 but consuming schema 1.0 data is not guaranteed to function + +### Write-compatibility examples + +1. A writer built on schema 1.2 but writing to a cluster containing schema 1.1 data will succeed +2. A writer built on schema 1.1 but writing to a cluster containing schema 1.2 data will succeed +3. A writer built on schema 1.0 but writing to a cluster containing schema 2.0 data will succeed +4. A writer built on schema 2.0 but writing to a cluster containing schema 1.0 data will succeed + +### Handling minor version updates + +Minor version updates will occur as new fields are needed to support new Trace Analytics features. The steps to update a schema minor version are to: + +1. Ensure both TA and DP owners are aligned with requirements +2. Update the schema in the schema repo + 1. Add new fields to the schema JSON + 2. Increment the `version` field of the schema by 1 + 3. Update the documentation to describe the new fields +3. Add test data to the TA plugin test suite + 1. Don't update existing data in place, instead add new data following the new schema version. The test suite is expected to pass with a range of minor versions being tested at the same time. +4. Update DP to start emitting documents following the new schema version +5. Add new features to the TA plugin utilizing the new data + +### Handling Major version updates + +As schema owners, we will do our best to avoid introducing major version changes. However as our schemas are heavily tied to the OpenTelemetry spec, there is always the risk of an upstream backwards-incompatible change requiring a major version increase. + +Due to the potentially disjointed release schedules of both OpenSearch and the managed offering, we need to ensure that rolling out a major version change is carefully planned. + +A typical migration plan will first make Data Prepper artifacts available so that users can start ingesting their data to the new index. To prevent data loss during the migration period, users can be encouraged to simultaneously write to both the old and new indexes. This can be done by either running both old and new versions of Data Prepper side-by-side, or perhaps Data Prepper itself can be updated to write to dual indexes (TODO). Once users have the ability to write to the new index, Trace Analytics plugin updates will be made available which make use of the new major version index. + +The steps to handle a schema major version update are to: + +1. Update Data Prepper to use the new schema. Increment the Data Prepper major version and make new artifacts available to users. + * This must *always be done first*. Plugin changes cannot go out before Data Prepper artifacts are made available. + * Encourage users to start using the new Data Prepper version ahead of the plugin release. These will allow the user to upgrade their Trace Analytics plugin and immediately have new major version data to work with. +2. Update the Trace Analytics plugin to read from the new indexes. Increment the plugin version and release to OpenSearch and/or the managed offering. + * Communicate to users the need to use the new version of Data Prepper after upgrading their TA plugin From 04dd7bd18977294800cf4b77d7f01914def75f23 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Thu, 20 May 2021 20:12:32 +0000 Subject: [PATCH 055/192] Revised per PR feedback. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../otel-v1-apm-span-index-template.md | 2 +- docs/schemas/trace-analytics/readme.md | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md index 0e775927dd..5dea6ebcf8 100644 --- a/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md +++ b/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md @@ -106,7 +106,7 @@ Documents in this index correspond to spans following the [OpenTelemetry tracing ``` ## Fields -Many fields are either copied or derived from the [trace specification protobuff](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) format. +Many fields are either copied or derived from the [trace specification protobuf](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto) format. * traceId - A unique identifier for a trace. All spans from the same trace share the same traceId. * spanId - A unique identifier for a span within a trace, assigned when the span is created. diff --git a/docs/schemas/trace-analytics/readme.md b/docs/schemas/trace-analytics/readme.md index 4d7ed27586..1e028e0bd8 100644 --- a/docs/schemas/trace-analytics/readme.md +++ b/docs/schemas/trace-analytics/readme.md @@ -4,7 +4,7 @@ The purpose of this document is to outline the structure and versioning formats ## Tenets -1. Schemas shall follow [semantic versioning](https://semver.org/), excluding patch version numbers. +1. Schemas shall follow the [Semantic Versioning 2.0.0 spec](https://semver.org/), excluding patch version numbers. 2. Schema versions shall be detached from Data Prepper and Trace Analytics plugin versions. 3. Index and index template names shall only include the major version (e.g. "otel-v1-apm-span"). The minor version shall be included within the actual schema as a field. 4. Forward and backward-compatibility promises shall only apply to schemas of the same major version. @@ -43,8 +43,8 @@ Minor version changes include adding a new field or adding a new nested field. The following compatibility promises are made *only for schemas of the same major version*. -* ***Backwards compatibility*** - features built on version 1.x of the schema **will not break, but may degrade** if data from a **prior** 1.x schema version is used. -* ***Forwards compatibility*** - features built on version 1.x of the schema **will not break** if data from a **later** 1.x schema version is used. +* ***Backwards compatibility*** - Trace Analytics UI features built on version 1.x of the schema **will not break, but may degrade** if data from a **prior** 1.x schema version is used. +* ***Forwards compatibility*** - Trace Analytics UI features built on version 1.x of the schema **will not break** if data from a **later** 1.x schema version is used. ### Read-compatibility examples @@ -74,18 +74,30 @@ Minor version updates will occur as new fields are needed to support new Trace A 4. Update DP to start emitting documents following the new schema version 5. Add new features to the TA plugin utilizing the new data -### Handling Major version updates +### Handling major version updates -As schema owners, we will do our best to avoid introducing major version changes. However as our schemas are heavily tied to the OpenTelemetry spec, there is always the risk of an upstream backwards-incompatible change requiring a major version increase. +As schema owners, we will do our best to avoid introducing major version changes. However, as our schemas are heavily tied to the OpenTelemetry spec, there is always the risk of an upstream backwards-incompatible change requiring a major version increase. -Due to the potentially disjointed release schedules of both OpenSearch and the managed offering, we need to ensure that rolling out a major version change is carefully planned. +The following procedures may be considered while performing a major upgrade across both Data Prepper and the Trace Analytics plugin. -A typical migration plan will first make Data Prepper artifacts available so that users can start ingesting their data to the new index. To prevent data loss during the migration period, users can be encouraged to simultaneously write to both the old and new indexes. This can be done by either running both old and new versions of Data Prepper side-by-side, or perhaps Data Prepper itself can be updated to write to dual indexes (TODO). Once users have the ability to write to the new index, Trace Analytics plugin updates will be made available which make use of the new major version index. +#### Approach #1: Upgrade both Data Prepper and the Trace Analytics plugin as simultaneously as possible +The simplest approach to perform a major version upgrade is to simultaneously upgrade both Data Prepper and the Trace Analytics plugin. This approach will result in a usability drop while both components are on differing major versions. -The steps to handle a schema major version update are to: +As an example, assume both DP and TA are being upgraded from v1 to v2: +* If DP is upgraded to v2 before TA, then the old TA v1 dashboard will not visualize the new data ingested by DP v2. +* IF TA is upgraded to v2 before DP, then the new TA v2 dashboard will not visualize the old data ingested by DP v1. -1. Update Data Prepper to use the new schema. Increment the Data Prepper major version and make new artifacts available to users. - * This must *always be done first*. Plugin changes cannot go out before Data Prepper artifacts are made available. - * Encourage users to start using the new Data Prepper version ahead of the plugin release. These will allow the user to upgrade their Trace Analytics plugin and immediately have new major version data to work with. -2. Update the Trace Analytics plugin to read from the new indexes. Increment the plugin version and release to OpenSearch and/or the managed offering. - * Communicate to users the need to use the new version of Data Prepper after upgrading their TA plugin +Both component upgrades should happen as close as possible to each other to minimize this window. + +#### Approach #2: Run both old and new Data Prepper versions side-by-side until the Trace Analytics plugin is upgraded +To avoid usability downtime of the Trace Analytics plugin, run two versions of Data Prepper simultaneously before upgrading the plugin. + +* DP v1 writes to the v1 index +* DP v2 writes to the v2 index +* TA v1 reads from the v1 index +* TA v2 reads from the v2 index + +Upgrading the Trace Analytics plugin from v1 to v2 will result in no usability downtime, as both plugin versions will have populated indexes to read from. This approach is more complex in that it requires additional Data Prepper instances and results in duplicate data being written (until the old DP instances are shut down). + +#### Mitigating data loss +Upgrading Data Prepper and the Trace Analytics plugin to a new major version schema will result in data written to old indexes being unusable. If users wish to avoid this data loss, the [reindexing APIs](https://opendistro.github.io/for-elasticsearch-docs/docs/elasticsearch/reindex-data/) must be used. Additionally, transforms might required depending on the differences between the two major schema versions. From 4438caf3e6289bc8f36aeeb881f5add61b6cae4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 06:21:12 +0000 Subject: [PATCH 056/192] Bump cloudwatch from 2.16.64 to 2.16.68 in /data-prepper-core Bumps cloudwatch from 2.16.64 to 2.16.68. Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 6ba0071092..bdbb950700 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" - implementation "software.amazon.awssdk:cloudwatch:2.16.64" + implementation "software.amazon.awssdk:cloudwatch:2.16.68" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.10.0" } From 4ab67dc4bd1cca406a34d6155ff0047f119cb6ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 06:25:17 +0000 Subject: [PATCH 057/192] Bump armeria from 1.7.2 to 1.8.0 in /data-prepper-plugins/peer-forwarder Bumps [armeria](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 6b316b4f09..8173aa9e6d 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -13,7 +13,7 @@ dependencies { compile project(':data-prepper-api') testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "com.linecorp.armeria:armeria:1.7.2" + implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.7.2" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" From b9ef21208f75d141c80ceaa086accb0dfcae5995 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 06:26:07 +0000 Subject: [PATCH 058/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria-grpc](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 9b6f52d40d..49e688336c 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.7.2" - implementation "com.linecorp.armeria:armeria-grpc:1.7.2" + implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' From 4104247cd91cf4b786d6779f21608f88f0781c3a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 06:26:11 +0000 Subject: [PATCH 059/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-source Bumps [armeria-grpc](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 264fbfe067..9667530e23 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.7.2" - implementation "com.linecorp.armeria:armeria-grpc:1.7.2" + implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.19.0' From d02d6b12cf4530f06cb1a8b26c24df93934b1509 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 06:26:19 +0000 Subject: [PATCH 060/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1020 to 1.11.1024. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1020...1.11.1024) Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 7d6455a6a9..e6458f704b 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1020" + implementation "com.amazonaws:aws-java-sdk-core:1.11.1024" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.0" testImplementation("junit:junit:4.13.2") { From b1e298cdbba8838cb7ff8e08cf88334defe48338 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 15:15:17 +0000 Subject: [PATCH 061/192] Bump armeria-grpc in /data-prepper-plugins/peer-forwarder Bumps [armeria-grpc](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 8173aa9e6d..95ab2b1954 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -14,7 +14,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.8.0" - implementation "com.linecorp.armeria:armeria-grpc:1.7.2" + implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" testImplementation "org.mockito:mockito-inline:3.10.0" From 4aa77d5fd89616ac5142f39b28c3027f4e580937 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 15:15:26 +0000 Subject: [PATCH 062/192] Bump armeria in /data-prepper-plugins/otel-trace-source Bumps [armeria](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 9667530e23..c7000e25ee 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' - implementation "com.linecorp.armeria:armeria:1.7.2" + implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From 193f8c12ee9c89440fe6437aa791a5c77612dd06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 May 2021 15:15:50 +0000 Subject: [PATCH 063/192] Bump armeria in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria](https://github.com/line/armeria) from 1.7.2 to 1.8.0. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.7.2...armeria-1.8.0) Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 49e688336c..913fc42cb5 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -9,7 +9,7 @@ dependencies { testCompile project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' - implementation "com.linecorp.armeria:armeria:1.7.2" + implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From f9cf80faca92a38a3c94439ac6c5f924ccfa6d89 Mon Sep 17 00:00:00 2001 From: David Venable Date: Mon, 24 May 2021 21:09:17 -0500 Subject: [PATCH 064/192] Fixed Gradle deprecation warnings. Updated the data-prepper-plugins:common and data-prepper-plugins projects to be Java libraries since that is how they are used. Signed-off-by: David Venable --- data-prepper-benchmarks/mapdb-benchmarks/build.gradle | 2 +- .../service-map-stateful-benchmarks/build.gradle | 2 +- data-prepper-core/build.gradle | 7 ++++--- data-prepper-core/integrationTest.gradle | 6 +++--- data-prepper-plugins/blocking-buffer/build.gradle | 2 +- data-prepper-plugins/build.gradle | 6 +++++- data-prepper-plugins/common/build.gradle | 4 ++-- data-prepper-plugins/elasticsearch/build.gradle | 6 +++--- data-prepper-plugins/mapdb-prepper-state/build.gradle | 8 ++++---- .../otel-trace-group-prepper/build.gradle | 6 +++--- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 8 ++++---- data-prepper-plugins/otel-trace-source/build.gradle | 8 ++++---- data-prepper-plugins/peer-forwarder/build.gradle | 4 ++-- data-prepper-plugins/service-map-stateful/build.gradle | 8 ++++---- .../sample-app/analytics-service/build.gradle | 6 +++--- release/archives/build.gradle | 2 +- release/build.gradle | 2 +- research/zipkin-elastic-to-otel/build.gradle | 7 ++++--- 18 files changed, 50 insertions(+), 44 deletions(-) diff --git a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle index 6b77f7d135..2d3761727b 100644 --- a/data-prepper-benchmarks/mapdb-benchmarks/build.gradle +++ b/data-prepper-benchmarks/mapdb-benchmarks/build.gradle @@ -13,5 +13,5 @@ repositories { } dependencies { - compile project(':data-prepper-plugins:mapdb-prepper-state') + implementation project(':data-prepper-plugins:mapdb-prepper-state') } diff --git a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle index 32041023bf..2a46700be4 100644 --- a/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle +++ b/data-prepper-benchmarks/service-map-stateful-benchmarks/build.gradle @@ -13,6 +13,6 @@ repositories { } dependencies { - compile project(':data-prepper-plugins:service-map-stateful') + implementation project(':data-prepper-plugins:service-map-stateful') jmh "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" } diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index bdbb950700..34dfbba34d 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -9,9 +9,10 @@ sourceSets { apply from: "integrationTest.gradle" dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins') - testCompile project(':data-prepper-plugins:common').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins') + implementation project(':data-prepper-plugins:common') + testImplementation project(':data-prepper-plugins:common').sourceSets.test.output implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "javax.validation:validation-api:2.0.1.Final" diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 40bfdbaebc..631ec30c04 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -27,9 +27,9 @@ sourceSets { } dependencies { - integrationTestCompile("junit:junit:4.13") - integrationTestCompile project(':data-prepper-plugins:elasticsearch') - integrationTestCompile project(':data-prepper-plugins:otel-trace-group-prepper') + integrationTestImplementation("junit:junit:4.13") + integrationTestImplementation project(':data-prepper-plugins:elasticsearch') + integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-prepper') integrationTestImplementation "org.awaitility:awaitility:4.0.3" integrationTestImplementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" integrationTestImplementation 'com.google.protobuf:protobuf-java-util:3.13.0' diff --git a/data-prepper-plugins/blocking-buffer/build.gradle b/data-prepper-plugins/blocking-buffer/build.gradle index fd83966b15..46bb3df3a3 100644 --- a/data-prepper-plugins/blocking-buffer/build.gradle +++ b/data-prepper-plugins/blocking-buffer/build.gradle @@ -2,7 +2,7 @@ plugins { id 'java' } dependencies { - compile project(':data-prepper-api') + implementation project(':data-prepper-api') testImplementation "junit:junit:4.13.2" } diff --git a/data-prepper-plugins/build.gradle b/data-prepper-plugins/build.gradle index b3395d9159..abb95bdeb8 100644 --- a/data-prepper-plugins/build.gradle +++ b/data-prepper-plugins/build.gradle @@ -1,3 +1,7 @@ +plugins { + id 'java-library' +} + dependencies { - subprojects.forEach { compile project(':data-prepper-plugins:' + it.name) } + subprojects.forEach { api project(':data-prepper-plugins:' + it.name) } } \ No newline at end of file diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 409f1eceda..a72ac5fc43 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -1,8 +1,8 @@ plugins { - id 'java' + id 'java-library' } dependencies { - compile project(':data-prepper-api') + api project(':data-prepper-api') implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.reflections:reflections:0.9.12" diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index e6458f704b..2114c483b3 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -28,9 +28,9 @@ ext { } dependencies { - compile project(':data-prepper-api') - testCompile project(':data-prepper-api').sourceSets.test.output - compile project(':data-prepper-plugins:common') + api project(':data-prepper-api') + testImplementation project(':data-prepper-api').sourceSets.test.output + api project(':data-prepper-plugins:common') implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index a2184fbad0..a4f5c184b5 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -12,14 +12,14 @@ repositories { } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.13.2' - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.0' - testCompile project(':data-prepper-plugins:common').sourceSets.test.output + testImplementation project(':data-prepper-plugins:common').sourceSets.test.output + testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation "org.hamcrest:hamcrest:2.2" } diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index ff11cf6b94..e3c9000873 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -7,9 +7,9 @@ ext { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:elasticsearch') - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:elasticsearch') + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 913fc42cb5..ca06313d61 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -3,10 +3,10 @@ plugins { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') - compile 'commons-codec:commons-codec:1.15' - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + implementation 'commons-codec:commons-codec:1.15' + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.8.0" diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index c7000e25ee..800010e350 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -3,10 +3,10 @@ plugins { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:blocking-buffer') - compile 'commons-codec:commons-codec:1.15' - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:blocking-buffer') + implementation 'commons-codec:commons-codec:1.15' + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.0' implementation "com.linecorp.armeria:armeria:1.8.0" diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 95ab2b1954..3a510dd6b7 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -10,8 +10,8 @@ repositories { } dependencies { - compile project(':data-prepper-api') - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index b297897e8c..6b10013ff2 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -12,10 +12,10 @@ repositories { } dependencies { - compile project(':data-prepper-api') - compile project(':data-prepper-plugins:common') - compile project(':data-prepper-plugins:mapdb-prepper-state') - testCompile project(':data-prepper-api').sourceSets.test.output + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + implementation project(':data-prepper-plugins:mapdb-prepper-state') + testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.micrometer:micrometer-core:1.7.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" diff --git a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle index ea8f560c27..db62f35193 100644 --- a/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle +++ b/examples/trace-analytics-sample-app/sample-app/analytics-service/build.gradle @@ -16,9 +16,9 @@ repositories { dependencies { implementation('org.springframework.boot:spring-boot-starter-web') testImplementation('org.springframework.boot:spring-boot-starter-test') - compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.8.1' - compile 'org.elasticsearch.client:elasticsearch-rest-client:7.8.1' - compile 'org.elasticsearch:elasticsearch:7.8.1' + implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.8.1' + implementation 'org.elasticsearch.client:elasticsearch-rest-client:7.8.1' + implementation 'org.elasticsearch:elasticsearch:7.8.1' } bootJar { diff --git a/release/archives/build.gradle b/release/archives/build.gradle index 228ec079de..c429bf17b4 100644 --- a/release/archives/build.gradle +++ b/release/archives/build.gradle @@ -86,7 +86,7 @@ subprojects { tasks.withType(Tar) { dependsOn ':release:releasePrerequisites' compression = Compression.GZIP - extension = 'tar.gz' + archiveExtension = 'tar.gz' } tasks.withType(Zip) { diff --git a/release/build.gradle b/release/build.gradle index 1aaebe8f12..6f87ba4102 100644 --- a/release/build.gradle +++ b/release/build.gradle @@ -2,7 +2,7 @@ apply from: file("build-resources.gradle") allprojects { dependencies { - compile project(':data-prepper-core') + implementation project(':data-prepper-core') } } diff --git a/research/zipkin-elastic-to-otel/build.gradle b/research/zipkin-elastic-to-otel/build.gradle index 5d65d39c29..7f20761f09 100644 --- a/research/zipkin-elastic-to-otel/build.gradle +++ b/research/zipkin-elastic-to-otel/build.gradle @@ -27,9 +27,10 @@ run { } dependencies { - compile project(':data-prepper-plugins:elasticsearch') - compile project(':data-prepper-plugins:otel-trace-source') - compile project(':data-prepper-plugins:otel-trace-raw-prepper') + implementation project(':data-prepper-plugins:blocking-buffer') + implementation project(':data-prepper-plugins:elasticsearch') + implementation project(':data-prepper-plugins:otel-trace-source') + implementation project(':data-prepper-plugins:otel-trace-raw-prepper') implementation "org.apache.commons:commons-lang3:3.11" implementation "com.linecorp.armeria:armeria:1.0.0" implementation "com.linecorp.armeria:armeria-grpc:1.0.0" From 5c2cdedb0d8fcd22f527efa24da1e961e1245e2f Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 25 May 2021 15:37:15 +0000 Subject: [PATCH 065/192] Renamed dockerfile. Added gradle.properties file. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- build.gradle | 2 +- gradle.properties | 1 + ...istroforelasticsearch-data-prepper.Dockerfile => Dockerfile} | 0 release/docker/build.gradle | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 gradle.properties rename release/docker/{opendistroforelasticsearch-data-prepper.Dockerfile => Dockerfile} (100%) diff --git a/build.gradle b/build.gradle index 40c2e91452..d127c0ab05 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ allprojects { apply plugin: 'com.diffplug.spotless' group = 'com.amazon' - version = '1.0.0' + repositories { mavenCentral() maven { url "https://jitpack.io" } diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000..aef125e083 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +version=1.0.0 diff --git a/release/docker/opendistroforelasticsearch-data-prepper.Dockerfile b/release/docker/Dockerfile similarity index 100% rename from release/docker/opendistroforelasticsearch-data-prepper.Dockerfile rename to release/docker/Dockerfile diff --git a/release/docker/build.gradle b/release/docker/build.gradle index ed29a38f37..bb97565a63 100644 --- a/release/docker/build.gradle +++ b/release/docker/build.gradle @@ -9,7 +9,7 @@ docker { buildArgs(['JAR_FILE' : project(':data-prepper-core').jar.archiveName, 'CONFIG_FILEPATH': '/usr/share/data-prepper/data-prepper-config.yaml', 'PIPELINE_FILEPATH': '/usr/share/data-prepper/pipelines.yaml']) - dockerfile file('opendistroforelasticsearch-data-prepper.Dockerfile') + dockerfile file('Dockerfile') } dockerPrepare.dependsOn ':release:releasePrerequisites' From 07ed0ed283fe7501ca6fc1dfc98f70f752698d89 Mon Sep 17 00:00:00 2001 From: David Venable Date: Fri, 28 May 2021 12:27:19 -0500 Subject: [PATCH 066/192] Updated the Project Setup documentation to include information about JDK versions and the difference between assemble and build. Signed-off-by: David Venable --- docs/readme/project_setup.md | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/readme/project_setup.md b/docs/readme/project_setup.md index d7de5bf99c..82eac18f8f 100644 --- a/docs/readme/project_setup.md +++ b/docs/readme/project_setup.md @@ -1,14 +1,43 @@ # Project Setup +## Installation Prerequisites + +### JDK Versions + +Running Data Prepper requires JDK 8 and above. + +Running the integration tests requires JDK 14 or 15. + + ## Building from source -To build the project from source, run + +The assemble task will build the Jar files without running the integration +tests. You can use these jar files for running DataPrepper. If you are just +looking to use DataPrepper and modify it, this build +is faster than running the integration test suite and only requires JDK 8+. + +To build the project from source, run ``` -./gradlew build +./gradlew assemble ``` from the project root. +### Full Project Build + +Running the build command will assemble the Jar files needed +for running DataPrepper. It will also run the integration test +suite. + +To build, run + +``` +./gradlew build +``` + +from the project root. + ## Running the project After building, the project can be run from the executable JAR **data-prepper-core-$VERSION** From d33d5599a89addf78c3add54e2422f6cfa8ebbcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 06:24:14 +0000 Subject: [PATCH 067/192] Bump kotlin-stdlib in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib](https://github.com/JetBrains/kotlin) from 1.5.0 to 1.5.10. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.0...v1.5.10) Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index a4f5c184b5..afbc1a09bf 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.10' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.0' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output From 5765e9e6afc520bba17df202294444ce6e33a032 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 2 Jun 2021 13:40:27 -0500 Subject: [PATCH 068/192] Bug Fixes: path_match in raw index template (#624) * FIX: path_match Signed-off-by: qchea * MAINT: update both span and service-map template version Signed-off-by: qchea * MAINT: revert version upgrade on service-map Signed-off-by: qchea --- .../otel-v1-apm-span-index-template.json | 6 ++--- .../src/test/resources/raw-span-1.json | 22 +++++++++---------- .../src/test/resources/raw-span-2.json | 22 +++++++++---------- .../src/test/resources/raw-span-error.json | 16 +++++++------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json index f7bc7b8727..9fefc33c18 100644 --- a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json +++ b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json @@ -1,5 +1,5 @@ { - "version": 0, + "version": 1, "mappings": { "date_detection": false, "dynamic_templates": [ @@ -12,11 +12,11 @@ } }, { - "attributes_map": { + "span_attributes_map": { "mapping": { "type":"keyword" }, - "path_match":"attributes.*" + "path_match":"span.attributes.*" } } ], diff --git a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-1.json b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-1.json index d2a2dc575f..ef3c514c21 100644 --- a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-1.json +++ b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-1.json @@ -12,17 +12,17 @@ "startTime":"2020-08-20T05:40:46.041011600Z", "endTime":"2020-08-20T05:40:46.089556800Z", "durationInNanos": 48545200, - "attributes.http@status_code":200, - "attributes.net@peer@port":41168, - "attributes.servlet@path":"/logs", - "attributes.http@response_content_length":7, - "attributes.http@user_agent":"curl/7.54.0", - "attributes.http@flavor":"HTTP/1.1", - "attributes.servlet@context":"", - "attributes.http@url":"http://0.0.0.0:8087/logs", - "attributes.net@peer@ip":"172.29.0.1", - "attributes.http@method":"POST", - "attributes.http@client_ip":"172.29.0.1", + "span.attributes.http@status_code":200, + "span.attributes.net@peer@port":41168, + "span.attributes.servlet@path":"/logs", + "span.attributes.http@response_content_length":7, + "span.attributes.http@user_agent":"curl/7.54.0", + "span.attributes.http@flavor":"HTTP/1.1", + "span.attributes.servlet@context":"", + "span.attributes.http@url":"http://0.0.0.0:8087/logs", + "span.attributes.net@peer@ip":"172.29.0.1", + "span.attributes.http@method":"POST", + "span.attributes.http@client_ip":"172.29.0.1", "resource.attributes@telemetry@sdk@language":"java", "resource.attributes@telemetry@sdk@name":"opentelemetry", "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" diff --git a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-2.json b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-2.json index 57e06dea33..4580705132 100644 --- a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-2.json +++ b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-2.json @@ -12,17 +12,17 @@ "startTime":"2020-08-20T05:40:46.042011600Z", "endTime":"2020-08-20T05:40:46.088556800Z", "durationInNanos": 46545200, - "attributes.http@status_code":200, - "attributes.net@peer@port":41168, - "attributes.servlet@path":"/logs", - "attributes.http@response_content_length":7, - "attributes.http@user_agent":"curl/7.54.0", - "attributes.http@flavor":"HTTP/1.1", - "attributes.servlet@context":"", - "attributes.http@url":"http://0.0.0.0:8087/logs", - "attributes.net@peer@ip":"172.29.0.1", - "attributes.http@method":"POST", - "attributes.http@client_ip":"172.29.0.1", + "span.attributes.http@status_code":200, + "span.attributes.net@peer@port":41168, + "span.attributes.servlet@path":"/logs", + "span.attributes.http@response_content_length":7, + "span.attributes.http@user_agent":"curl/7.54.0", + "span.attributes.http@flavor":"HTTP/1.1", + "span.attributes.servlet@context":"", + "span.attributes.http@url":"http://0.0.0.0:8087/logs", + "span.attributes.net@peer@ip":"172.29.0.1", + "span.attributes.http@method":"POST", + "span.attributes.http@client_ip":"172.29.0.1", "resource.attributes@telemetry@sdk@language":"java", "resource.attributes@telemetry@sdk@name":"opentelemetry", "resource.attributes@telemetry@sdk@version":"0.8.0-SNAPSHOT" diff --git a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-error.json b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-error.json index 522220fe17..3a7956523b 100644 --- a/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-error.json +++ b/data-prepper-plugins/elasticsearch/src/test/resources/raw-span-error.json @@ -7,14 +7,14 @@ "status":{}, "startTime":"2020-09-01T22:01:55.103171300Z", "endTime":"2020-09-01T22:01:55.113684900Z", - "attributes.component":"mysql", - "attributes.db.type":"sql", - "attributes.db.instance":"APM", - "attributes.db.statement":"INSERT INTO Inventory_Items (ItemId, TotalQty) VALUES (%(ItemId)s, %(Qty)s) ON DUPLICATE KEY UPDATE TotalQty = TotalQty + %(Qty)s", - "attributes.db.user":"root", - "attributes.net.peer.name":"mysql", - "attributes.net.peer.port":3306, - "attributes.db.statement.parameters":"{'ItemId': 'banana', 'Qty': '6'}", + "span.attributes.component":"mysql", + "span.attributes.db.type":"sql", + "span.attributes.db.instance":"APM", + "span.attributes.db.statement":"INSERT INTO Inventory_Items (ItemId, TotalQty) VALUES (%(ItemId)s, %(Qty)s) ON DUPLICATE KEY UPDATE TotalQty = TotalQty + %(Qty)s", + "span.attributes.db.statement.parameters":"{'ItemId': 'banana', 'Qty': '6'}", + "span.attributes.db.user":"root", + "span.attributes.net.peer.name":"mysql", + "span.attributes.net.peer.port":3306, "resource.attributes.service.name":"database", "resource.attributes.service.instance.id":"140683459769616", "resource.attributes.telemetry.sdk.name":"opentelemetry", From 8a7cde99913ba7f4487e3e9d09df4b877c09660d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:15:25 +0000 Subject: [PATCH 069/192] Bump cloudwatch from 2.16.68 to 2.16.83 in /data-prepper-core Bumps cloudwatch from 2.16.68 to 2.16.83. --- updated-dependencies: - dependency-name: software.amazon.awssdk:cloudwatch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 34dfbba34d..1531762ef4 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" - implementation "software.amazon.awssdk:cloudwatch:2.16.68" + implementation "software.amazon.awssdk:cloudwatch:2.16.83" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.10.0" } From 91cc5ccf6af3da46e72ebaf9b0f0f558d6d1bad3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:15:33 +0000 Subject: [PATCH 070/192] Bump mockito-core from 3.10.0 to 3.11.1 in /data-prepper-core Bumps [mockito-core](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 34dfbba34d..ce0a846ac4 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" implementation "software.amazon.awssdk:cloudwatch:2.16.68" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-core:3.10.0" + testImplementation "org.mockito:mockito-core:3.11.1" } jar { From 7c55ae7f97402885b6b530dab30136f017bd5c6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:20:18 +0000 Subject: [PATCH 071/192] Bump commons-io from 2.8.0 to 2.10.0 in /data-prepper-plugins/common Bumps commons-io from 2.8.0 to 2.10.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index a72ac5fc43..a671fc6daa 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -7,7 +7,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.reflections:reflections:0.9.12" testImplementation "junit:junit:4.13.2" - testImplementation "commons-io:commons-io:2.8.0" + testImplementation "commons-io:commons-io:2.10.0" } jacocoTestCoverageVerification { From 673383065a9812a859268b25c78a3b50706ad695 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:20:52 +0000 Subject: [PATCH 072/192] Bump mockito-inline in /data-prepper-plugins/peer-forwarder Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 3a510dd6b7..786f2114f1 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-inline:3.10.0" + testImplementation "org.mockito:mockito-inline:3.11.1" } jacocoTestCoverageVerification { From 819656c4be18f0fc84fa6904d8cf8ed315d22447 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:21:27 +0000 Subject: [PATCH 073/192] Bump snakeyaml from 1.28 to 1.29 in /data-prepper-plugins/elasticsearch Bumps [snakeyaml](https://bitbucket.org/asomov/snakeyaml) from 1.28 to 1.29. - [Commits](https://bitbucket.org/asomov/snakeyaml/branches/compare/snakeyaml-1.29..snakeyaml-1.28) --- updated-dependencies: - dependency-name: org.yaml:snakeyaml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 2114c483b3..525dc6ce0f 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -63,7 +63,7 @@ configurations.all { force 'org.apache.httpcomponents:httpclient:4.5.10' force "org.hdrhistogram:HdrHistogram:2.1.12" force 'joda-time:joda-time:2.10.10' - force 'org.yaml:snakeyaml:1.28' + force 'org.yaml:snakeyaml:1.29' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.3' force 'junit:junit:4.13' force "org.slf4j:slf4j-api:1.7.30" From 71139a25daa4a432b44a7573699c327d0e5fdafb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:21:45 +0000 Subject: [PATCH 074/192] Bump mockito-inline in /data-prepper-plugins/service-map-stateful Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 6b10013ff2..c721ea842f 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-inline:3.10.0" + testImplementation "org.mockito:mockito-inline:3.11.1" } jacocoTestCoverageVerification { From c3a57234f0d9f94a6fc9a15242119b14dfa05f2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:22:13 +0000 Subject: [PATCH 075/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-raw-prepper Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.17.0 to 3.17.3. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.17.0...v3.17.3) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java-util dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index ca06313d61..7c03b473de 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.15' testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.17.0' + implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From bf83f779bf3b3986434caa2cd4d9d01008e8510e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:22:19 +0000 Subject: [PATCH 076/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-raw-prepper Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index ca06313d61..309707b9d3 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.10.0" + testImplementation "org.mockito:mockito-inline:3.11.1" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.awaitility:awaitility:4.1.0" } From 2a780a3e736de508a9501d41702bb88d606d9940 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:23:18 +0000 Subject: [PATCH 077/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-source Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 800010e350..98596ea0cc 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.19.0' - testImplementation "org.mockito:mockito-inline:3.10.0" + testImplementation "org.mockito:mockito-inline:3.11.1" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') From ab23496be117ff2bf522fd13daeefcd726e02b41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 06:23:24 +0000 Subject: [PATCH 078/192] Bump mockito-junit-jupiter in /data-prepper-plugins/otel-trace-source Bumps [mockito-junit-jupiter](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-junit-jupiter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 800010e350..f486326800 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -20,7 +20,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects testImplementation("org.mockito:mockito-core:3.10.0") - testImplementation("org.mockito:mockito-junit-jupiter:3.10.0") + testImplementation("org.mockito:mockito-junit-jupiter:3.11.1") } test { From 6c255cfc70d216b8b63fc4390d1c291368b612bc Mon Sep 17 00:00:00 2001 From: James Christian Date: Tue, 15 Jun 2021 17:27:41 +0100 Subject: [PATCH 079/192] Fixes #666 by including AWS STS library required for IRSA in EKS --- data-prepper-plugins/elasticsearch/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 2114c483b3..c652f075a4 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -36,6 +36,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation "com.amazonaws:aws-java-sdk-core:1.11.1024" + implementation "com.amazonaws:aws-java-sdk-sts:1.11.1024" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.0" testImplementation("junit:junit:4.13.2") { From 2ad6cca3073124ffa9aa985bb6d9fdc57cce78f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:36:34 +0000 Subject: [PATCH 080/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1024 to 1.12.7. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1024...1.12.7) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index c652f075a4..1b4c7d1072 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.11.1024" + implementation "com.amazonaws:aws-java-sdk-core:1.12.7" implementation "com.amazonaws:aws-java-sdk-sts:1.11.1024" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.0" @@ -53,7 +53,7 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.11.1020' + force 'com.amazonaws:aws-java-sdk-core:1.12.7' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' From dd559b18d46b3f71911512e15333b53e2271a0cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:37:14 +0000 Subject: [PATCH 081/192] Bump commons-io in /data-prepper-plugins/elasticsearch Bumps commons-io from 2.8.0 to 2.10.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index c652f075a4..33c27ad191 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -44,7 +44,7 @@ dependencies { } testImplementation "org.awaitility:awaitility:4.1.0" testImplementation "org.elasticsearch.test:framework:${es_version}" - testImplementation "commons-io:commons-io:2.8.0" + testImplementation "commons-io:commons-io:2.10.0" } // Workaround for Werror From 6c097f9b9a5427c6690e27aef9b79527f6bc603c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:37:35 +0000 Subject: [PATCH 082/192] Bump mockito-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [mockito-core](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index e3c9000873..bd50739646 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -14,6 +14,6 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "io.micrometer:micrometer-core:1.7.0" - testImplementation "org.mockito:mockito-core:3.10.0" + testImplementation "org.mockito:mockito-core:3.11.1" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From 10b1de28a67b2cbaf563c1072da2939c47648c91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:37:47 +0000 Subject: [PATCH 083/192] Bump mockito-core in /data-prepper-plugins/otel-trace-source Bumps [mockito-core](https://github.com/mockito/mockito) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.10.0...v3.11.1) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 6643017e7c..194efc04df 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -19,7 +19,7 @@ dependencies { testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects - testImplementation("org.mockito:mockito-core:3.10.0") + testImplementation("org.mockito:mockito-core:3.11.1") testImplementation("org.mockito:mockito-junit-jupiter:3.11.1") } From 97656db9b360018c3082fbc7a95bbb6d143a371b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:38:53 +0000 Subject: [PATCH 084/192] Bump protobuf-java-util in /data-prepper-plugins/otel-trace-source Bumps [protobuf-java-util](https://github.com/protocolbuffers/protobuf) from 3.17.0 to 3.17.3. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/master/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.17.0...v3.17.3) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java-util dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 194efc04df..89e434407d 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.15' testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation 'com.google.protobuf:protobuf-java-util:3.17.0' + implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.8.0" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" From f1ad8276721221aaf949c5964f8183f8929a1363 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:40:13 +0000 Subject: [PATCH 085/192] Bump kotlin-stdlib-common in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib-common](https://github.com/JetBrains/kotlin) from 1.5.0 to 1.5.10. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.0...v1.5.10) Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index afbc1a09bf..c61de6a6b2 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.10' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.0' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.10' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output testImplementation group: 'junit', name: 'junit', version: '4.13.2' From d82ca36f5663f90e38f5e07c339e41c36547bc79 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Wed, 16 Jun 2021 22:16:38 +0000 Subject: [PATCH 086/192] Refactor of ServiceMapStatefulPrepper, moved MapDbPrepperState from file to heap. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../prepper/state/PrepperState.java | 6 + .../prepper/state/MapDbPrepperState.java | 34 ++- .../prepper/state/MapDbPrepperStateTest.java | 15 +- .../prepper/ServiceMapStatefulPrepper.java | 211 +++++++----------- .../ServiceMapStatefulPrepperTest.java | 6 +- 5 files changed, 122 insertions(+), 150 deletions(-) diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java index 824b9614c3..550f8bb5c4 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/prepper/state/PrepperState.java @@ -57,8 +57,14 @@ public interface PrepperState { /** * @return Size of the prepper state data stored in file, in bytes. */ + // TODO: Potentially remove, this is file-specific public long sizeInBytes(); + /** + * Clear internal state + */ + void clear(); + /** * Any cleanup code goes here */ diff --git a/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java b/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java index e3e5984f8a..993e024515 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java +++ b/data-prepper-plugins/mapdb-prepper-state/src/main/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperState.java @@ -1,18 +1,20 @@ package com.amazon.dataprepper.plugins.prepper.state; -import com.google.common.primitives.SignedBytes; import com.amazon.dataprepper.prepper.state.PrepperState; +import com.google.common.primitives.SignedBytes; +import org.mapdb.BTreeMap; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; +import org.mapdb.serializer.SerializerByteArray; + import java.io.File; import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.function.BiFunction; -import org.mapdb.BTreeMap; -import org.mapdb.DBMaker; -import org.mapdb.Serializer; -import org.mapdb.serializer.SerializerByteArray; public class MapDbPrepperState implements PrepperState { @@ -30,14 +32,11 @@ public int compare(byte[] o1, byte[] o2) { private final File dbFile; public MapDbPrepperState(final File dbPath, final String dbName, final int concurrencyScale) { + // TODO: Cleanup references to file-based map this.dbFile = new File(String.join("/", dbPath.getPath(), dbName)); map = - (BTreeMap) DBMaker.fileDB(dbFile) - .fileDeleteAfterClose() - .fileMmapEnable() //MapDB uses the (slower) Random Access Files by default - .fileMmapPreclearDisable() + (BTreeMap) DBMaker.heapDB() .executorEnable() - .transactionEnable() .closeOnJvmShutdown() .concurrencyScale(concurrencyScale) .make() @@ -87,12 +86,18 @@ public List iterate(BiFunction fn, final int segments, fina return returnList; } + public Iterator> getIterator(final int segments, final int index) { + final KeyRange iterationEndpoints = getIterationEndpoints(segments, index); + return map.entryIterator(iterationEndpoints.low, true, iterationEndpoints.high, false); + } + /** * Gets iteration endpoints by taking the lowest and highest key and splitting the keyrange into segments. * These endpoints are an approximation of segments, and segments are guaranteed to cover the entire key range, * but there is no guarantee that all segments contain an equal number of elements. + * * @param segments Number of segments - * @param index Index to find segment endpoints for + * @param index Index to find segment endpoints for * @return KeyRange containing the two endpoints */ private KeyRange getIterationEndpoints(final int segments, final int index) { @@ -103,7 +108,7 @@ private KeyRange getIterationEndpoints(final int segments, final int index) { final byte[] highIndex = index == segments - 1 ? highEnd.add(new BigInteger("1")).toByteArray() : - lowEnd.add(step.multiply(new BigInteger(String.valueOf(index+1)))).toByteArray(); + lowEnd.add(step.multiply(new BigInteger(String.valueOf(index + 1)))).toByteArray(); return new KeyRange(lowIndex, highIndex); } @@ -118,6 +123,11 @@ public long sizeInBytes() { return dbFile.length(); } + @Override + public void clear() { + map.clear(); + } + @Override public void delete() { map.close(); diff --git a/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java b/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java index 8567a9ab5c..16b1bcc4bc 100644 --- a/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java +++ b/data-prepper-plugins/mapdb-prepper-state/src/test/java/com/amazon/dataprepper/plugins/prepper/state/MapDbPrepperStateTest.java @@ -1,14 +1,15 @@ package com.amazon.dataprepper.plugins.prepper.state; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.function.BiFunction; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; public class MapDbPrepperStateTest extends PrepperStateTest { @@ -22,9 +23,9 @@ public void setPrepperState() throws Exception { @Test public void testIterateSegment() throws IOException { - final byte[] key1 = new byte[]{-64, 0, -64, 0}; + final byte[] key1 = new byte[]{-64, 0, -64, 0}; final byte[] key2 = new byte[]{0}; - final byte[] key3 = new byte[]{64, 64, 64 , 64}; + final byte[] key3 = new byte[]{64, 64, 64, 64}; final byte[] key4 = new byte[]{126, 126, 126, 126}; final DataClass data1 = new DataClass(UUID.randomUUID().toString(), random.nextInt()); @@ -61,8 +62,6 @@ public String apply(byte[] bytes, DataClass s) { data3.stringVal, data4.stringVal ))); - - Assert.assertEquals( 1048576, prepperState.sizeInBytes()); } } diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java index 00e331ca06..72b0b636da 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java @@ -7,8 +7,8 @@ import com.amazon.dataprepper.model.prepper.AbstractPrepper; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.plugins.prepper.state.MapDbPrepperState; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.google.common.primitives.SignedBytes; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import org.slf4j.Logger; @@ -21,13 +21,11 @@ import java.util.Collections; import java.util.HashSet; import java.util.Map; -import java.util.Objects; +import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; -import java.util.stream.Collectors; -import java.util.stream.Stream; @SingleThread @DataPrepperPlugin(name = "service_map_stateful", type = PluginType.PREPPER) @@ -46,17 +44,16 @@ public class ServiceMapStatefulPrepper extends AbstractPrepper previousWindow; private volatile static MapDbPrepperState currentWindow; private volatile static MapDbPrepperState previousTraceGroupWindow; private volatile static MapDbPrepperState currentTraceGroupWindow; //TODO: Consider keeping this state in a db - private volatile static HashSet relationshipState = new HashSet<>(); + private static final Set RELATIONSHIP_STATE = Sets.newConcurrentHashSet(); private static File dbPath; private static Clock clock; - private static int processWorkers; private final int thisPrepperId; @@ -77,15 +74,18 @@ public ServiceMapStatefulPrepper(final long windowDurationMillis, ServiceMapStatefulPrepper.clock = clock; this.thisPrepperId = preppersCreated.getAndIncrement(); + if (isMasterInstance()) { previousTimestamp = ServiceMapStatefulPrepper.clock.millis(); ServiceMapStatefulPrepper.windowDurationMillis = windowDurationMillis; ServiceMapStatefulPrepper.dbPath = createPath(databasePath); - ServiceMapStatefulPrepper.processWorkers = processWorkers; + currentWindow = new MapDbPrepperState<>(dbPath, getNewDbName(), processWorkers); previousWindow = new MapDbPrepperState<>(dbPath, getNewDbName() + EMPTY_SUFFIX, processWorkers); currentTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName(), processWorkers); previousTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName() + EMPTY_SUFFIX, processWorkers); + + allThreadsCyclicBarrier = new CyclicBarrier(processWorkers); } pluginMetrics.gauge(SPANS_DB_SIZE, this, serviceMapStateful -> serviceMapStateful.getSpansDbSize()); @@ -165,88 +165,81 @@ public Collection> doExecute(Collection containing json representation of ServiceMapRelationships found */ private Collection> evaluateEdges() { + LOG.info("Evaluating service map edges"); try { - final Stream previousStream = previousWindow.iterate(relationshipIterationFunction, preppersCreated.get(), thisPrepperId).stream().flatMap(serviceMapEdgeStream -> serviceMapEdgeStream); - final Stream currentStream = currentWindow.iterate(relationshipIterationFunction, preppersCreated.get(), thisPrepperId).stream().flatMap(serviceMapEdgeStream -> serviceMapEdgeStream); - - final Collection> serviceDependencyRecords = - Stream.concat(previousStream, currentStream).filter(Objects::nonNull) - .filter(serviceMapRelationship -> !relationshipState.contains(serviceMapRelationship)) - .map(serviceMapRelationship -> { - try { - relationshipState.add(serviceMapRelationship); - return new Record<>(OBJECT_MAPPER.writeValueAsString(serviceMapRelationship)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toSet()); - - if (edgeEvaluationLatch == null) { - initEdgeEvaluationLatch(); - } - doneEvaluatingEdges(); - waitForEvaluationFinish(); + final Collection> serviceDependencyRecords = new HashSet<>(); + + serviceDependencyRecords.addAll(iteratePrepperState(previousWindow)); + serviceDependencyRecords.addAll(iteratePrepperState(currentWindow)); + LOG.info("Done evaluating service map edges"); + + // Wait for all workers before rotating windows + allThreadsCyclicBarrier.await(); if (isMasterInstance()) { rotateWindows(); - resetWorkState(); - } else { - waitForRotationFinish(); } + // Wait for all workers before exiting this method + allThreadsCyclicBarrier.await(); + return serviceDependencyRecords; - } catch (InterruptedException e) { + } catch (InterruptedException | BrokenBarrierException e) { throw new RuntimeException(e); } } - private static synchronized void initEdgeEvaluationLatch() { - if (edgeEvaluationLatch == null) { - edgeEvaluationLatch = new CountDownLatch(preppersCreated.get()); - } - } + private Collection> iteratePrepperState(final MapDbPrepperState prepperState) { + final Collection> serviceDependencyRecords = new HashSet<>(); - /** - * This function is used to iterate over the current window and find parent/child relationships in the current and - * previous windows. - */ - private final BiFunction> relationshipIterationFunction = new BiFunction>() { - @Override - public Stream apply(byte[] s, ServiceMapStateData serviceMapStateData) { - return lookupParentSpan(serviceMapStateData, true); - } - }; - - private Stream lookupParentSpan(final ServiceMapStateData serviceMapStateData, final boolean checkPrev) { - if (serviceMapStateData.parentSpanId != null) { - final ServiceMapStateData parentStateData = getParentStateData(serviceMapStateData.parentSpanId, checkPrev); - final String traceGroupName = getTraceGroupName(serviceMapStateData.traceId); - if (traceGroupName != null && parentStateData != null && !parentStateData.serviceName.equals(serviceMapStateData.serviceName)) { - return Stream.of( - ServiceMapRelationship.newDestinationRelationship(parentStateData.serviceName, parentStateData.spanKind, serviceMapStateData.serviceName, serviceMapStateData.name, traceGroupName), - //This extra edge is added for compatibility of the index for both stateless and stateful preppers - ServiceMapRelationship.newTargetRelationship(serviceMapStateData.serviceName, serviceMapStateData.spanKind, serviceMapStateData.serviceName, serviceMapStateData.name, traceGroupName) - ); - } - } - return Stream.empty(); - } + if (prepperState.getAll() != null && !prepperState.getAll().isEmpty()) { + prepperState.getIterator(preppersCreated.get(), thisPrepperId).forEachRemaining(entry -> { + final ServiceMapStateData child = entry.getValue(); - /** - * Checks both current and previous windows for the given parent span id - * - * @param spanId - * @return ServiceMapStateData for the parent span, if exists. Otherwise null - */ - private ServiceMapStateData getParentStateData(final byte[] spanId, final boolean checkPrev) { - try { - final ServiceMapStateData serviceMapStateData = currentWindow.get(spanId); - return serviceMapStateData != null ? serviceMapStateData : checkPrev ? previousWindow.get(spanId) : null; - } catch (RuntimeException e) { - LOG.error("Caught exception trying to get parent state data", e); - return null; + if (child.parentSpanId == null) { + return; + } + + ServiceMapStateData parent = currentWindow.get(child.parentSpanId); + if (parent == null) { + parent = previousWindow.get(child.parentSpanId); + } + + + final String traceGroupName = getTraceGroupName(child.traceId); + if (traceGroupName == null || parent == null || parent.serviceName.equals(child.serviceName)) { + return; + } + + final ServiceMapRelationship destinationRelationship = + ServiceMapRelationship.newDestinationRelationship(parent.serviceName, + parent.spanKind, child.serviceName, child.name, traceGroupName); + final ServiceMapRelationship targetRelationship = ServiceMapRelationship.newTargetRelationship(child.serviceName, + child.spanKind, child.serviceName, child.name, traceGroupName); + + + // check if relationshipState has it + if (!RELATIONSHIP_STATE.contains(destinationRelationship)) { + try { + serviceDependencyRecords.add(new Record<>(OBJECT_MAPPER.writeValueAsString(destinationRelationship))); + RELATIONSHIP_STATE.add(destinationRelationship); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + if (!RELATIONSHIP_STATE.contains(targetRelationship)) { + try { + serviceDependencyRecords.add(new Record<>(OBJECT_MAPPER.writeValueAsString(targetRelationship))); + RELATIONSHIP_STATE.add(targetRelationship); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); } + + return serviceDependencyRecords; } /** @@ -287,64 +280,30 @@ public void shutdown() { // TODO: Temp code, complex instance creation logic should be moved to a separate class static void resetStaticCounters() { preppersCreated.set(0); - edgeEvaluationLatch = null; } - /** - * Indicate/notify that this instance has finished evaluating edges - */ - private void doneEvaluatingEdges() { - edgeEvaluationLatch.countDown(); - } - - /** - * Wait on all instances to finish evaluating edges - * - * @throws InterruptedException - */ - private void waitForEvaluationFinish() throws InterruptedException { - edgeEvaluationLatch.await(); - } - - /** - * Indicate that window rotation is complete - */ - private void doneRotatingWindows() { - windowRotationLatch.countDown(); - } - - /** - * Wait on window rotation to complete - * - * @throws InterruptedException - */ - private void waitForRotationFinish() throws InterruptedException { - windowRotationLatch.await(); - } - - /** - * Reset state that indicates whether edge evaluation and window rotation is complete - */ - private void resetWorkState() { - windowRotationLatch = new CountDownLatch(1); - edgeEvaluationLatch = new CountDownLatch(preppersCreated.get()); - } /** * Rotate windows for prepper state */ - private void rotateWindows() { - LOG.debug("Rotating windows at " + clock.instant().toString()); - previousWindow.delete(); - previousTraceGroupWindow.delete(); + private void rotateWindows() throws InterruptedException { + LOG.info("Rotating service map windows at " + clock.instant().toString()); + + MapDbPrepperState tempWindow = previousWindow; previousWindow = currentWindow; - currentWindow = new MapDbPrepperState<>(dbPath, getNewDbName(), processWorkers); + currentWindow = tempWindow; + currentWindow.clear(); + + tempWindow = previousTraceGroupWindow; previousTraceGroupWindow = currentTraceGroupWindow; - currentTraceGroupWindow = new MapDbPrepperState<>(dbPath, getNewTraceDbName(), processWorkers); + currentTraceGroupWindow = tempWindow; + currentTraceGroupWindow.clear(); + previousTimestamp = clock.millis(); - doneRotatingWindows(); + LOG.info("Done rotating service map windows"); } + /** * @return Spans database size in bytes */ diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java index 1f59bfd521..e32436f568 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java @@ -92,8 +92,8 @@ public void testTraceGroups() throws Exception { Mockito.when(clock.instant()).thenReturn(Instant.now()); ExecutorService threadpool = Executors.newCachedThreadPool(); final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 16, PLUGIN_SETTING); - final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 16, PLUGIN_SETTING); + final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); + final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); final byte[] rootSpanId1 = ServiceMapTestUtils.getRandomBytes(8); final byte[] rootSpanId2 = ServiceMapTestUtils.getRandomBytes(8); @@ -194,13 +194,11 @@ public void testTraceGroups() throws Exception { new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") .add(ServiceMapStatefulPrepper.SPANS_DB_SIZE).toString()); Assert.assertEquals(1, spansDbSizeMeasurement.size()); - Assert.assertEquals(2097152, spansDbSizeMeasurement.get(0).getValue(), 0); final List traceGroupDbSizeMeasurement = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") .add(ServiceMapStatefulPrepper.TRACE_GROUP_DB_SIZE).toString()); Assert.assertEquals(1, traceGroupDbSizeMeasurement.size()); - Assert.assertEquals(2097152, traceGroupDbSizeMeasurement.get(0).getValue(), 0); //Make sure that future relationships that are equivalent are caught by cache From 1c7f27e1e8a8c5a8b954dc33840d6560e3114c1b Mon Sep 17 00:00:00 2001 From: Bala Yadav <55510769+yadavcbala@users.noreply.github.com> Date: Fri, 18 Jun 2021 11:56:47 -0500 Subject: [PATCH 087/192] Metric dimension (#673) * Add dimensions to all metrics Signed-off-by: Bala Yadav * Update java doc Signed-off-by: Bala Yadav * Update string null condition to use StringUtils Signed-off-by: Bala Yadav --- .../amazon/dataprepper/metrics/MetricNames.java | 7 +++++++ .../java/com/amazon/dataprepper/DataPrepper.java | 12 ++++++++++++ .../parser/model/MetricRegistryType.java | 15 ++++++++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java index 5204b085b5..cba16007c2 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java @@ -2,6 +2,13 @@ public class MetricNames { private MetricNames() {} + + /** + * Metric dimension representing service name. + * Applicable to all components + */ + public static final String SERVICE_NAME = "serviceName"; + /** * Metric representing the ingress of records to a pipeline component. * Applicable to preppers and sinks diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java index 3698d55a7c..c00bc67955 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepper.java @@ -12,6 +12,7 @@ import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.core.instrument.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +28,8 @@ */ public class DataPrepper { private static final Logger LOG = LoggerFactory.getLogger(DataPrepper.class); + private static final String DATAPREPPER_SERVICE_NAME = "DATAPREPPER_SERVICE_NAME"; + private static final String DEFAULT_SERVICE_NAME = "dataprepper"; private static final CompositeMeterRegistry systemMeterRegistry = new CompositeMeterRegistry(); @@ -55,6 +58,15 @@ public static void configureWithDefaults() { configureMeterRegistry(); } + /** + * returns serviceName if exists or default serviceName + * @return serviceName for data-prepper + */ + public static String getServiceNameForMetrics() { + final String serviceName = System.getenv(DATAPREPPER_SERVICE_NAME); + return StringUtils.isNotBlank(serviceName) ? serviceName : DEFAULT_SERVICE_NAME; + } + public static DataPrepper getInstance() { if (dataPrepper == null) { synchronized (DataPrepper.class) { diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java index 29417f990e..c33038d3d7 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/parser/model/MetricRegistryType.java @@ -2,9 +2,14 @@ import com.amazon.dataprepper.pipeline.server.CloudWatchMeterRegistryProvider; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; +import java.util.Arrays; + +import static com.amazon.dataprepper.DataPrepper.getServiceNameForMetrics; +import static com.amazon.dataprepper.metrics.MetricNames.SERVICE_NAME; import static java.lang.String.format; public enum MetricRegistryType { @@ -12,14 +17,18 @@ public enum MetricRegistryType { CloudWatch; public static MeterRegistry getDefaultMeterRegistryForType(final MetricRegistryType metricRegistryType) { + MeterRegistry meterRegistry = null; switch (metricRegistryType) { case Prometheus: - return new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + meterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + break; case CloudWatch: - return new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); + meterRegistry = new CloudWatchMeterRegistryProvider().getCloudWatchMeterRegistry(); + break; default: throw new IllegalArgumentException(format("Invalid metricRegistryType %s", metricRegistryType)); - } + meterRegistry.config().commonTags(Arrays.asList(Tag.of(SERVICE_NAME, getServiceNameForMetrics()))); + return meterRegistry; } } \ No newline at end of file From 2e86ecc95dab68822efd4147bf0d15c92601bfcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:07:12 +0000 Subject: [PATCH 088/192] Bump assertj-core in /data-prepper-plugins/otel-trace-source Bumps [assertj-core](https://github.com/assertj/assertj-core) from 3.19.0 to 3.20.2. - [Release notes](https://github.com/assertj/assertj-core/releases) - [Commits](https://github.com/assertj/assertj-core/compare/assertj-core-3.19.0...assertj-core-3.20.2) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 89e434407d..f816dcd8a0 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - testImplementation 'org.assertj:assertj-core:3.19.0' + testImplementation 'org.assertj:assertj-core:3.20.2' testImplementation "org.mockito:mockito-inline:3.11.1" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation(platform('org.junit:junit-bom:5.7.2')) From ab9a20dcd3fae2b214eb4ee6e2ba885014c33d88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jun 2021 04:07:52 +0000 Subject: [PATCH 089/192] Bump assertj-core in /data-prepper-plugins/otel-trace-raw-prepper Bumps [assertj-core](https://github.com/assertj/assertj-core) from 3.19.0 to 3.20.2. - [Release notes](https://github.com/assertj/assertj-core/releases) - [Commits](https://github.com/assertj/assertj-core/compare/assertj-core-3.19.0...assertj-core-3.20.2) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 63191d55bf..03373eaa08 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' - testImplementation 'org.assertj:assertj-core:3.19.0' + testImplementation 'org.assertj:assertj-core:3.20.2' testImplementation "org.mockito:mockito-inline:3.11.1" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.awaitility:awaitility:4.1.0" From 634633e97bf8a8ebbe9153c8204d82d4448ac1bc Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Fri, 25 Jun 2021 19:10:42 +0000 Subject: [PATCH 090/192] Refactor of OTelTraceRawPrepper to improve performance. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../model/buffer/AbstractBuffer.java | 43 ++-- .../raw-span-e2e-pipeline-latest-release.yml | 1 - .../resources/raw-span-e2e-pipeline.yml | 1 - .../amazon/dataprepper/pipeline/Pipeline.java | 3 +- .../otel-trace-raw-prepper/README.md | 1 - .../oteltrace/OTelTraceRawPrepper.java | 209 ++++++++---------- .../oteltrace/OtelTraceRawPrepperConfig.java | 4 +- .../oteltrace/model/OTelProtoHelper.java | 11 +- .../oteltrace/OTelTraceRawPrepperTest.java | 18 +- .../oteltrace/OTelTraceGrpcService.java | 12 +- 10 files changed, 139 insertions(+), 164 deletions(-) diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java index 5fcb9fcfd2..46cf6a9090 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java @@ -1,18 +1,19 @@ package com.amazon.dataprepper.model.buffer; -import com.amazon.dataprepper.model.CheckpointState; -import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.metrics.MetricNames; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.CheckpointState; +import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; + import java.util.Collection; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; - /** * Abstract implementation of the Buffer interface to record boilerplate metrics */ @@ -50,41 +51,37 @@ private AbstractBuffer(final PluginMetrics pluginMetrics, final String pipelineN /** * Records metrics for ingress, time elapsed, and timeouts, while calling the doWrite method * to perform the actual write - * @param record the Record to add + * + * @param record the Record to add * @param timeoutInMillis how long to wait before giving up * @throws TimeoutException */ @Override public void write(T record, int timeoutInMillis) throws TimeoutException { + long startTime = System.nanoTime(); + try { - writeTimer.record(() -> { - try { - doWrite(record, timeoutInMillis); - } catch (TimeoutException e) { - writeTimeoutCounter.increment(); - throw new RuntimeException(e); - } - }); + doWrite(record, timeoutInMillis); recordsWrittenCounter.increment(); - } catch (RuntimeException e) { - if(e.getCause() instanceof TimeoutException) { - throw (TimeoutException) e.getCause(); - } else { - throw e; - } + } catch (TimeoutException e) { + writeTimeoutCounter.increment(); + throw e; + } finally { + writeTimer.record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } } /** * Records egress and time elapsed metrics, while calling the doRead function to * do the actual read + * * @param timeoutInMillis how long to wait before giving up * @return Records collection and checkpoint state read from the buffer */ @Override public Map.Entry, CheckpointState> read(int timeoutInMillis) { final Map.Entry, CheckpointState> readResult = readTimer.record(() -> doRead(timeoutInMillis)); - recordsReadCounter.increment(readResult.getKey().size()*1.0); + recordsReadCounter.increment(readResult.getKey().size() * 1.0); recordsInFlight.addAndGet(readResult.getValue().getNumRecordsToBeChecked()); return readResult; } @@ -103,7 +100,8 @@ protected int getRecordsInFlight() { /** * This method should implement the logic for writing to the buffer - * @param record Record to write to buffer + * + * @param record Record to write to buffer * @param timeoutInMillis Timeout for write operation in millis * @throws TimeoutException */ @@ -111,6 +109,7 @@ protected int getRecordsInFlight() { /** * This method should implement the logic for reading from the buffer + * * @param timeoutInMillis Timeout in millis * @return Records collection and checkpoint state read from the buffer */ diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml index 0d7df44b0e..bcbed8a970 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -16,7 +16,6 @@ raw-pipeline: name: "entry-pipeline" prepper: - otel_trace_raw_prepper: - root_span_flush_delay: 3 trace_flush_interval: 5 sink: - elasticsearch: diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 6b93c3039b..24f8417df0 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -11,7 +11,6 @@ raw-pipeline: name: "entry-pipeline" prepper: - otel_trace_raw_prepper: - root_span_flush_delay: 1 trace_flush_interval: 5 - otel_trace_group_prepper: hosts: [ "https://node-0.example.com:9200" ] diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java index bc1eb3c8a8..4ce10787f2 100644 --- a/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java +++ b/data-prepper-core/src/main/java/com/amazon/dataprepper/pipeline/Pipeline.java @@ -79,7 +79,8 @@ public Pipeline( this.prepperExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(prepperThreads, new PipelineThreadFactory(format("%s-prepper-worker", name)), this); - this.sinkExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(sinks.size(), + // TODO: allow this to be configurable as well? + this.sinkExecutorService = PipelineThreadPoolExecutor.newFixedThreadPool(prepperThreads, new PipelineThreadFactory(format("%s-sink-worker", name)), this); stopRequested = false; diff --git a/data-prepper-plugins/otel-trace-raw-prepper/README.md b/data-prepper-plugins/otel-trace-raw-prepper/README.md index 0b6a0716ba..674ce093ce 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/README.md +++ b/data-prepper-plugins/otel-trace-raw-prepper/README.md @@ -11,7 +11,6 @@ prepper: ## Configuration -* `root_span_flush_delay`: An `int` represents the time interval in seconds to flush all the root spans in the prepper together with their descendants. Default to 30. * `trace_flush_interval`: An `int` represents the time interval in seconds to flush all the descendant spans without any root span. Default to 180. ## Metrics diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java index 232c34f6a6..bd18d4de30 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java @@ -11,12 +11,10 @@ import com.amazon.dataprepper.plugins.prepper.oteltrace.model.RawSpanSet; import com.amazon.dataprepper.plugins.prepper.oteltrace.model.TraceGroup; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableList; -import com.google.common.primitives.Ints; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.util.StringUtils; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; @@ -29,18 +27,15 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.DelayQueue; -import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @DataPrepperPlugin(name = "otel_trace_raw_prepper", type = PluginType.PREPPER) public class OTelTraceRawPrepper extends AbstractPrepper, Record> { - private static final long SEC_TO_MILLIS = 1_000L; private static final Logger LOG = LoggerFactory.getLogger(OTelTraceRawPrepper.class); @@ -49,17 +44,11 @@ public class OTelTraceRawPrepper extends AbstractPrepper delayedParentSpanQueue = new DelayQueue<>(); - // TODO: replace with file store, e.g. MapDB? - // TODO: introduce a gauge to monitor the size private final Map traceIdRawSpanSetMap = new ConcurrentHashMap<>(); private final Cache traceIdTraceGroupCache; @@ -69,15 +58,13 @@ public class OTelTraceRawPrepper extends AbstractPrepper> doExecute(Collection> records) { + final List rawSpans = new LinkedList<>(); + for (Record ets : records) { for (ResourceSpans rs : ets.getData().getResourceSpansList()) { try { @@ -108,7 +97,8 @@ public Collection> doExecute(Collection> doExecute(Collection rawSpans = new LinkedList<>(); - rawSpans.addAll(getTracesToFlushByRootSpan()); rawSpans.addAll(getTracesToFlushByGarbageCollection()); return convertRawSpansToJsonRecords(rawSpans); } - private void processRawSpan(final RawSpan rawSpan) { - if (rawSpan.getParentSpanId() == null || "".equals(rawSpan.getParentSpanId())) { - processRootRawSpan(rawSpan); + /** + * Branching logic to handle root and child spans. + * A root span is the first span of a trace, it has no parentSpanId. + * + * @param rawSpan Span to be evaluated + * @param spanSet Collection to insert spans to + */ + private void processRawSpan(final RawSpan rawSpan, final Collection spanSet) { + if (StringUtils.isBlank(rawSpan.getParentSpanId())) { + final List rootSpanAndChildren = processRootSpan(rawSpan); + spanSet.addAll(rootSpanAndChildren); } else { - processDescendantRawSpan(rawSpan); + final Optional populatedChildSpanOptional = processChildSpan(rawSpan); + if (populatedChildSpanOptional.isPresent()) { + spanSet.add(populatedChildSpanOptional.get()); + } } } - private void processRootRawSpan(final RawSpan rawSpan) { - // TODO: flush descendants here to get rid of DelayQueue - // TODO: safe-guard against null traceGroup for rootSpan? - traceIdTraceGroupCache.put(rawSpan.getTraceId(), rawSpan.getTraceGroup()); - final long now = System.currentTimeMillis(); - final long nowPlusOffset = now + rootSpanFlushDelay; - final DelayedParentSpan delayedParentSpan = new DelayedParentSpan(rawSpan, nowPlusOffset); - delayedParentSpanQueue.add(delayedParentSpan); - } + /** + * Retrieves all child spans from memory and returns them as a set with the root span. + * Also adds an entry to the traceID cache so that later child spans can be tagged, + * in the case where a child span is processed AFTER the root span. + * + * @param parentSpan + * @return List containing root span, along with any child spans that have already been processed. + */ + private List processRootSpan(final RawSpan parentSpan) { + traceIdTraceGroupCache.put(parentSpan.getTraceId(), parentSpan.getTraceGroup()); + + final List recordsToFlush = new LinkedList<>(); + recordsToFlush.add(parentSpan); + + final TraceGroup traceGroup = parentSpan.getTraceGroup(); + final String parentSpanTraceId = parentSpan.getTraceId(); - private void processDescendantRawSpan(final RawSpan rawSpan) { - traceIdRawSpanSetMap.compute(rawSpan.getTraceId(), (traceId, rawSpanSet) -> { - if (rawSpanSet == null) { - rawSpanSet = new RawSpanSet(); + final RawSpanSet rawSpanSet = traceIdRawSpanSetMap.get(parentSpanTraceId); + if (rawSpanSet != null) { + for (final RawSpan rawSpan : rawSpanSet.getRawSpans()) { + rawSpan.setTraceGroup(traceGroup); + recordsToFlush.add(rawSpan); } - rawSpanSet.addRawSpan(rawSpan); - return rawSpanSet; - }); + + traceIdRawSpanSetMap.remove(parentSpanTraceId); + } + + return recordsToFlush; + } + + /** + * Attempts to populate the traceGroup of the child span by fetching from a cache. If the traceGroup is not in the cache, + * the child span is kept in memory to be populated when its corresponding root span arrives. + * + * @param childSpan + * @return Optional containing childSpan if its traceGroup is in memory, otherwise an empty Optional + */ + private Optional processChildSpan(final RawSpan childSpan) { + final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(childSpan.getTraceId()); + + if (traceGroup != null) { + childSpan.setTraceGroup(traceGroup); + return Optional.of(childSpan); + } else { + traceIdRawSpanSetMap.compute(childSpan.getTraceId(), (traceId, rawSpanSet) -> { + if (rawSpanSet == null) { + rawSpanSet = new RawSpanSet(); + } + rawSpanSet.addRawSpan(childSpan); + return rawSpanSet; + }); + + return Optional.empty(); + } } private List> convertRawSpansToJsonRecords(final List rawSpans) { @@ -174,51 +209,32 @@ private List> convertRawSpansToJsonRecords(final List ra return records; } - private List getTracesToFlushByRootSpan() { - final List recordsToFlush = new LinkedList<>(); - - for (DelayedParentSpan delayedParentSpan; (delayedParentSpan = delayedParentSpanQueue.poll()) != null;) { - RawSpan parentSpan = delayedParentSpan.getRawSpan(); - recordsToFlush.add(parentSpan); - - TraceGroup traceGroup = parentSpan.getTraceGroup(); - String parentSpanTraceId = parentSpan.getTraceId(); - - RawSpanSet rawSpanSet = traceIdRawSpanSetMap.get(parentSpanTraceId); - if (rawSpanSet != null) { - for (RawSpan rawSpan : rawSpanSet.getRawSpans()) { - rawSpan.setTraceGroup(traceGroup); - recordsToFlush.add(rawSpan); - } - - traceIdRawSpanSetMap.remove(parentSpanTraceId); - } - } - - return recordsToFlush; - } - - + /** + * Periodically flush spans from memory. Typically all spans of a trace are written + * once the trace's root span arrives, however some child spans my arrive after the root span. + * This method ensures "orphaned" child spans are eventually flushed from memory. + * @return List of RawSpans to be sent down the pipeline + */ private List getTracesToFlushByGarbageCollection() { final List recordsToFlush = new LinkedList<>(); if (shouldGarbageCollect()) { - boolean isLockAcquired = traceFlushLock.tryLock(); + final boolean isLockAcquired = traceFlushLock.tryLock(); if (isLockAcquired) { try { final long now = System.currentTimeMillis(); lastTraceFlushTime = now; - Iterator> entryIterator = traceIdRawSpanSetMap.entrySet().iterator(); + final Iterator> entryIterator = traceIdRawSpanSetMap.entrySet().iterator(); while (entryIterator.hasNext()) { - Map.Entry entry = entryIterator.next(); - String traceId = entry.getKey(); - TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(traceId); - RawSpanSet rawSpanSet = entry.getValue(); - long traceTime = rawSpanSet.getTimeSeen(); - if (now - traceTime >= traceFlushInterval) { - Set rawSpans = rawSpanSet.getRawSpans(); + final Map.Entry entry = entryIterator.next(); + final String traceId = entry.getKey(); + final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(traceId); + final RawSpanSet rawSpanSet = entry.getValue(); + final long traceTime = rawSpanSet.getTimeSeen(); + if (now - traceTime >= traceFlushInterval || isShuttingDown) { + final Set rawSpans = rawSpanSet.getRawSpans(); if (traceGroup != null) { rawSpans.forEach(rawSpan -> { rawSpan.setTraceGroup(traceGroup); @@ -247,11 +263,11 @@ private List getTracesToFlushByGarbageCollection() { } private boolean shouldGarbageCollect() { - return System.currentTimeMillis() - lastTraceFlushTime >= traceFlushInterval; + return System.currentTimeMillis() - lastTraceFlushTime >= traceFlushInterval || isShuttingDown; } /** - * Re-enqueues all spans with time "0" so that all will be available for consumption. + * Forces a flush of all spans in memory */ @Override public void prepareForShutdown() { @@ -259,16 +275,8 @@ public void prepareForShutdown() { if (isLockAcquired) { try { - LOG.info("Preparing for shutdown, re-enqueueing {} spans", delayedParentSpanQueue.size()); - Iterator iterator = delayedParentSpanQueue.iterator(); - List delayedParentSpanList = ImmutableList.copyOf(iterator); - - delayedParentSpanQueue.clear(); - - for (final DelayedParentSpan delayedParentSpan : delayedParentSpanList) { - final DelayedParentSpan newSpan = new DelayedParentSpan(delayedParentSpan.getRawSpan(), 0L); - delayedParentSpanQueue.add(newSpan); - } + LOG.info("Preparing for shutdown, will attempt to flush {} spans", traceIdRawSpanSetMap.size()); + isShuttingDown = true; } finally { prepareForShutdownLock.unlock(); } @@ -277,38 +285,11 @@ public void prepareForShutdown() { @Override public boolean isReadyForShutdown() { - return delayedParentSpanQueue.isEmpty() - && traceIdRawSpanSetMap.isEmpty(); + return traceIdRawSpanSetMap.isEmpty(); } @Override public void shutdown() { traceIdTraceGroupCache.cleanUp(); } - - class DelayedParentSpan implements Delayed { - private RawSpan rawSpan; - private long delayTime; - - public DelayedParentSpan(RawSpan rawSpan, long startTime) { - this.rawSpan = rawSpan; - this.delayTime = startTime; - } - - @Override - public long getDelay(TimeUnit unit) { - long diff = delayTime - System.currentTimeMillis(); - return unit.convert(diff, TimeUnit.MILLISECONDS); - } - - @Override - public int compareTo(Delayed o) { - return Ints.saturatedCast( - this.delayTime - ((DelayedParentSpan) o).delayTime); - } - - public RawSpan getRawSpan() { - return rawSpan; - } - } } diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java index 150474ef20..6091cad530 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java @@ -3,8 +3,6 @@ public class OtelTraceRawPrepperConfig { static final String TRACE_FLUSH_INTERVAL = "trace_flush_interval"; static final long DEFAULT_TG_FLUSH_INTERVAL_SEC = 180L; - static final String ROOT_SPAN_FLUSH_DELAY = "root_span_flush_delay"; - static final long DEFAULT_ROOT_SPAN_FLUSH_DELAY_SEC = 30L; - static final long DEFAULT_TRACE_ID_TTL_SEC = 300L; + static final long DEFAULT_TRACE_ID_TTL_SEC = 15L; static final long MAX_TRACE_ID_CACHE_SIZE = 1000_000L; } diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java index df2962e73c..5ae92afe43 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java @@ -8,8 +8,6 @@ import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -19,9 +17,7 @@ public final class OTelProtoHelper { - private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final BigDecimal MILLIS_TO_NANOS = new BigDecimal(1_000_000); - private static final BigDecimal SEC_TO_MILLIS = new BigDecimal(1_000); + private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final String SERVICE_NAME = "service.name"; private static final String SPAN_ATTRIBUTES = "span.attributes"; static final String RESOURCE_ATTRIBUTES = "resource.attributes"; @@ -136,10 +132,7 @@ public static Map getSpanStatusAttributes(final Status status) { } private static String convertUnixNanosToISO8601(final long unixNano) { - final BigDecimal nanos = new BigDecimal(unixNano); - final long epochSeconds = nanos.divide(MILLIS_TO_NANOS.multiply(SEC_TO_MILLIS), RoundingMode.DOWN).longValue(); - final int nanoAdj = nanos.remainder(MILLIS_TO_NANOS.multiply(SEC_TO_MILLIS)).intValue(); - return Instant.ofEpochSecond(epochSeconds, nanoAdj).toString(); + return Instant.ofEpochSecond(0L, unixNano).toString(); } public static String getStartTimeISO8601(final Span span) { diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java index cf4b846e8e..d0d8d7cac2 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java +++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java @@ -50,7 +50,6 @@ public class OTelTraceRawPrepperTest { private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final long TEST_TRACE_FLUSH_INTERVAL = 3L; - private static final long TEST_ROOT_SPAN_FLUSH_DELAY = 1L; private static final int TEST_CONCURRENCY_SCALE = 2; private static final String TEST_REQUEST_ONE_FULL_TRACE_GROUP_JSON_FILE = "sample-request-one-full-trace-group.json"; @@ -71,7 +70,6 @@ public void setup() { "OTelTrace", new HashMap() {{ put(OtelTraceRawPrepperConfig.TRACE_FLUSH_INTERVAL, TEST_TRACE_FLUSH_INTERVAL); - put(OtelTraceRawPrepperConfig.ROOT_SPAN_FLUSH_DELAY, TEST_ROOT_SPAN_FLUSH_DELAY); }}); pluginSetting.setPipelineName("pipelineOTelTrace"); pluginSetting.setProcessWorkers(TEST_CONCURRENCY_SCALE); @@ -130,12 +128,12 @@ public void testEmptySpans() { @Test public void testExportRequestFlushByParentSpan() throws IOException { final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE); - oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); - await().atMost(2 * TEST_ROOT_SPAN_FLUSH_DELAY, TimeUnit.SECONDS).untilAsserted(() -> { - final List> processedRecords = (List>) oTelTraceRawPrepper.doExecute(Collections.emptyList()); - Assertions.assertThat(processedRecords.size()).isEqualTo(6); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); - }); + final List> processedRecords = (List>)oTelTraceRawPrepper.doExecute( + Collections.singletonList(new Record<>(exportTraceServiceRequest)) + ); + + Assertions.assertThat(processedRecords.size()).isEqualTo(6); + Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); } @Test @@ -149,7 +147,7 @@ public void testExportRequestFlushByParentSpanMultiThread() throws IOException, for (Future>> future : futures) { processedRecords.addAll(future.get()); } - await().atMost(2 * TEST_ROOT_SPAN_FLUSH_DELAY, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(2 * TEST_TRACE_FLUSH_INTERVAL, TimeUnit.SECONDS).untilAsserted(() -> { List>>> futureList = submitExportTraceServiceRequests(Collections.emptyList()); for (Future>> future : futureList) { processedRecords.addAll(future.get()); @@ -197,7 +195,7 @@ public void testPrepareForShutdown() throws Exception { assertTrue(oTelTraceRawPrepper.isReadyForShutdown()); // Add records to memory/queue - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE); + final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_MISSING_ROOTS_JSON_FILE); oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); // Assert records exist in memory diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index 49f05f0bb2..43d4e31caa 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -3,6 +3,7 @@ import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.buffer.Buffer; import com.amazon.dataprepper.model.record.Record; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import io.micrometer.core.instrument.Counter; @@ -37,16 +38,23 @@ public OTelTraceGrpcService(int bufferWriteTimeoutInMillis, @Override public void export(ExportTraceServiceRequest request, StreamObserver responseObserver) { + requestsReceivedCounter.increment(); + + if (Context.current().isCancelled()) { + requestTimeoutCounter.increment(); + responseObserver.onError(Status.CANCELLED.withDescription("Cancelled by client").asRuntimeException()); + return; + } + try { - requestsReceivedCounter.increment(); buffer.write(new Record<>(request), bufferWriteTimeoutInMillis); responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build()); responseObserver.onCompleted(); } catch (TimeoutException e) { + requestTimeoutCounter.increment(); responseObserver .onError(Status.RESOURCE_EXHAUSTED.withDescription("Buffer is full, request timed out.") .asException()); - requestTimeoutCounter.increment(); } } } From aa94453aa6c21b3508d50edbb45edeb4b2b477bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 04:04:44 +0000 Subject: [PATCH 091/192] Bump micrometer-core in /data-prepper-plugins/service-map-stateful Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index c721ea842f..dd00a413ed 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation project(':data-prepper-plugins:mapdb-prepper-state') testImplementation project(':data-prepper-api').sourceSets.test.output - implementation "io.micrometer:micrometer-core:1.7.0" + implementation "io.micrometer:micrometer-core:1.7.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" From 5d428b6cdad8d4f0a36c4c0f06fd28e474543958 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 04:04:53 +0000 Subject: [PATCH 092/192] Bump mockito-junit-jupiter in /data-prepper-plugins/otel-trace-source Bumps [mockito-junit-jupiter](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-junit-jupiter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 89e434407d..26ff1f7e33 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -20,7 +20,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects testImplementation("org.mockito:mockito-core:3.11.1") - testImplementation("org.mockito:mockito-junit-jupiter:3.11.1") + testImplementation("org.mockito:mockito-junit-jupiter:3.11.2") } test { From f69b90200c092e886cbbe85c06171145afd4cd6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 04:05:37 +0000 Subject: [PATCH 093/192] Bump kotlin-stdlib in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib](https://github.com/JetBrains/kotlin) from 1.5.10 to 1.5.20. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/v1.5.20/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.10...v1.5.20) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin:kotlin-stdlib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index c61de6a6b2..3ab8535369 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.10' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.20' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.10' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output From 895efcefc40daa0aa1c45501b104b5e3f8bf8068 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 04:06:01 +0000 Subject: [PATCH 094/192] Bump micrometer-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index bd50739646..9e3553f0c5 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - implementation "io.micrometer:micrometer-core:1.7.0" + implementation "io.micrometer:micrometer-core:1.7.1" testImplementation "org.mockito:mockito-core:3.11.1" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From 8d8f556f495ad5412aab730313fb840986c3b538 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 29 Jun 2021 18:33:25 +0000 Subject: [PATCH 095/192] Re-added config value to e2e test files for compatibility. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../resources/raw-span-e2e-pipeline-latest-release.yml | 1 + .../src/integrationTest/resources/raw-span-e2e-pipeline.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml index bcbed8a970..a21c367f24 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -16,6 +16,7 @@ raw-pipeline: name: "entry-pipeline" prepper: - otel_trace_raw_prepper: + root_span_flush_delay: 3 # TODO: remove after 1.1 release trace_flush_interval: 5 sink: - elasticsearch: diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml index 24f8417df0..e4278e7614 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -11,6 +11,7 @@ raw-pipeline: name: "entry-pipeline" prepper: - otel_trace_raw_prepper: + root_span_flush_delay: 1 # TODO: remove after 1.1 release trace_flush_interval: 5 - otel_trace_group_prepper: hosts: [ "https://node-0.example.com:9200" ] From 43ffdf94d3c352a8bf37b1b8b01804d9fb798bad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:08:00 +0000 Subject: [PATCH 096/192] Bump mockito-inline in /data-prepper-plugins/service-map-stateful Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index dd00a413ed..07e35db802 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-inline:3.11.1" + testImplementation "org.mockito:mockito-inline:3.11.2" } jacocoTestCoverageVerification { From 98d4f909847767d1dc4f8266f42f481405630bdb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:08:03 +0000 Subject: [PATCH 097/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-source Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index f816dcd8a0..4f1ab76816 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.20.2' - testImplementation "org.mockito:mockito-inline:3.11.1" + testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') From c56c3f12bc28ddf6c43e89c20ad97c7bd9d01ec4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:08:25 +0000 Subject: [PATCH 098/192] Bump mockito-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [mockito-core](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index 9e3553f0c5..b41798a363 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -14,6 +14,6 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "io.micrometer:micrometer-core:1.7.1" - testImplementation "org.mockito:mockito-core:3.11.1" + testImplementation "org.mockito:mockito-core:3.11.2" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From 918bc32ee9dfd845611e99f64d28448174069389 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:08:25 +0000 Subject: [PATCH 099/192] Bump mockito-core from 3.11.1 to 3.11.2 in /data-prepper-core Bumps [mockito-core](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index b9bb47fb19..2eef4ad945 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" implementation "software.amazon.awssdk:cloudwatch:2.16.83" testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation "org.mockito:mockito-core:3.11.1" + testImplementation "org.mockito:mockito-core:3.11.2" } jar { From bbf650e6f65f8754067139f5f73ca1f29972ce9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:09:38 +0000 Subject: [PATCH 100/192] Bump mockito-inline in /data-prepper-plugins/peer-forwarder Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 786f2114f1..18d3889d66 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" - testImplementation "org.mockito:mockito-inline:3.11.1" + testImplementation "org.mockito:mockito-inline:3.11.2" } jacocoTestCoverageVerification { From 85df7e90abbbd57b8b15c6121c261706c307ac96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:09:38 +0000 Subject: [PATCH 101/192] Bump mockito-core in /data-prepper-plugins/otel-trace-source Bumps [mockito-core](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 439197e5e6..5d06cd19f4 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -19,7 +19,7 @@ dependencies { testImplementation(platform('org.junit:junit-bom:5.7.2')) testImplementation('org.junit.jupiter:junit-jupiter') // TODO: update versionMap to use a higher version of mockito for all subprojects - testImplementation("org.mockito:mockito-core:3.11.1") + testImplementation("org.mockito:mockito-core:3.11.2") testImplementation("org.mockito:mockito-junit-jupiter:3.11.2") } From 69d300bc45ecaaed48bcd68a27d3151ce3cbde47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:09:58 +0000 Subject: [PATCH 102/192] Bump micrometer-core from 1.7.0 to 1.7.1 in /data-prepper-api Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index 20b3c6b42d..f680e9e11f 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -2,7 +2,7 @@ plugins { } dependencies { - implementation "io.micrometer:micrometer-core:1.7.0" + implementation "io.micrometer:micrometer-core:1.7.1" testImplementation "org.hamcrest:hamcrest:2.2" } From d68268dd48d471cf2c6a94b14dd1d148f4e16a98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:10:00 +0000 Subject: [PATCH 103/192] Bump mockito-inline in /data-prepper-plugins/otel-trace-raw-prepper Bumps [mockito-inline](https://github.com/mockito/mockito) from 3.11.1 to 3.11.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.11.1...v3.11.2) --- updated-dependencies: - dependency-name: org.mockito:mockito-inline dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 03373eaa08..148bf07cea 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' testImplementation 'org.assertj:assertj-core:3.20.2' - testImplementation "org.mockito:mockito-inline:3.11.1" + testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.awaitility:awaitility:4.1.0" } From 91763fee8fd8c2277c4b4b61e0dcc4afedd51248 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:10:00 +0000 Subject: [PATCH 104/192] Bump micrometer-core in /data-prepper-plugins/elasticsearch Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 288c562c68..b76f31e130 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -38,7 +38,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-core:1.12.7" implementation "com.amazonaws:aws-java-sdk-sts:1.11.1024" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" - implementation "io.micrometer:micrometer-core:1.7.0" + implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } From 02e4a3f09c41f6313ed93c710eafc3633d22a394 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:10:30 +0000 Subject: [PATCH 105/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria-grpc](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 03373eaa08..38e153842f 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.8.0" - implementation "com.linecorp.armeria:armeria-grpc:1.8.0" + implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' From 12cb15215fb655a153ea343561cc6f5e8a7cd07d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:10:31 +0000 Subject: [PATCH 106/192] Bump micrometer-registry-cloudwatch2 in /data-prepper-core Bumps [micrometer-registry-cloudwatch2](https://github.com/micrometer-metrics/micrometer) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-registry-cloudwatch2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 2eef4ad945..79a0b6e056 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation "org.reflections:reflections:0.9.12" implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" - implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.0" + implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.1" implementation "software.amazon.awssdk:cloudwatch:2.16.83" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.11.2" From 1fb9e11ef4cfaae68cc6cd322501eb91aa8f6b78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:11:38 +0000 Subject: [PATCH 107/192] Bump cloudwatch from 2.16.83 to 2.16.92 in /data-prepper-core Bumps cloudwatch from 2.16.83 to 2.16.92. --- updated-dependencies: - dependency-name: software.amazon.awssdk:cloudwatch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 79a0b6e056..281f621c09 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.1" - implementation "software.amazon.awssdk:cloudwatch:2.16.83" + implementation "software.amazon.awssdk:cloudwatch:2.16.92" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.11.2" } From 2dd8bdaae7014c5aa0f80ead5d29aa08b79abe53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:11:50 +0000 Subject: [PATCH 108/192] Bump aws-java-sdk-sts in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-sts](https://github.com/aws/aws-sdk-java) from 1.11.1024 to 1.12.14. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1024...1.12.14) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-sts dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index b76f31e130..3f575a5782 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation "com.amazonaws:aws-java-sdk-core:1.12.7" - implementation "com.amazonaws:aws-java-sdk-sts:1.11.1024" + implementation "com.amazonaws:aws-java-sdk-sts:1.12.14" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { From 538a3a6b27c7ecce2d8c4cb3d04f55ddfd593cae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:12:39 +0000 Subject: [PATCH 109/192] Bump armeria-grpc in /data-prepper-plugins/peer-forwarder Bumps [armeria-grpc](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 18d3889d66..2669a02c07 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -14,7 +14,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.8.0" - implementation "com.linecorp.armeria:armeria-grpc:1.8.0" + implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" testImplementation "org.mockito:mockito-inline:3.11.2" From 94f8cc65de0a3ab2756322c134fe4e872f178f17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:12:51 +0000 Subject: [PATCH 110/192] Bump kotlin-stdlib-common in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib-common](https://github.com/JetBrains/kotlin) from 1.5.10 to 1.5.20. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/v1.5.20/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.10...v1.5.20) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin:kotlin-stdlib-common dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index 3ab8535369..909edcd5f9 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.20' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.10' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.20' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output testImplementation group: 'junit', name: 'junit', version: '4.13.2' From 7a1b42b562eef02f6635029826f7eba8a98c36b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:26:59 +0000 Subject: [PATCH 111/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.12.7 to 1.12.14. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.7...1.12.14) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 3f575a5782..d91cd55734 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.12.7" + implementation "com.amazonaws:aws-java-sdk-core:1.12.14" implementation "com.amazonaws:aws-java-sdk-sts:1.12.14" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.1" From ab1fe428b418cc1bb8e1f8ed27dcdcf7a50e0e40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:27:39 +0000 Subject: [PATCH 112/192] Bump armeria from 1.8.0 to 1.9.1 in /data-prepper-plugins/peer-forwarder Bumps [armeria](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 2669a02c07..42785f0643 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':data-prepper-api') testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "com.linecorp.armeria:armeria:1.8.0" + implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" From ce86391285610cfa0c5f6d5b2cf454dfde7d1147 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:27:39 +0000 Subject: [PATCH 113/192] Bump armeria in /data-prepper-plugins/otel-trace-source Bumps [armeria](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 5d06cd19f4..e4c98c180c 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' - implementation "com.linecorp.armeria:armeria:1.8.0" + implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.8.0" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From 0edd344341a98cfeb299063cc7ce2fdc88a1b1f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:27:40 +0000 Subject: [PATCH 114/192] Bump armeria in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 3b11a018d7..5845fbcda9 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' - implementation "com.linecorp.armeria:armeria:1.8.0" + implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From c8c51a7b1b063860d432317a49e97a6ad82e0e41 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 29 Jun 2021 19:30:08 +0000 Subject: [PATCH 115/192] Changing Dependabot scans from weekly to monthly. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .github/dependabot.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d0cd1a4a76..12bf448fc7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,54 +4,54 @@ updates: - package-ecosystem: gradle directory: "/data-prepper-api" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-core" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/blocking-buffer" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/common" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/elasticsearch" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/mapdb-prepper-state" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-raw-prepper" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-group-prepper" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/otel-trace-source" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/peer-forwarder" schedule: - interval: weekly + interval: monthly - package-ecosystem: gradle directory: "/data-prepper-plugins/service-map-stateful" schedule: - interval: weekly + interval: monthly From f4661b8c61cfa6e097e819d5d428bd8810901068 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Jun 2021 20:00:41 +0000 Subject: [PATCH 116/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-source Bumps [armeria-grpc](https://github.com/line/armeria) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.8.0...armeria-1.9.1) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index e4c98c180c..ac660a3c38 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" - implementation "com.linecorp.armeria:armeria-grpc:1.8.0" + implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" testImplementation 'org.assertj:assertj-core:3.20.2' From d2772a6028f9f6d082815de0cec3665275cbb31d Mon Sep 17 00:00:00 2001 From: dinujoh <86094133+dinujoh@users.noreply.github.com> Date: Tue, 29 Jun 2021 20:33:47 -0500 Subject: [PATCH 117/192] Add support to use SSL certificate from ACM for TLS termination (#705) Currently the SSL certificates are read from local file system. This change supports using ACM certificate for SSL certificate and private key. The certificate and encrypted private key is read from the provided ACM arn and is used to set TLS parameters for the server. Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- .../otel-trace-source/build.gradle | 4 + .../data/certificate/test_cert.crt | 14 ++ .../data/certificate/test_decrypted_key.key | 15 ++ .../data/certificate/test_encrypted_key.key | 17 ++ .../acm/ACMCertificateProvider.java | 135 ++++++++++++++ .../certificate/model/Certificate.java | 29 +++ .../source/oteltrace/OTelTraceSource.java | 40 +++- .../oteltrace/OTelTraceSourceConfig.java | 43 ++++- .../acm/CertificateProviderTest.java | 104 +++++++++++ .../source/oteltrace/OTelTraceSourceTest.java | 175 +++++++++++++++--- 10 files changed, 537 insertions(+), 39 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt create mode 100644 data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key create mode 100644 data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index ac660a3c38..fd024bccb3 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,11 +8,14 @@ dependencies { implementation 'commons-codec:commons-codec:1.15' testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "org.bouncycastle:bcprov-jdk15on:1.69" + implementation "org.bouncycastle:bcpkix-jdk15on:1.69" testImplementation 'org.assertj:assertj-core:3.20.2' testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.hamcrest:hamcrest:2.2" @@ -21,6 +24,7 @@ dependencies { // TODO: update versionMap to use a higher version of mockito for all subprojects testImplementation("org.mockito:mockito-core:3.11.2") testImplementation("org.mockito:mockito-junit-jupiter:3.11.2") + testImplementation("commons-io:commons-io:2.10.0") } test { diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt b/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt new file mode 100644 index 0000000000..26c78d1411 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_cert.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHTCCAYYCCQD4hqYeYDQZADANBgkqhkiG9w0BAQUFADBSMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCVFgxDzANBgNVBAcMBkF1c3RpbjEPMA0GA1UECgwGQW1hem9u +MRQwEgYDVQQLDAtEYXRhcHJlcHBlcjAgFw0yMTA2MjUxOTIzMTBaGA8yMTIxMDYw +MTE5MjMxMFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlRYMQ8wDQYDVQQHDAZB +dXN0aW4xDzANBgNVBAoMBkFtYXpvbjEUMBIGA1UECwwLRGF0YXByZXBwZXIwgZ8w +DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrb3YhdKbQ5PtLHall10iLZC9ZdDVrq +HOvqVSM8NHlL8f82gJ3l0n9k7hYc5eKisutaS9eDTmJ+Dnn8xn/qPSKTIq9Wh+OZ +O+e9YEEpI/G4F9KpGULgMyRg9sJK0GlZdEt9o5GJNJIJUkptJU5eiLuE0IV+jyJo +Nvm8OE6EJPqxAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAjgnX5n/Tt7eo9uakIGAb +uBhvYdR8JqKXqF9rjFJ/MIK7FdQSF/gCdjnvBhzLlZFK/Nb6MGKoSKm5Lcr75LgC +FyhIwp3WlqQksiMFnOypYVY71vqDgj6UKdMaOBgthsYhngj8lC+wsVzWqQvkJ2Qg +/GAIzJwiZfXiaevQHRk79qI= +-----END CERTIFICATE----- diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key b/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key new file mode 100644 index 0000000000..479b877131 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_decrypted_key.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCq292IXSm0OT7Sx2pZddIi2QvWXQ1a6hzr6lUjPDR5S/H/NoCd +5dJ/ZO4WHOXiorLrWkvXg05ifg55/MZ/6j0ikyKvVofjmTvnvWBBKSPxuBfSqRlC +4DMkYPbCStBpWXRLfaORiTSSCVJKbSVOXoi7hNCFfo8iaDb5vDhOhCT6sQIDAQAB +AoGANrrhFqpJDpr7vcb1ER0Fp/YArbT27zVo+EUC6puBb41dQlQyFOImcHpjLaAq +H1PgnjU5cBp2hGQ+vOK0rwrYc/HNl6vfh6N3NbDptMiuoBafRJA9JzYourAM09BU +zmXyr61Yn3KHzx1PRwWe37icX93oXP3P0qHb3dI1ZF4jG0ECQQDU5N/a7ogoz2zn +ZssD6FvUOUQDsdBWdXmhUvg+YdZrV44e4xk+FVzwEONoRktEYKz9MFXlsgNHr445 +KRguHWcJAkEAzXQkwOkN8WID1wrwoobUIMbZSGAZzofwkKXgTTnllnT1qOQXuRbS +aCMejFEymBBef4aXP6N4+va2FKW/MF34aQJAO2oMl1sOoOUSrZngepy0VAwPUUCk +thxe74jqQu6nGpn6zd/vQYZQw6bS8Fz90H1yic6dilcd1znFZWp0lxoZkQJBALeI +xoBycRsuFQIYasi1q3AwUtBd0Q/3zkZZeBtk2hzjFMUwJaUZpxKSNOrialD/ZnuD +jz+xWBTRKe0d98JMX+kCQCmsJEj/HYQAC1GamZ7JQWogRSRF2KTgTWRaDXDxy0d4 +yUQgwHB+HZLFcbi1JEK6eIixCsX8iifrrkteh+1npJ0= +-----END RSA PRIVATE KEY----- diff --git a/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key b/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key new file mode 100644 index 0000000000..285efc8d82 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/data/certificate/test_encrypted_key.key @@ -0,0 +1,17 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIICojAcBgoqhkiG9w0BDAEDMA4ECAd2FKZw2oGwAgIIAASCAoDTgiaXkazaotc7 +SxQK3bX34sEvdkLXg/O4ZpHTb0f4gLPxNhDe7ZPKrAS2TdywpSHT0189MVl+PIvw +4YQDaGVHL1SM5ukJu+PQkfQAMigdCJ+bUsG6hkrUDC74qYhHZHj1yVGavL6I4KHT +Ixh9IV2GMRS4m6HGJ17nYsdiTFFNdK++WcTMxbVWv3SNdKGZG79T32pjMxuIUPWr +3dB+ZXM+FSqwlBLZxPvvjlP6ETw7QXrlBHcQh1tHSh10bM+q0c5CktZoXLwpc+gv +ZsGXzpVjqFrAw4Vw0ikJl1mUCoGOtqqP0P6QWwbIJZBxNoO0MvWcXW+U3AGNFFze +nMR8UTXdga2l1Lx7pokQkWUpp48SDRjDx/RdZTRXCgtRcKuBcm0x2lxNILdwOzjJ +5GlKMvvc2OXXTnYqSCTqdfbuR3XBYmWgFki92D6JnVIYq+QJr5qj8IJDJ7mADQ1i +Za6PEJnrT641fLeSKRq7QiTydMQ3JXa9DFqUPwdZPPHLr/hC19sWHrq7gxvhkcLI +wrTtTIi8/iV4IVaiHk7YU83IM6sGExabQ3BRXcHMr+7i1vVxtEsFNC6ycTfJ8gpJ +YsnpXUQe912l5sk7GRSP1InNRF7kzMD0QeOAQ0UVfmsbHOPSXvD7fXkJWIb6N+zW +qCQYRmBwc7Bz2KZein5MLsMeNz2AWj/DcA2fMC+4+QtI0nF5BFtaR0V0npWhsbPu +3rj+AXipnvVhDIkfl8495j7ybCBj7HAZk8Ux8GmiZ3PGFO1C7XCQaLPWJ4Aw4Kb3 +EUqtVtpbwsCov5PDmMDXgz8qOxWMdQsP/dPF1HnVAg7SoFG9xA4nHdZ2WAFZwYtf +rRxEd7br +-----END ENCRYPTED PRIVATE KEY----- diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java new file mode 100644 index 0000000000..9d24dddbd2 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java @@ -0,0 +1,135 @@ +package com.amazon.dataprepper.plugins.certificate.acm; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; +import com.amazonaws.services.certificatemanager.model.*; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.PEMEncryptedKeyPair; +import org.bouncycastle.openssl.PEMKeyPair; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.bc.BcPEMDecryptorProvider; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; +import org.bouncycastle.pkcs.PKCSException; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEInputDecryptorProviderBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.security.PrivateKey; +import java.security.Security; + +public class ACMCertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); + private static final long SLEEP_INTERVAL = 10000L; + private static final String BOUNCY_CASTLE_PROVIDER = "BC"; + private final AWSCertificateManager awsCertificateManager; + private final long totalTimeout; + + public ACMCertificateProvider(final String awsRegion, final long totalTimeout) { + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final ClientConfiguration clientConfig = new ClientConfiguration() + .withThrottledRetries(true); + this.awsCertificateManager = AWSCertificateManagerClientBuilder.standard() + .withRegion(awsRegion) + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig) + .build(); + this.totalTimeout = totalTimeout; + Security.addProvider(new BouncyCastleProvider()); + } + + // accessible only in the same package for unit test + ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, final long totalTimeout) { + this.awsCertificateManager = awsCertificateManager; + this.totalTimeout = totalTimeout; + Security.addProvider(new BouncyCastleProvider()); + } + + public Certificate getACMCertificate(final String acmArn, final String passphrase) { + ExportCertificateResult exportCertificateResult = null; + long timeSlept = 0L; + while (exportCertificateResult == null && timeSlept < totalTimeout) { + try { + final ExportCertificateRequest exportCertificateRequest = new ExportCertificateRequest() + .withCertificateArn(acmArn) + .withPassphrase(ByteBuffer.wrap(passphrase.getBytes())); + exportCertificateResult = awsCertificateManager.exportCertificate(exportCertificateRequest); + + } catch (final RequestInProgressException ex) { + try { + Thread.sleep(SLEEP_INTERVAL); + } catch (InterruptedException iex) { + throw new RuntimeException(iex); + } + } catch (final ResourceNotFoundException | InvalidArnException ex) { + LOG.error("Exception retrieving the certificate with arn: {}", acmArn, ex); + throw ex; + } + timeSlept += SLEEP_INTERVAL; + } + if(exportCertificateResult != null) { + final String decryptedPrivateKey = getDecryptedPrivateKey(exportCertificateResult.getPrivateKey(), passphrase); + return new Certificate(exportCertificateResult.getCertificate(), decryptedPrivateKey); + } else { + throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); + } + } + + private String getDecryptedPrivateKey(final String encryptedPrivateKey, final String keyPassword) { + try { + final PrivateKey rsaPrivateKey = encryptedPrivateKeyStringToPrivateKey(encryptedPrivateKey, keyPassword.toCharArray()); + return privateKeyAsString(rsaPrivateKey); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private String privateKeyAsString(final PrivateKey key) throws IOException { + final StringWriter sw = new StringWriter(); + try (JcaPEMWriter pw = new JcaPEMWriter(sw)) + { + pw.writeObject(key); + } + return sw.toString(); + } + + private PrivateKey encryptedPrivateKeyStringToPrivateKey(final String encryptedPrivateKey, final char[] password) + throws IOException, PKCSException { + final PrivateKeyInfo pki; + try (final PEMParser pemParser = new PEMParser(new StringReader(encryptedPrivateKey))) { + final Object o = pemParser.readObject(); + if (o instanceof PKCS8EncryptedPrivateKeyInfo) { // encrypted private key in pkcs8-format + LOG.debug("key in pkcs8 encoding"); + final PKCS8EncryptedPrivateKeyInfo epki = (PKCS8EncryptedPrivateKeyInfo) o; + final JcePKCSPBEInputDecryptorProviderBuilder builder = + new JcePKCSPBEInputDecryptorProviderBuilder().setProvider(BOUNCY_CASTLE_PROVIDER); + final InputDecryptorProvider idp = builder.build(password); + pki = epki.decryptPrivateKeyInfo(idp); + } else if (o instanceof PEMEncryptedKeyPair) { // encrypted private key in pkcs1-format + LOG.debug("key in pkcs1 encoding"); + final PEMEncryptedKeyPair epki = (PEMEncryptedKeyPair) o; + final PEMKeyPair pkp = epki.decryptKeyPair(new BcPEMDecryptorProvider(password)); + pki = pkp.getPrivateKeyInfo(); + } else if (o instanceof PEMKeyPair) { // unencrypted private key + LOG.debug("key unencrypted"); + final PEMKeyPair pkp = (PEMKeyPair) o; + pki = pkp.getPrivateKeyInfo(); + } else { + throw new PKCSException("Invalid encrypted private key class: " + o != null ? o.getClass().getName() : null); + } + final JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider(BOUNCY_CASTLE_PROVIDER); + return converter.getPrivateKey(pki); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java new file mode 100644 index 0000000000..647eaa5ab9 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/model/Certificate.java @@ -0,0 +1,29 @@ +package com.amazon.dataprepper.plugins.certificate.model; + +import static java.util.Objects.requireNonNull; + +public class Certificate { + /** + * The base64 PEM-encoded certificate. + */ + private String certificate; + + /** + * The decrypted private key associated with the public key in the certificate. The key is output in PKCS #8 format + * and is base64 PEM-encoded. + */ + private String privateKey; + + public Certificate(final String certificate, final String privateKey) { + this.certificate = requireNonNull(certificate, "certificate must not be null"); + this.privateKey = requireNonNull(privateKey, "privateKey must not be null"); + } + + public String getCertificate() { + return certificate; + } + + public String getPrivateKey() { + return privateKey; + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 9199ee06cf..95d6c01ed7 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -7,7 +7,10 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.source.Source; +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.amazon.dataprepper.plugins.health.HealthGrpcService; +import com.amazonaws.arn.Arn; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -17,7 +20,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -27,12 +34,20 @@ public class OTelTraceSource implements Source private final OTelTraceSourceConfig oTelTraceSourceConfig; private Server server; private final PluginMetrics pluginMetrics; + private ACMCertificateProvider acmCertificateProvider; public OTelTraceSource(final PluginSetting pluginSetting) { oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); } + // accessible only in the same package for unit test + OTelTraceSource(final PluginSetting pluginSetting, final ACMCertificateProvider acmCertificateProvider) { + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); + this.acmCertificateProvider = acmCertificateProvider; + } + @Override public void start(Buffer> buffer) { if (buffer == null) { @@ -66,10 +81,23 @@ public void start(Buffer> buffer) { sb.service(grpcServiceBuilder.build()); sb.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); - if (oTelTraceSourceConfig.isSsl()) { - LOG.info("SSL/TLS is enabled"); - sb.https(oTelTraceSourceConfig.getPort()).tls(new File(oTelTraceSourceConfig.getSslKeyCertChainFile()), - new File(oTelTraceSourceConfig.getSslKeyFile())); + // ACM Cert for SSL takes preference + if (oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("SSL/TLS is enabled. Using ACM certificate for SSL."); + final Arn acmArn = Arn.fromString(oTelTraceSourceConfig.getAcmCertificateArn()); + final ACMCertificateProvider acmCertificateProvider = getACMCertificateProvider(acmArn.getRegion()); + final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn.toString(), UUID.randomUUID().toString()); + sb.https(oTelTraceSourceConfig.getPort()).tls( + new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) + ) + ); + } else if (oTelTraceSourceConfig.isSsl()) { + LOG.info("SSL/TLS is enabled. Using KeyCertChainFile and KeyFile for SSL."); + sb.https(oTelTraceSourceConfig.getPort()).tls( + new File(oTelTraceSourceConfig.getSslKeyCertChainFile()), + new File(oTelTraceSourceConfig.getSslKeyFile()) + ); } else { sb.http(oTelTraceSourceConfig.getPort()); } @@ -96,6 +124,10 @@ public void start(Buffer> buffer) { LOG.info("Started otel_trace_source..."); } + private ACMCertificateProvider getACMCertificateProvider(final String region) { + return Objects.requireNonNullElseGet(acmCertificateProvider, () -> new ACMCertificateProvider(region, oTelTraceSourceConfig.getAcmCertIssueTimeOutMillis())); + } + @Override public void stop() { if (server != null) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index 1e0b5ad6b4..a06caa436d 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -6,10 +6,13 @@ public class OTelTraceSourceConfig { static final String REQUEST_TIMEOUT = "request_timeout"; static final String PORT = "port"; static final String SSL = "ssl"; + static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; + static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; static final String HEALTH_CHECK_SERVICE = "health_check_service"; static final String PROTO_REFLECTION_SERVICE = "proto_reflection_service"; static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; static final String SSL_KEY_FILE = "sslKeyFile"; + static final String ACM_CERT_ARN = "acmCertificateArn"; static final String THREAD_COUNT = "thread_count"; static final String MAX_CONNECTION_COUNT = "max_connection_count"; static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; @@ -18,14 +21,19 @@ public class OTelTraceSourceConfig { static final int DEFAULT_THREAD_COUNT = 200; static final int DEFAULT_MAX_CONNECTION_COUNT = 500; static final boolean DEFAULT_SSL = true; + static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; private final int requestTimeoutInMillis; private final int port; private final boolean healthCheck; private final boolean protoReflectionService; private final boolean enableUnframedRequests; private final boolean ssl; + private final boolean useAcmCertForSSL; + private final long acmCertIssueTimeOutMillis; private final String sslKeyCertChainFile; private final String sslKeyFile; + private final String acmCertificateArn; private final int threadCount; private final int maxConnectionCount; @@ -35,8 +43,11 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, final boolean protoReflectionService, final boolean enableUnframedRequests, final boolean isSSL, + final boolean useAcmCertForSSL, + final long acmCertIssueTimeOutMillis, final String sslKeyCertChainFile, final String sslKeyFile, + final String acmCertificateArn, final int threadCount, final int maxConnectionCount) { this.requestTimeoutInMillis = requestTimeoutInMillis; @@ -45,15 +56,22 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, this.protoReflectionService = protoReflectionService; this.enableUnframedRequests = enableUnframedRequests; this.ssl = isSSL; + this.useAcmCertForSSL = useAcmCertForSSL; + this.acmCertIssueTimeOutMillis = acmCertIssueTimeOutMillis; this.sslKeyCertChainFile = sslKeyCertChainFile; this.sslKeyFile = sslKeyFile; + this.acmCertificateArn = acmCertificateArn; this.threadCount = threadCount; this.maxConnectionCount = maxConnectionCount; - if (ssl && (sslKeyCertChainFile == null || sslKeyCertChainFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); - } - if (ssl && (sslKeyFile == null || sslKeyFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + if (useAcmCertForSSL && (acmCertificateArn == null || acmCertificateArn.isEmpty())) { + throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", USE_ACM_CERT_FOR_SSL, ACM_CERT_ARN)); + } else { + if (ssl && (sslKeyCertChainFile == null || sslKeyCertChainFile.isEmpty())) { + throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + } + if (ssl && (sslKeyFile == null || sslKeyFile.isEmpty())) { + throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + } } } @@ -64,8 +82,11 @@ public static OTelTraceSourceConfig buildConfig(final PluginSetting pluginSettin pluginSetting.getBooleanOrDefault(PROTO_REFLECTION_SERVICE, false), pluginSetting.getBooleanOrDefault(ENABLE_UNFRAMED_REQUESTS, false), pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL), + pluginSetting.getBooleanOrDefault(USE_ACM_CERT_FOR_SSL, DEFAULT_USE_ACM_CERT_FOR_SSL), + pluginSetting.getLongOrDefault(ACM_CERT_ISSUE_TIME_OUT_MILLIS, DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS), pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null), pluginSetting.getStringOrDefault(SSL_KEY_FILE, null), + pluginSetting.getStringOrDefault(ACM_CERT_ARN, null), pluginSetting.getIntegerOrDefault(THREAD_COUNT, DEFAULT_THREAD_COUNT), pluginSetting.getIntegerOrDefault(MAX_CONNECTION_COUNT, DEFAULT_MAX_CONNECTION_COUNT)); } @@ -94,6 +115,14 @@ public boolean isSsl() { return ssl; } + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis; + } + public String getSslKeyCertChainFile() { return sslKeyCertChainFile; } @@ -102,6 +131,10 @@ public String getSslKeyFile() { return sslKeyFile; } + public String getAcmCertificateArn() { + return acmCertificateArn; + } + public int getThreadCount() { return threadCount; } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java new file mode 100644 index 0000000000..24c95adbc1 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java @@ -0,0 +1,104 @@ +package com.amazon.dataprepper.plugins.certificate.acm; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class CertificateProviderTest { + @Mock + private AWSCertificateManager awsCertificateManager; + + @Mock + private ExportCertificateResult exportCertificateResult; + + private ACMCertificateProvider acmCertificateProvider; + + @BeforeEach + public void beforeEach() { + acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, 2000L); + } + + @Test + public void getACMCertificateWithEncryptedPrivateKeySuccess() throws IOException { + final String acmArn = UUID.randomUUID().toString(); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path encryptedKeyFilePath = Path.of("data/certificate/test_encrypted_key.key"); + final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String passphrase = "password"; + final String certAsString = Files.readString(certFilePath); + final String encryptedKeyAsString = Files.readString(encryptedKeyFilePath); + final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); + when(exportCertificateResult.getCertificate()).thenReturn(certAsString); + when(exportCertificateResult.getPrivateKey()).thenReturn(encryptedKeyAsString); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn, passphrase); + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); + } + + @Test + public void getACMCertificateWithUnencryptedPrivateKeySuccess() throws IOException { + final String acmArn = UUID.randomUUID().toString(); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String passphrase = "password"; + final String certAsString = Files.readString(certFilePath); + final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); + when(exportCertificateResult.getCertificate()).thenReturn(certAsString); + when(exportCertificateResult.getPrivateKey()).thenReturn(decryptedKeyAsString); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn, passphrase); + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); + } + + @Test + public void getACMCertificateWithInvalidPrivateKeyException() { + final String acmArn = UUID.randomUUID().toString(); + final String passphrase = "password"; + when(exportCertificateResult.getPrivateKey()).thenReturn(UUID.randomUUID().toString()); + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); + assertThrows(RuntimeException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + } + + @Test + public void getACMCertificateRequestInProgressException() { + final String acmArn = UUID.randomUUID().toString(); + final String passphrase = "password"; + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new RequestInProgressException("Request in progress.")); + assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + } + + @Test + public void getACMCertificateResourceNotFoundException() { + final String acmArn = UUID.randomUUID().toString(); + final String passphrase = "password"; + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new ResourceNotFoundException("Resource not found.")); + assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + } + + @Test + public void getACMCertificateInvalidArnException() { + final String acmArn = UUID.randomUUID().toString(); + final String passphrase = "password"; + when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new InvalidArnException("Invalid certificate arn.")); + assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 68bd736586..c168984431 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -3,9 +3,11 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpMethod; @@ -20,29 +22,41 @@ import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import static com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -54,10 +68,16 @@ public class OTelTraceSourceTest { @Mock private Server server; + @Mock + private ACMCertificateProvider acmCertificateProvider; + + @Mock + private Certificate certificate; + @Mock private CompletableFuture completableFuture; - PluginSetting pluginSetting; - PluginSetting testPluginSetting; + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; private BlockingBuffer> buffer; @@ -65,17 +85,13 @@ public class OTelTraceSourceTest { .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() .addSpans(Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); - private static OTelTraceSource SOURCE; + private OTelTraceSource SOURCE; private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); private static TraceServiceGrpc.TraceServiceBlockingStub CLIENT; - private static String getUri() { - return "gproto+http://127.0.0.1:" + SOURCE.getoTelTraceSourceConfig().getPort() + '/'; - } - private BlockingBuffer> getBuffer() { final HashMap integerHashMap = new HashMap<>(); integerHashMap.put("buffer_size", 1); @@ -87,20 +103,17 @@ private BlockingBuffer> getBuffer() { public void beforeEach() { lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); lenient().when(serverBuilder.build()).thenReturn(server); lenient().when(server.start()).thenReturn(completableFuture); - final HashMap settingsMap = new HashMap<>(); + final Map settingsMap = new HashMap<>(); settingsMap.put("request_timeout", 5); settingsMap.put(SSL, false); pluginSetting = new PluginSetting("otel_trace", settingsMap); pluginSetting.setPipelineName("pipeline"); SOURCE = new OTelTraceSource(pluginSetting); - buffer = getBuffer(); - SOURCE.start(buffer); - - CLIENT = Clients.newClient(getUri(), TraceServiceGrpc.TraceServiceBlockingStub.class); } @AfterEach @@ -110,6 +123,7 @@ public void afterEach() { @Test void testHttpFullJson() throws InvalidProtocolBufferException { + SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -119,9 +133,7 @@ void testHttpFullJson() throws InvalidProtocolBufferException { .build(), HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); - }).join(); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -131,15 +143,54 @@ void testHttpFullJson() throws InvalidProtocolBufferException { .build(), HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415) //validateBuffer(); - }).join(); + ).join(); + } + + @Test + void testHttpsFullJson() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("request_timeout", 5); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + pluginSetting = new PluginSetting("otel_trace", settingsMap); + pluginSetting.setPipelineName("pipeline"); + SOURCE = new OTelTraceSource(pluginSetting); + buffer = getBuffer(); + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415) + //validateBuffer(); + ).join(); } @Test void testHttpFullBytes() { + SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -149,9 +200,7 @@ void testHttpFullBytes() { .build(), HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) .aggregate() - .whenComplete((i, ex) -> { - assertThat(i.status().code()).isEqualTo(415); - }).join(); + .whenComplete((i, ex) -> assertThat(i.status().code()).isEqualTo(415)).join(); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -167,13 +216,79 @@ void testHttpFullBytes() { }).join(); } + @Test + public void testServerStartCertFileSuccess() { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting); + source.start(buffer); + source.stop(); + + final ArgumentCaptor sslKeyCertChainFile = ArgumentCaptor.forClass(File.class); + final ArgumentCaptor sslKeyFile = ArgumentCaptor.forClass(File.class); + verify(serverBuilder).tls(sslKeyCertChainFile.capture(), sslKeyFile.capture()); + assertThat(sslKeyCertChainFile.getValue().getPath()).isEqualTo("data/certificate/test_cert.crt"); + assertThat(sslKeyFile.getValue().getPath()).isEqualTo("data/certificate/test_decrypted_key.key"); + } + } + + @Test + public void testServerStartACMCertSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(acmCertificateProvider.getACMCertificate(anyString(), anyString())).thenReturn(certificate); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, acmCertificateProvider); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate).isEqualTo(certAsString); + assertThat(actualPrivateKey).isEqualTo(keyAsString); + } + } + @Test public void testDoubleStart() { + // starting server + SOURCE.start(buffer); + // double start server Assertions.assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); } @Test public void testRunAnotherSourceWithSamePort() { + // starting server + SOURCE.start(buffer); + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); testPluginSetting.setPipelineName("pipeline"); final OTelTraceSource source = new OTelTraceSource(testPluginSetting); @@ -194,7 +309,7 @@ public void testStartWithServerExecutionExceptionNoCause() throws ExecutionExcep // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); when(completableFuture.get()).thenThrow(new ExecutionException("", null)); // When/Then @@ -207,7 +322,7 @@ public void testStartWithServerExecutionExceptionWithCause() throws ExecutionExc // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); final NullPointerException expCause = new NullPointerException(); when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); @@ -222,7 +337,7 @@ public void testStopWithServerExecutionExceptionNoCause() throws ExecutionExcept // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); @@ -237,7 +352,7 @@ public void testStartWithInterruptedException() throws ExecutionException, Inter // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); when(completableFuture.get()).thenThrow(new InterruptedException()); // When/Then @@ -251,7 +366,7 @@ public void testStopWithServerExecutionExceptionWithCause() throws ExecutionExce // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); final NullPointerException expCause = new NullPointerException(); @@ -268,7 +383,7 @@ public void testStopWithInterruptedException() throws ExecutionException, Interr // Prepare final OTelTraceSource source = new OTelTraceSource(pluginSetting); try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(() -> Server.builder()).thenReturn(serverBuilder); + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); source.start(buffer); when(server.stop()).thenReturn(completableFuture); when(completableFuture.get()).thenThrow(new InterruptedException()); From 63ba62f401c6818491198891c8fe0961832d5463 Mon Sep 17 00:00:00 2001 From: dinujoh <86094133+dinujoh@users.noreply.github.com> Date: Wed, 30 Jun 2021 15:05:12 -0500 Subject: [PATCH 118/192] Add support to read ssl certificate from S3 (#711) * Add support to read ssl certificate from S3 When ssl is enabled, the sslKeyCertChainFile and sslKeyFile can be AWS S3 path to get the certificate instead of local file system. Changes include: * Creating certificate provider manager which can provide ACM, S3 or File certificate provider. * Implementation of ACM certificate, S3 and File certificate provider and unit test. * Updated the README file to reflect the new configuration parameter to use certificate from ACM and S3. Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- .../otel-trace-source/README.md | 14 +++- .../otel-trace-source/build.gradle | 3 + .../certificate/CertificateProvider.java | 7 ++ .../CertificateProviderFactory.java | 52 +++++++++++++ .../acm/ACMCertificateProvider.java | 50 ++++++------ .../file/FileCertificateProvider.java | 38 +++++++++ .../certificate/s3/S3CertificateProvider.java | 49 ++++++++++++ .../source/oteltrace/OTelTraceSource.java | 35 +++------ .../oteltrace/OTelTraceSourceConfig.java | 60 ++++++++++++-- .../CertificateProviderFactoryTest.java | 76 ++++++++++++++++++ ...t.java => ACMCertificateProviderTest.java} | 31 +++----- .../file/FileCertificateProviderTest.java | 48 ++++++++++++ .../s3/S3CertificateProviderTest.java | 78 +++++++++++++++++++ .../source/oteltrace/OTelTraceSourceTest.java | 39 ++++++---- 14 files changed, 483 insertions(+), 97 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java rename data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/{CertificateProviderTest.java => ACMCertificateProviderTest.java} (81%) create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index d73b0155f1..7a0b15c516 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -17,11 +17,17 @@ source: * health_check_service(Optional) => A boolean enables a gRPC health check service under ```grpc.health.v1 / Health / Check```. Default is ```false```. * proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is ```false```. * unframed_requests(Optional) => A boolean to enable requests not framed using the gRPC wire protocol. +* thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200`. +* max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500`. + +### SSL + * ssl(Optional) => A boolean enables TLS/SSL. Default is ```true```. -* sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path. Required if ```ssl``` is set to ```true``` -* sslKeyFile(Optional) => A `String` represents the SSL key file path. Required if ```ssl``` is set to ```true``` -* thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200` -* max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500` +* sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* sslKeyFile(Optional) => A `String` represents the SSL key file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* useAcmCertForSSL(Optional) => A boolean enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is ```false```. +* acmCertificateArn(Optional) => A `String` represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if ```useAcmCertForSSL``` is set to ```true```. +* awsRegion(Optional) => A `String` represents the AWS region to use ACM or S3. Required if ```useAcmCertForSSL``` is set to ```true``` or ```sslKeyCertChainFile``` and ```sslKeyFile``` is ```AWS S3 path```. ## Metrics diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index fd024bccb3..4bed040fbe 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,12 +8,15 @@ dependencies { implementation 'commons-codec:commons-codec:1.15' testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" + implementation "commons-io:commons-io:2.10.0" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.12" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "org.apache.commons:commons-lang3:3.11" implementation "org.bouncycastle:bcprov-jdk15on:1.69" implementation "org.bouncycastle:bcpkix-jdk15on:1.69" testImplementation 'org.assertj:assertj-core:3.20.2' diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java new file mode 100644 index 0000000000..f34092450a --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProvider.java @@ -0,0 +1,7 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; + +public interface CertificateProvider { + Certificate getCertificate(); +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java new file mode 100644 index 0000000000..4c3237cf48 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactory.java @@ -0,0 +1,52 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CertificateProviderFactory { + private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); + + final OTelTraceSourceConfig oTelTraceSourceConfig; + public CertificateProviderFactory(final OTelTraceSourceConfig oTelTraceSourceConfig) { + this.oTelTraceSourceConfig = oTelTraceSourceConfig; + } + + public CertificateProvider getCertificateProvider() { + // ACM Cert for SSL takes preference + if (oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("Using ACM certificate and private key for SSL/TLS."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final ClientConfiguration clientConfig = new ClientConfiguration() + .withThrottledRetries(true); + final AWSCertificateManager awsCertificateManager = AWSCertificateManagerClientBuilder.standard() + .withRegion(oTelTraceSourceConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig) + .build(); + return new ACMCertificateProvider(awsCertificateManager, oTelTraceSourceConfig.getAcmCertificateArn(), + oTelTraceSourceConfig.getAcmCertIssueTimeOutMillis(), oTelTraceSourceConfig.getAcmPrivateKeyPassword()); + } else if (oTelTraceSourceConfig.isSslCertAndKeyFileInS3()) { + LOG.info("Using S3 to fetch certificate and private key for SSL/TLS."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(oTelTraceSourceConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .build(); + return new S3CertificateProvider(s3Client, oTelTraceSourceConfig.getSslKeyCertChainFile(), oTelTraceSourceConfig.getSslKeyFile()); + } else { + LOG.info("Using local file system to get certificate and private key for SSL/TLS."); + return new FileCertificateProvider(oTelTraceSourceConfig.getSslKeyCertChainFile(), oTelTraceSourceConfig.getSslKeyFile()); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java index 9d24dddbd2..935bab4229 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java @@ -1,11 +1,8 @@ package com.amazon.dataprepper.plugins.certificate.acm; +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; import com.amazon.dataprepper.plugins.certificate.model.Certificate; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.certificatemanager.AWSCertificateManager; -import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; import com.amazonaws.services.certificatemanager.model.*; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -28,42 +25,43 @@ import java.nio.ByteBuffer; import java.security.PrivateKey; import java.security.Security; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; -public class ACMCertificateProvider { +public class ACMCertificateProvider implements CertificateProvider { private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); private static final long SLEEP_INTERVAL = 10000L; private static final String BOUNCY_CASTLE_PROVIDER = "BC"; private final AWSCertificateManager awsCertificateManager; + private final String acmArn; private final long totalTimeout; - - public ACMCertificateProvider(final String awsRegion, final long totalTimeout) { - final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - final ClientConfiguration clientConfig = new ClientConfiguration() - .withThrottledRetries(true); - this.awsCertificateManager = AWSCertificateManagerClientBuilder.standard() - .withRegion(awsRegion) - .withCredentials(credentialsProvider) - .withClientConfiguration(clientConfig) - .build(); - this.totalTimeout = totalTimeout; + private final String passphrase; + public ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, + final String acmArn, + final long totalTimeout, + final String passphrase) { + this.awsCertificateManager = Objects.requireNonNull(awsCertificateManager); + this.acmArn = Objects.requireNonNull(acmArn); + this.totalTimeout = Objects.requireNonNull(totalTimeout); + // Passphrase can be null. If null a random passphrase will be generated. + this.passphrase = passphrase; Security.addProvider(new BouncyCastleProvider()); } - // accessible only in the same package for unit test - ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, final long totalTimeout) { - this.awsCertificateManager = awsCertificateManager; - this.totalTimeout = totalTimeout; - Security.addProvider(new BouncyCastleProvider()); - } - - public Certificate getACMCertificate(final String acmArn, final String passphrase) { + public Certificate getCertificate() { ExportCertificateResult exportCertificateResult = null; long timeSlept = 0L; + + // The private key from ACM is encrypted. Passphrase is the privateKey password that will be used to decrypt the + // private key. If it's not provided, UUID is used generate a temporary password. The configured passphrase can + // be used to decrypt the private key manually using openssl commands for any inspection or debugging. + final String pkPassphrase = Optional.ofNullable(passphrase).orElse(UUID.randomUUID().toString()); while (exportCertificateResult == null && timeSlept < totalTimeout) { try { final ExportCertificateRequest exportCertificateRequest = new ExportCertificateRequest() .withCertificateArn(acmArn) - .withPassphrase(ByteBuffer.wrap(passphrase.getBytes())); + .withPassphrase(ByteBuffer.wrap(pkPassphrase.getBytes())); exportCertificateResult = awsCertificateManager.exportCertificate(exportCertificateRequest); } catch (final RequestInProgressException ex) { @@ -79,7 +77,7 @@ public Certificate getACMCertificate(final String acmArn, final String passphras timeSlept += SLEEP_INTERVAL; } if(exportCertificateResult != null) { - final String decryptedPrivateKey = getDecryptedPrivateKey(exportCertificateResult.getPrivateKey(), passphrase); + final String decryptedPrivateKey = getDecryptedPrivateKey(exportCertificateResult.getPrivateKey(), pkPassphrase); return new Certificate(exportCertificateResult.getCertificate(), decryptedPrivateKey); } else { throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java new file mode 100644 index 0000000000..f18892bac2 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProvider.java @@ -0,0 +1,38 @@ +package com.amazon.dataprepper.plugins.certificate.file; + +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +public class FileCertificateProvider implements CertificateProvider { + private final String certificateFilePath; + private final String privateKeyFilePath; + + public FileCertificateProvider(final String certificateFilePath, + final String privateKeyFilePath) { + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + this.privateKeyFilePath = Objects.requireNonNull(privateKeyFilePath); + } + + private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); + + public Certificate getCertificate() { + try { + final Path certFilePath = Path.of(certificateFilePath); + final Path pkFilePath = Path.of(privateKeyFilePath); + + final String certAsString = Files.readString(certFilePath); + final String privateKeyAsString = Files.readString(pkFilePath); + + return new Certificate(certAsString, privateKeyAsString); + } catch (final Exception ex) { + LOG.error("Error encountered while reading the certificate.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java new file mode 100644 index 0000000000..f72ae4e303 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProvider.java @@ -0,0 +1,49 @@ +package com.amazon.dataprepper.plugins.certificate.s3; + +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class S3CertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(S3CertificateProvider.class); + private final AmazonS3 s3Client; + private final String certificateFilePath; + private final String privateKeyFilePath; + + public S3CertificateProvider(final AmazonS3 s3Client, + final String certificateFilePath, + final String privateKeyFilePath) { + this.s3Client = Objects.requireNonNull(s3Client); + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + this.privateKeyFilePath = Objects.requireNonNull(privateKeyFilePath); + } + + public Certificate getCertificate() { + final AmazonS3URI certificateS3URI = new AmazonS3URI(certificateFilePath); + final AmazonS3URI privateKeyS3URI = new AmazonS3URI(privateKeyFilePath); + final String certificate = getObjectWithKey(certificateS3URI.getBucket(), certificateS3URI.getKey()); + final String privateKey = getObjectWithKey(privateKeyS3URI.getBucket(), privateKeyS3URI.getKey()); + + return new Certificate(certificate, privateKey); + } + + private String getObjectWithKey(final String bucketName, final String key) { + + // Download the object + try (final S3Object s3Object = s3Client.getObject(bucketName, key)) { + LOG.info("Object with key \"{}\" downloaded.", key); + return IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8); + } catch (final Exception ex) { + LOG.error("Error encountered while processing the response from Amazon S3.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 95d6c01ed7..26d09c6a62 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -7,10 +7,10 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.source.Source; -import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProviderFactory; import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.amazon.dataprepper.plugins.health.HealthGrpcService; -import com.amazonaws.arn.Arn; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -21,10 +21,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; -import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.Objects; -import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -34,18 +31,19 @@ public class OTelTraceSource implements Source private final OTelTraceSourceConfig oTelTraceSourceConfig; private Server server; private final PluginMetrics pluginMetrics; - private ACMCertificateProvider acmCertificateProvider; + private final CertificateProviderFactory certificateProviderFactory; public OTelTraceSource(final PluginSetting pluginSetting) { oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); } // accessible only in the same package for unit test - OTelTraceSource(final PluginSetting pluginSetting, final ACMCertificateProvider acmCertificateProvider) { + OTelTraceSource(final PluginSetting pluginSetting, final CertificateProviderFactory certificateProviderFactory) { oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); - this.acmCertificateProvider = acmCertificateProvider; + this.certificateProviderFactory = certificateProviderFactory; } @Override @@ -74,7 +72,7 @@ public void start(Buffer> buffer) { LOG.info("Proto reflection service is enabled"); grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } - + grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); final ServerBuilder sb = Server.builder(); @@ -82,22 +80,15 @@ public void start(Buffer> buffer) { sb.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); // ACM Cert for SSL takes preference - if (oTelTraceSourceConfig.useAcmCertForSSL()) { - LOG.info("SSL/TLS is enabled. Using ACM certificate for SSL."); - final Arn acmArn = Arn.fromString(oTelTraceSourceConfig.getAcmCertificateArn()); - final ACMCertificateProvider acmCertificateProvider = getACMCertificateProvider(acmArn.getRegion()); - final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn.toString(), UUID.randomUUID().toString()); + if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("SSL/TLS is enabled."); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + final Certificate certificate = certificateProvider.getCertificate(); sb.https(oTelTraceSourceConfig.getPort()).tls( new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) ) ); - } else if (oTelTraceSourceConfig.isSsl()) { - LOG.info("SSL/TLS is enabled. Using KeyCertChainFile and KeyFile for SSL."); - sb.https(oTelTraceSourceConfig.getPort()).tls( - new File(oTelTraceSourceConfig.getSslKeyCertChainFile()), - new File(oTelTraceSourceConfig.getSslKeyFile()) - ); } else { sb.http(oTelTraceSourceConfig.getPort()); } @@ -124,10 +115,6 @@ public void start(Buffer> buffer) { LOG.info("Started otel_trace_source..."); } - private ACMCertificateProvider getACMCertificateProvider(final String region) { - return Objects.requireNonNullElseGet(acmCertificateProvider, () -> new ACMCertificateProvider(region, oTelTraceSourceConfig.getAcmCertIssueTimeOutMillis())); - } - @Override public void stop() { if (server != null) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index a06caa436d..d01f54b56f 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -2,6 +2,8 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; +import org.apache.commons.lang3.StringUtils; + public class OTelTraceSourceConfig { static final String REQUEST_TIMEOUT = "request_timeout"; static final String PORT = "port"; @@ -13,6 +15,8 @@ public class OTelTraceSourceConfig { static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; static final String SSL_KEY_FILE = "sslKeyFile"; static final String ACM_CERT_ARN = "acmCertificateArn"; + static final String ACM_PRIVATE_KEY_PASSWORD = "acmPrivateKeyPassword"; + static final String AWS_REGION = "awsRegion"; static final String THREAD_COUNT = "thread_count"; static final String MAX_CONNECTION_COUNT = "max_connection_count"; static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; @@ -23,6 +27,7 @@ public class OTelTraceSourceConfig { static final boolean DEFAULT_SSL = true; static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; + private static final String S3_PREFIX = "s3://"; private final int requestTimeoutInMillis; private final int port; private final boolean healthCheck; @@ -33,7 +38,10 @@ public class OTelTraceSourceConfig { private final long acmCertIssueTimeOutMillis; private final String sslKeyCertChainFile; private final String sslKeyFile; + private final boolean sslCertAndKeyFileInS3; private final String acmCertificateArn; + private final String acmPrivateKeyPassword; + private final String awsRegion; private final int threadCount; private final int maxConnectionCount; @@ -48,6 +56,8 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, final String sslKeyCertChainFile, final String sslKeyFile, final String acmCertificateArn, + final String acmPrivateKeyPassword, + final String awsRegion, final int threadCount, final int maxConnectionCount) { this.requestTimeoutInMillis = requestTimeoutInMillis; @@ -61,18 +71,38 @@ private OTelTraceSourceConfig(final int requestTimeoutInMillis, this.sslKeyCertChainFile = sslKeyCertChainFile; this.sslKeyFile = sslKeyFile; this.acmCertificateArn = acmCertificateArn; + this.acmPrivateKeyPassword = acmPrivateKeyPassword; + this.awsRegion = awsRegion; this.threadCount = threadCount; this.maxConnectionCount = maxConnectionCount; - if (useAcmCertForSSL && (acmCertificateArn == null || acmCertificateArn.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", USE_ACM_CERT_FOR_SSL, ACM_CERT_ARN)); - } else { - if (ssl && (sslKeyCertChainFile == null || sslKeyCertChainFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); - } - if (ssl && (sslKeyFile == null || sslKeyFile.isEmpty())) { - throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); + boolean certAndKeyFileInS3 = false; + if (useAcmCertForSSL) { + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), acmCertificateArn, ACM_CERT_ARN); + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), awsRegion, AWS_REGION); + } else if(ssl) { + validateSSLCertificateFiles(); + certAndKeyFileInS3 = isSSLCertificateLocatedInS3(); + if (certAndKeyFileInS3) { + validateSSLArgument("The certificate and key files are located in S3", awsRegion, AWS_REGION); } } + this.sslCertAndKeyFileInS3 = certAndKeyFileInS3; + } + + private void validateSSLArgument(final String sslTypeMessage, final String argument, final String argumentName) { + if (StringUtils.isEmpty(argument)) { + throw new IllegalArgumentException(String.format("%s, %s can not be empty or null", sslTypeMessage, argumentName)); + } + } + + private void validateSSLCertificateFiles() { + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyCertChainFile, SSL_KEY_CERT_FILE); + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyFile, SSL_KEY_FILE); + } + + private boolean isSSLCertificateLocatedInS3() { + return sslKeyCertChainFile.toLowerCase().startsWith(S3_PREFIX) && + sslKeyFile.toLowerCase().startsWith(S3_PREFIX); } public static OTelTraceSourceConfig buildConfig(final PluginSetting pluginSetting) { @@ -87,6 +117,8 @@ public static OTelTraceSourceConfig buildConfig(final PluginSetting pluginSettin pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null), pluginSetting.getStringOrDefault(SSL_KEY_FILE, null), pluginSetting.getStringOrDefault(ACM_CERT_ARN, null), + pluginSetting.getStringOrDefault(ACM_PRIVATE_KEY_PASSWORD, null), + pluginSetting.getStringOrDefault(AWS_REGION, null), pluginSetting.getIntegerOrDefault(THREAD_COUNT, DEFAULT_THREAD_COUNT), pluginSetting.getIntegerOrDefault(MAX_CONNECTION_COUNT, DEFAULT_MAX_CONNECTION_COUNT)); } @@ -135,6 +167,18 @@ public String getAcmCertificateArn() { return acmCertificateArn; } + public String getAcmPrivateKeyPassword() { + return acmPrivateKeyPassword; + } + + public boolean isSslCertAndKeyFileInS3() { + return sslCertAndKeyFileInS3; + } + + public String getAwsRegion() { + return awsRegion; + } + public int getThreadCount() { return threadCount; } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java new file mode 100644 index 0000000000..634dad17e9 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/CertificateProviderFactoryTest.java @@ -0,0 +1,76 @@ +package com.amazon.dataprepper.plugins.certificate; + +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import org.hamcrest.core.IsInstanceOf; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +public class CertificateProviderFactoryTest { + private OTelTraceSourceConfig oTelTraceSourceConfig; + private CertificateProviderFactory certificateProviderFactory; + + @Test + public void getCertificateProviderAcmProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); + } + + @Test + public void getCertificateProviderS3ProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("sslKeyCertChainFile", "s3://data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "s3://data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); + } + + @Test + public void getCertificateProviderFileProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OTelTraceSourceConfig.buildConfig(pluginSetting); + + certificateProviderFactory = new CertificateProviderFactory(oTelTraceSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java similarity index 81% rename from data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java rename to data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java index 24c95adbc1..54e0773815 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/CertificateProviderTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProviderTest.java @@ -21,7 +21,10 @@ import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class CertificateProviderTest { +public class ACMCertificateProviderTest { + private static final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + private static final long acmCertIssueTimeOutMillis = 2000L; + private static final String acmPrivateKeyPassword = "password"; @Mock private AWSCertificateManager awsCertificateManager; @@ -32,73 +35,61 @@ public class CertificateProviderTest { @BeforeEach public void beforeEach() { - acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, 2000L); + acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, acmCertificateArn, acmCertIssueTimeOutMillis, acmPrivateKeyPassword); } @Test public void getACMCertificateWithEncryptedPrivateKeySuccess() throws IOException { - final String acmArn = UUID.randomUUID().toString(); final Path certFilePath = Path.of("data/certificate/test_cert.crt"); final Path encryptedKeyFilePath = Path.of("data/certificate/test_encrypted_key.key"); final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String passphrase = "password"; final String certAsString = Files.readString(certFilePath); final String encryptedKeyAsString = Files.readString(encryptedKeyFilePath); final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); when(exportCertificateResult.getCertificate()).thenReturn(certAsString); when(exportCertificateResult.getPrivateKey()).thenReturn(encryptedKeyAsString); when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); - final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn, passphrase); + final Certificate certificate = acmCertificateProvider.getCertificate(); assertThat(certificate.getCertificate(), is(certAsString)); assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); } @Test public void getACMCertificateWithUnencryptedPrivateKeySuccess() throws IOException { - final String acmArn = UUID.randomUUID().toString(); final Path certFilePath = Path.of("data/certificate/test_cert.crt"); final Path decryptedKeyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String passphrase = "password"; final String certAsString = Files.readString(certFilePath); final String decryptedKeyAsString = Files.readString(decryptedKeyFilePath); when(exportCertificateResult.getCertificate()).thenReturn(certAsString); when(exportCertificateResult.getPrivateKey()).thenReturn(decryptedKeyAsString); when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); - final Certificate certificate = acmCertificateProvider.getACMCertificate(acmArn, passphrase); + final Certificate certificate = acmCertificateProvider.getCertificate(); assertThat(certificate.getCertificate(), is(certAsString)); assertThat(certificate.getPrivateKey(), is(decryptedKeyAsString)); } @Test public void getACMCertificateWithInvalidPrivateKeyException() { - final String acmArn = UUID.randomUUID().toString(); - final String passphrase = "password"; when(exportCertificateResult.getPrivateKey()).thenReturn(UUID.randomUUID().toString()); when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenReturn(exportCertificateResult); - assertThrows(RuntimeException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + assertThrows(RuntimeException.class, () -> acmCertificateProvider.getCertificate()); } @Test public void getACMCertificateRequestInProgressException() { - final String acmArn = UUID.randomUUID().toString(); - final String passphrase = "password"; when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new RequestInProgressException("Request in progress.")); - assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getCertificate()); } @Test public void getACMCertificateResourceNotFoundException() { - final String acmArn = UUID.randomUUID().toString(); - final String passphrase = "password"; when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new ResourceNotFoundException("Resource not found.")); - assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getCertificate()); } @Test public void getACMCertificateInvalidArnException() { - final String acmArn = UUID.randomUUID().toString(); - final String passphrase = "password"; when(awsCertificateManager.exportCertificate(any(ExportCertificateRequest.class))).thenThrow(new InvalidArnException("Invalid certificate arn.")); - assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getACMCertificate(acmArn, passphrase)); + assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getCertificate()); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java new file mode 100644 index 0000000000..7b0fbc9e44 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/file/FileCertificateProviderTest.java @@ -0,0 +1,48 @@ +package com.amazon.dataprepper.plugins.certificate.file; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +public class FileCertificateProviderTest { + + private FileCertificateProvider fileCertificateProvider; + + @Test + public void getCertificateValidPathSuccess() throws IOException { + final String certificateFilePath = "data/certificate/test_cert.crt"; + final String privateKeyFilePath = "data/certificate/test_decrypted_key.key"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath, privateKeyFilePath); + + final Certificate certificate = fileCertificateProvider.getCertificate(); + + final Path certFilePath = Path.of(certificateFilePath); + final Path keyFilePath = Path.of(privateKeyFilePath); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + assertThat(certificate.getCertificate(), is(certAsString)); + assertThat(certificate.getPrivateKey(), is(keyAsString)); + } + + @Test + public void getCertificateInvalidPathSuccess() { + final String certificateFilePath = "path_does_not_exit/test_cert.crt"; + final String privateKeyFilePath = "path_does_not_exit/test_decrypted_key.key"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath, privateKeyFilePath); + + Assertions.assertThrows(RuntimeException.class, () -> fileCertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java new file mode 100644 index 0000000000..cdf7c36dfe --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/certificate/s3/S3CertificateProviderTest.java @@ -0,0 +1,78 @@ +package com.amazon.dataprepper.plugins.certificate.s3; + +import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class S3CertificateProviderTest { + @Mock + private AmazonS3 amazonS3; + + @Mock + private S3Object certS3Object; + + @Mock + private S3Object privateKeyS3Object; + + private S3CertificateProvider s3CertificateProvider; + + @Test + public void getCertificateValidKeyPathSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + final String privateKeyContent = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String certificatePath = UUID.randomUUID().toString(); + final String privateKeyPath = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + final String s3SslKeyFile = String.format("s3://%s/%s",bucketName, privateKeyPath); + + final InputStream certObjectStream = IOUtils.toInputStream(certificateContent, StandardCharsets.UTF_8); + final InputStream privateKeyObjectStream = IOUtils.toInputStream(privateKeyContent, StandardCharsets.UTF_8); + + when(certS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(certObjectStream,null)); + when(privateKeyS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(privateKeyObjectStream,null)); + + when(amazonS3.getObject(bucketName, certificatePath)).thenReturn(certS3Object); + when(amazonS3.getObject(bucketName, privateKeyPath)).thenReturn(privateKeyS3Object); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile, s3SslKeyFile); + + final Certificate certificate = s3CertificateProvider.getCertificate(); + + assertThat(certificate.getCertificate(), is(certificateContent)); + assertThat(certificate.getPrivateKey(), is(privateKeyContent)); + } + + @Test + public void getCertificateValidKeyPathS3Exception() { + final String certificatePath = UUID.randomUUID().toString(); + final String privateKeyPath = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + final String s3SslKeyFile = String.format("s3://%s/%s",bucketName, privateKeyPath); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile, s3SslKeyFile); + when(amazonS3.getObject(anyString(), anyString())).thenThrow(new RuntimeException("S3 exception")); + + Assertions.assertThrows(RuntimeException.class, () -> s3CertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index c168984431..c2123acdea 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -3,7 +3,8 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; -import com.amazon.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.certificate.CertificateProviderFactory; import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; @@ -18,7 +19,6 @@ import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.Span; @@ -34,7 +34,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -54,7 +53,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -69,7 +67,10 @@ public class OTelTraceSourceTest { private Server server; @Mock - private ACMCertificateProvider acmCertificateProvider; + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; @Mock private Certificate certificate; @@ -90,8 +91,6 @@ public class OTelTraceSourceTest { .addResourceSpans(ResourceSpans.newBuilder() .addInstrumentationLibrarySpans(InstrumentationLibrarySpans.newBuilder() .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); - private static TraceServiceGrpc.TraceServiceBlockingStub CLIENT; - private BlockingBuffer> getBuffer() { final HashMap integerHashMap = new HashMap<>(); integerHashMap.put("buffer_size", 1); @@ -217,10 +216,16 @@ void testHttpFullBytes() { } @Test - public void testServerStartCertFileSuccess() { + public void testServerStartCertFileSuccess() throws IOException { try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); when(server.stop()).thenReturn(completableFuture); + + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + final Map settingsMap = new HashMap<>(); settingsMap.put(SSL, true); settingsMap.put("useAcmCertForSSL", false); @@ -233,11 +238,13 @@ public void testServerStartCertFileSuccess() { source.start(buffer); source.stop(); - final ArgumentCaptor sslKeyCertChainFile = ArgumentCaptor.forClass(File.class); - final ArgumentCaptor sslKeyFile = ArgumentCaptor.forClass(File.class); - verify(serverBuilder).tls(sslKeyCertChainFile.capture(), sslKeyFile.capture()); - assertThat(sslKeyCertChainFile.getValue().getPath()).isEqualTo("data/certificate/test_cert.crt"); - assertThat(sslKeyFile.getValue().getPath()).isEqualTo("data/certificate/test_decrypted_key.key"); + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate).isEqualTo(certAsString); + assertThat(actualPrivateKey).isEqualTo(keyAsString); } } @@ -252,17 +259,19 @@ public void testServerStartACMCertSuccess() throws IOException { final String keyAsString = Files.readString(keyFilePath); when(certificate.getCertificate()).thenReturn(certAsString); when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(acmCertificateProvider.getACMCertificate(anyString(), anyString())).thenReturn(certificate); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); final Map settingsMap = new HashMap<>(); settingsMap.put(SSL, true); settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); testPluginSetting = new PluginSetting(null, settingsMap); testPluginSetting.setPipelineName("pipeline"); - final OTelTraceSource source = new OTelTraceSource(testPluginSetting, acmCertificateProvider); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); source.start(buffer); source.stop(); From ba8c1ed06469fe34e030005243d4c25af827d3f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jul 2021 04:01:45 +0000 Subject: [PATCH 119/192] Bump cloudwatch from 2.16.92 to 2.16.93 in /data-prepper-core Bumps cloudwatch from 2.16.92 to 2.16.93. --- updated-dependencies: - dependency-name: software.amazon.awssdk:cloudwatch dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 281f621c09..917e9de9ac 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.1" - implementation "software.amazon.awssdk:cloudwatch:2.16.92" + implementation "software.amazon.awssdk:cloudwatch:2.16.93" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.11.2" } From a474d2713c4166d6d68511bdebecef07a4884e03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jul 2021 04:01:51 +0000 Subject: [PATCH 120/192] Bump commons-lang3 in /data-prepper-plugins/otel-trace-source Bumps commons-lang3 from 3.11 to 3.12.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 4bed040fbe..fe9576b592 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - implementation "org.apache.commons:commons-lang3:3.11" + implementation "org.apache.commons:commons-lang3:3.12.0" implementation "org.bouncycastle:bcprov-jdk15on:1.69" implementation "org.bouncycastle:bcpkix-jdk15on:1.69" testImplementation 'org.assertj:assertj-core:3.20.2' From 57a2b8f47c98d0c5a0253c42a28267ee227bb5c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jul 2021 04:02:06 +0000 Subject: [PATCH 121/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.12.14 to 1.12.15. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.14...1.12.15) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index d91cd55734..26b91ccaba 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.12.14" + implementation "com.amazonaws:aws-java-sdk-core:1.12.15" implementation "com.amazonaws:aws-java-sdk-sts:1.12.14" implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" implementation "io.micrometer:micrometer-core:1.7.1" @@ -53,7 +53,7 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.7' + force 'com.amazonaws:aws-java-sdk-core:1.12.15' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' From a92114bf638974850c73935d2e53c0128756edc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Jul 2021 04:02:11 +0000 Subject: [PATCH 122/192] Bump aws-java-sdk-s3 in /data-prepper-plugins/otel-trace-source Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.12.12 to 1.12.15. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.12...1.12.15) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 4bed040fbe..a8aa8463d4 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "commons-io:commons-io:2.10.0" - implementation "com.amazonaws:aws-java-sdk-s3:1.12.12" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" From eb6606aaf6c990568e91d77bfc985306d5c23837 Mon Sep 17 00:00:00 2001 From: Kowshik Nagarajaan Date: Mon, 12 Jul 2021 12:57:06 -0500 Subject: [PATCH 123/192] Upgrade AWS Java SDK v2. --- .../elasticsearch/build.gradle | 8 +- .../ConnectionConfiguration.java | 18 +- .../AwsRequestSigningApacheInterceptor.java | 227 ++++++++++++++++++ ...wsRequestSigningApacheInterceptorTest.java | 138 +++++++++++ 4 files changed, 378 insertions(+), 13 deletions(-) create mode 100644 data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java create mode 100644 data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 26b91ccaba..7e42e12121 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,9 +35,11 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation "com.amazonaws:aws-java-sdk-core:1.12.15" - implementation "com.amazonaws:aws-java-sdk-sts:1.12.14" - implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da" + implementation 'software.amazon.awssdk:auth:2.16.95' + implementation 'software.amazon.awssdk:http-client-spi:2.16.95' + implementation 'software.amazon.awssdk:sdk-core:2.16.95' + implementation 'software.amazon.awssdk:regions:2.16.95' + implementation 'software.amazon.awssdk:utils:2.16.95' implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index a83ba66329..e8b4394763 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -1,10 +1,7 @@ package com.amazon.dataprepper.plugins.sink.elasticsearch; +import com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http.AwsRequestSigningApacheInterceptor; import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazonaws.auth.AWS4Signer; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.http.AWSRequestSigningApacheInterceptor; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; import org.apache.http.auth.AuthScope; @@ -22,6 +19,9 @@ import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.signer.Aws4Signer; import javax.net.ssl.SSLContext; import java.io.InputStream; @@ -175,12 +175,10 @@ private void attachSigV4(final RestClientBuilder restClientBuilder) { //if aws signing is enabled we will add AWSRequestSigningApacheInterceptor interceptor, //if not follow regular credentials process LOG.info("{} is set, will sign requests using AWSRequestSigningApacheInterceptor", AWS_SIGV4); - final AWS4Signer aws4Signer = new AWS4Signer(); - aws4Signer.setServiceName(SERVICE_NAME); - aws4Signer.setRegionName(awsRegion); - final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - final HttpRequestInterceptor httpRequestInterceptor = new AWSRequestSigningApacheInterceptor(SERVICE_NAME, aws4Signer, - credentialsProvider); + final Aws4Signer aws4Signer = Aws4Signer.create(); + final AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create(); + final HttpRequestInterceptor httpRequestInterceptor = new AwsRequestSigningApacheInterceptor(SERVICE_NAME, aws4Signer, + credentialsProvider, awsRegion); restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.addInterceptorLast(httpRequestInterceptor); attachSSLContext(httpClientBuilder); diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java new file mode 100644 index 0000000000..d4466c16cd --- /dev/null +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java @@ -0,0 +1,227 @@ +/* + * Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with + * the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http; + +import org.apache.http.Header; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.regions.Region; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static org.apache.http.protocol.HttpCoreContext.HTTP_TARGET_HOST; + +/** + * An {@link HttpRequestInterceptor} that signs requests using any AWS {@link Signer} + * and {@link AwsCredentialsProvider}. + */ +public class AwsRequestSigningApacheInterceptor implements HttpRequestInterceptor { + /** + * The service that we're connecting to. + */ + private final String service; + + /** + * The particular signer implementation. + */ + private final Signer signer; + + /** + * The source of AWS credentials for signing. + */ + private final AwsCredentialsProvider awsCredentialsProvider; + + /** + * The region signing region. + */ + private final Region region; + + /** + * + * @param service service that we're connecting to + * @param signer particular signer implementation + * @param awsCredentialsProvider source of AWS credentials for signing + * @param region signing region + */ + public AwsRequestSigningApacheInterceptor(final String service, + final Signer signer, + final AwsCredentialsProvider awsCredentialsProvider, + final Region region) { + this.service = service; + this.signer = signer; + this.awsCredentialsProvider = awsCredentialsProvider; + this.region = Objects.requireNonNull(region); + } + + /** + * + * @param service service that we're connecting to + * @param signer particular signer implementation + * @param awsCredentialsProvider source of AWS credentials for signing + * @param region signing region + */ + public AwsRequestSigningApacheInterceptor(final String service, + final Signer signer, + final AwsCredentialsProvider awsCredentialsProvider, + final String region) { + this(service, signer, awsCredentialsProvider, Region.of(region)); + } + + /** + * {@inheritDoc} + */ + @Override + public void process(final HttpRequest request, final HttpContext context) + throws HttpException, IOException { + URIBuilder uriBuilder; + try { + uriBuilder = new URIBuilder(request.getRequestLine().getUri()); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI" , e); + } + + // Copy Apache HttpRequest to AWS Request + SdkHttpFullRequest.Builder requestBuilder = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.fromValue(request.getRequestLine().getMethod())) + .uri(buildUri(context, uriBuilder)); + + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest httpEntityEnclosingRequest = + (HttpEntityEnclosingRequest) request; + if (httpEntityEnclosingRequest.getEntity() != null) { + InputStream content = httpEntityEnclosingRequest.getEntity().getContent(); + requestBuilder.contentStreamProvider(() -> content); + } + } + requestBuilder.rawQueryParameters(nvpToMapParams(uriBuilder.getQueryParams())); + requestBuilder.headers(headerArrayToMap(request.getAllHeaders())); + + ExecutionAttributes attributes = new ExecutionAttributes(); + attributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, awsCredentialsProvider.resolveCredentials()); + attributes.putAttribute(AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME, service); + attributes.putAttribute(AwsSignerExecutionAttribute.SIGNING_REGION, region); + + // Sign it + SdkHttpFullRequest signedRequest = signer.sign(requestBuilder.build(), attributes); + + // Now copy everything back + request.setHeaders(mapToHeaderArray(signedRequest.headers())); + if (request instanceof HttpEntityEnclosingRequest) { + HttpEntityEnclosingRequest httpEntityEnclosingRequest = + (HttpEntityEnclosingRequest) request; + if (httpEntityEnclosingRequest.getEntity() != null) { + BasicHttpEntity basicHttpEntity = new BasicHttpEntity(); + basicHttpEntity.setContent(signedRequest.contentStreamProvider() + .orElseThrow(() -> new IllegalStateException("There must be content")) + .newStream()); + httpEntityEnclosingRequest.setEntity(basicHttpEntity); + } + } + } + + private URI buildUri(final HttpContext context, URIBuilder uriBuilder) throws IOException { + try { + HttpHost host = (HttpHost) context.getAttribute(HTTP_TARGET_HOST); + + if (host != null) { + uriBuilder.setHost(host.getHostName()); + uriBuilder.setScheme(host.getSchemeName()); + uriBuilder.setPort(host.getPort()); + } + + return uriBuilder.build(); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI", e); + } + } + + /** + * + * @param params list of HTTP query params as NameValuePairs + * @return a multimap of HTTP query params + */ + private static Map> nvpToMapParams(final List params) { + Map> parameterMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (NameValuePair nvp : params) { + List argsList = + parameterMap.computeIfAbsent(nvp.getName(), k -> new ArrayList<>()); + argsList.add(nvp.getValue()); + } + return parameterMap; + } + + /** + * @param headers modelled Header objects + * @return a Map of header entries + */ + private static Map> headerArrayToMap(final Header[] headers) { + Map> headersMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (Header header : headers) { + if (!skipHeader(header)) { + headersMap.put(header.getName(), headersMap + .getOrDefault(header.getName(), + new LinkedList<>(Collections.singletonList(header.getValue())))); + } + } + return headersMap; + } + + /** + * @param header header line to check + * @return true if the given header should be excluded when signing + */ + private static boolean skipHeader(final Header header) { + return ("content-length".equalsIgnoreCase(header.getName()) + && "0".equals(header.getValue())) // Strip Content-Length: 0 + || "host".equalsIgnoreCase(header.getName()); // Host comes from endpoint + } + + /** + * @param mapHeaders Map of header entries + * @return modelled Header objects + */ + private static Header[] mapToHeaderArray(final Map> mapHeaders) { + Header[] headers = new Header[mapHeaders.size()]; + int i = 0; + for (Map.Entry> headerEntry : mapHeaders.entrySet()) { + for (String value : headerEntry.getValue()) { + headers[i++] = new BasicHeader(headerEntry.getKey(), value); + } + } + return headers; + } +} diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java new file mode 100644 index 0000000000..c655332540 --- /dev/null +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with + * the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http; + + +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHttpEntityEnclosingRequest; +import org.apache.http.message.BasicHttpRequest; +import org.apache.http.protocol.BasicHttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.signer.Signer; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.regions.Region; + +import java.io.IOException; + +public class AwsRequestSigningApacheInterceptorTest { + private AwsRequestSigningApacheInterceptor interceptor; + + @Before + void createInterceptor() { + AwsCredentialsProvider anonymousCredentialsProvider = + StaticCredentialsProvider.create(AnonymousCredentialsProvider.create().resolveCredentials()); + interceptor = new AwsRequestSigningApacheInterceptor("servicename", + new AddHeaderSigner("Signature", "wuzzle"), + anonymousCredentialsProvider, + Region.AF_SOUTH_1); + } + + @Test + void testSimpleSigner() throws Exception { + HttpEntityEnclosingRequest request = + new BasicHttpEntityEnclosingRequest(new MockRequestLine("/query?a=b")); + request.setEntity(new StringEntity("I'm an entity")); + request.addHeader("foo", "bar"); + request.addHeader("content-length", "0"); + + HttpCoreContext context = new HttpCoreContext(); + context.setTargetHost(HttpHost.create("localhost")); + + interceptor.process(request, context); + + Assert.assertEquals("bar", request.getFirstHeader("foo").getValue()); + Assert.assertEquals("wuzzle", request.getFirstHeader("Signature").getValue()); + Assert.assertNull(request.getFirstHeader("content-length")); + } + + @Test + void testBadRequest() throws Exception { + HttpRequest badRequest = new BasicHttpRequest("GET", "?#!@*%"); + Assert.assertThrows(IOException.class, () -> { + interceptor.process(badRequest, new BasicHttpContext()); + }); + } + + @Test + void testEncodedUriSigner() throws Exception { + HttpEntityEnclosingRequest request = new BasicHttpEntityEnclosingRequest( + new MockRequestLine("/foo-2017-02-25%2Cfoo-2017-02-26/_search?a=b")); + request.setEntity(new StringEntity("I'm an entity")); + request.addHeader("foo", "bar"); + request.addHeader("content-length", "0"); + + HttpCoreContext context = new HttpCoreContext(); + context.setTargetHost(HttpHost.create("localhost")); + + interceptor.process(request, context); + + Assert.assertEquals("bar", request.getFirstHeader("foo").getValue()); + Assert.assertEquals("wuzzle", request.getFirstHeader("Signature").getValue()); + Assert.assertNotNull(request.getFirstHeader("content-length")); + Assert.assertEquals("/foo-2017-02-25%2Cfoo-2017-02-26/_search", request.getFirstHeader("resourcePath").getValue()); + } + + private static class AddHeaderSigner implements Signer { + private final String name; + private final String value; + + private AddHeaderSigner(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public SdkHttpFullRequest sign(SdkHttpFullRequest request, ExecutionAttributes ea) { + if(request.contentStreamProvider().isPresent()){ + ContentStreamProvider contentStreamProvider = request.contentStreamProvider().get(); + return SdkHttpFullRequest.builder() + .uri(request.getUri()) + .method(SdkHttpMethod.GET) + .contentStreamProvider(contentStreamProvider) + .headers(request.headers()) + .appendHeader(name, value) + .appendHeader("resourcePath", request.getUri().getRawPath()) + .build(); + } + return null; + } + } + + private static class MockRequestLine implements RequestLine { + private final String uri; + + public MockRequestLine(String uri) { this.uri = uri; } + + @Override public String getMethod() { return "GET"; } + @Override public String getUri() { return uri; } + + @Override + public ProtocolVersion getProtocolVersion() { + throw new UnsupportedOperationException("Not supported yet."); + } + } +} From bb22c87a2458f64e75aeebb20a2dd144998cce3a Mon Sep 17 00:00:00 2001 From: Kowshik Nagarajaan Date: Tue, 13 Jul 2021 12:22:18 -0500 Subject: [PATCH 124/192] Fix Typo. --- .../ConnectionConfiguration.java | 2 +- .../AwsRequestSigningApacheInterceptor.java | 2 +- ...wsRequestSigningApacheInterceptorTest.java | 29 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) rename data-prepper-plugins/elasticsearch/src/main/java/com/amazon/{dataprpper/plugins/sink/elasticssarch => dataprepper/plugins/sink/elasticsearch}/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java (99%) rename data-prepper-plugins/elasticsearch/src/test/java/com/amazon/{dataprpper/plugins/sink/elasticssarch => dataprepper/plugins/sink/elasticsearch}/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java (82%) diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index e8b4394763..1a690f42ea 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -1,6 +1,6 @@ package com.amazon.dataprepper.plugins.sink.elasticsearch; -import com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http.AwsRequestSigningApacheInterceptor; +import com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http.AwsRequestSigningApacheInterceptor; import com.amazon.dataprepper.model.configuration.PluginSetting; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java similarity index 99% rename from data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java rename to data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java index d4466c16cd..0a8e287d43 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java @@ -10,7 +10,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. */ -package com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http; +package com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http; import org.apache.http.Header; import org.apache.http.HttpEntityEnclosingRequest; diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java similarity index 82% rename from data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java rename to data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java index c655332540..22d1b9d4cd 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprpper/plugins/sink/elasticssarch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java @@ -10,7 +10,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. */ -package com.amazon.dataprpper.plugins.sink.elasticssarch.aws.interceptor.http; +package com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http; import org.apache.http.HttpEntityEnclosingRequest; @@ -42,7 +42,7 @@ public class AwsRequestSigningApacheInterceptorTest { private AwsRequestSigningApacheInterceptor interceptor; @Before - void createInterceptor() { + public void createInterceptor() { AwsCredentialsProvider anonymousCredentialsProvider = StaticCredentialsProvider.create(AnonymousCredentialsProvider.create().resolveCredentials()); interceptor = new AwsRequestSigningApacheInterceptor("servicename", @@ -52,7 +52,7 @@ void createInterceptor() { } @Test - void testSimpleSigner() throws Exception { + public void testSimpleSigner() throws Exception { HttpEntityEnclosingRequest request = new BasicHttpEntityEnclosingRequest(new MockRequestLine("/query?a=b")); request.setEntity(new StringEntity("I'm an entity")); @@ -70,7 +70,7 @@ void testSimpleSigner() throws Exception { } @Test - void testBadRequest() throws Exception { + public void testBadRequest() throws Exception { HttpRequest badRequest = new BasicHttpRequest("GET", "?#!@*%"); Assert.assertThrows(IOException.class, () -> { interceptor.process(badRequest, new BasicHttpContext()); @@ -78,7 +78,26 @@ void testBadRequest() throws Exception { } @Test - void testEncodedUriSigner() throws Exception { + public void testEncodedUriSigner() throws Exception { + HttpEntityEnclosingRequest request = new BasicHttpEntityEnclosingRequest( + new MockRequestLine("/foo-2017-02-25%2Cfoo-2017-02-26/_search?a=b")); + request.setEntity(new StringEntity("I'm an entity")); + request.addHeader("foo", "bar"); + request.addHeader("content-length", "0"); + + HttpCoreContext context = new HttpCoreContext(); + context.setTargetHost(HttpHost.create("localhost")); + + interceptor.process(request, context); + + Assert.assertEquals("bar", request.getFirstHeader("foo").getValue()); + Assert.assertEquals("wuzzle", request.getFirstHeader("Signature").getValue()); + Assert.assertNotNull(request.getFirstHeader("content-length")); + Assert.assertEquals("/foo-2017-02-25%2Cfoo-2017-02-26/_search", request.getFirstHeader("resourcePath").getValue()); + } + + @Test + public void testHttpRequest () throws Exception { HttpEntityEnclosingRequest request = new BasicHttpEntityEnclosingRequest( new MockRequestLine("/foo-2017-02-25%2Cfoo-2017-02-26/_search?a=b")); request.setEntity(new StringEntity("I'm an entity")); From 6143a753a8776c946af42d9cc116e41a86d2a44d Mon Sep 17 00:00:00 2001 From: Kowshik Nagarajaan Date: Tue, 13 Jul 2021 12:31:33 -0500 Subject: [PATCH 125/192] Make Private and Final. --- .../http => }/AwsRequestSigningApacheInterceptor.java | 4 ++-- .../plugins/sink/elasticsearch/ConnectionConfiguration.java | 1 - .../http => }/AwsRequestSigningApacheInterceptorTest.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) rename data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/{aws/interceptor/http => }/AwsRequestSigningApacheInterceptor.java (98%) rename data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/{aws/interceptor/http => }/AwsRequestSigningApacheInterceptorTest.java (98%) diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java similarity index 98% rename from data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java rename to data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java index 0a8e287d43..af0fcc6ecc 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptor.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java @@ -10,7 +10,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. */ -package com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http; +package com.amazon.dataprepper.plugins.sink.elasticsearch; import org.apache.http.Header; import org.apache.http.HttpEntityEnclosingRequest; @@ -49,7 +49,7 @@ * An {@link HttpRequestInterceptor} that signs requests using any AWS {@link Signer} * and {@link AwsCredentialsProvider}. */ -public class AwsRequestSigningApacheInterceptor implements HttpRequestInterceptor { +final class AwsRequestSigningApacheInterceptor implements HttpRequestInterceptor { /** * The service that we're connecting to. */ diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index 1a690f42ea..51d8d5ab08 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -1,6 +1,5 @@ package com.amazon.dataprepper.plugins.sink.elasticsearch; -import com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http.AwsRequestSigningApacheInterceptor; import com.amazon.dataprepper.model.configuration.PluginSetting; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptorTest.java similarity index 98% rename from data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java rename to data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptorTest.java index 22d1b9d4cd..55e4d4c565 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/aws/interceptor/http/AwsRequestSigningApacheInterceptorTest.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptorTest.java @@ -10,7 +10,7 @@ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. */ -package com.amazon.dataprepper.plugins.sink.elasticsearch.aws.interceptor.http; +package com.amazon.dataprepper.plugins.sink.elasticsearch; import org.apache.http.HttpEntityEnclosingRequest; From af00f7454c675bedd7d9f8915b640afab6ec4361 Mon Sep 17 00:00:00 2001 From: Kowshik Nagarajaan Date: Tue, 13 Jul 2021 12:39:54 -0500 Subject: [PATCH 126/192] Fix PR comments on Magic strings and Null checks. --- .../AwsRequestSigningApacheInterceptor.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java index af0fcc6ecc..7737b56a7f 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/AwsRequestSigningApacheInterceptor.java @@ -50,6 +50,20 @@ * and {@link AwsCredentialsProvider}. */ final class AwsRequestSigningApacheInterceptor implements HttpRequestInterceptor { + + /** + * Constant to check content-length + */ + private static final String CONTENT_LENGTH = "content-length"; + /** + * Constant to check Zero content length + */ + private static final String ZERO_CONTENT_LENGTH = "0"; + /** + * Constant to check if host is the endpoint + */ + private static final String HOST = "host"; + /** * The service that we're connecting to. */ @@ -81,9 +95,9 @@ public AwsRequestSigningApacheInterceptor(final String service, final Signer signer, final AwsCredentialsProvider awsCredentialsProvider, final Region region) { - this.service = service; - this.signer = signer; - this.awsCredentialsProvider = awsCredentialsProvider; + this.service = Objects.requireNonNull(service); + this.signer = Objects.requireNonNull(signer); + this.awsCredentialsProvider = Objects.requireNonNull(awsCredentialsProvider); this.region = Objects.requireNonNull(region); } @@ -205,9 +219,9 @@ private static Map> headerArrayToMap(final Header[] headers * @return true if the given header should be excluded when signing */ private static boolean skipHeader(final Header header) { - return ("content-length".equalsIgnoreCase(header.getName()) - && "0".equals(header.getValue())) // Strip Content-Length: 0 - || "host".equalsIgnoreCase(header.getName()); // Host comes from endpoint + return (CONTENT_LENGTH.equalsIgnoreCase(header.getName()) + && ZERO_CONTENT_LENGTH.equals(header.getValue())) // Strip Content-Length: 0 + || HOST.equalsIgnoreCase(header.getName()); // Host comes from endpoint } /** From dfe692418695965f10d6ec76d8a704b2caa4ec84 Mon Sep 17 00:00:00 2001 From: dinujoh <86094133+dinujoh@users.noreply.github.com> Date: Thu, 15 Jul 2021 09:32:16 -0500 Subject: [PATCH 127/192] Add support to read certificate from ACM and S3 for peer forwarder (#726) Add support to read certificate from ACM and S3 for peer forwarder trust manager Changes include: * Added support to build the trust manager by abstracting how the certificates are read. The certificate can be local file system location, AWS ACM ARN or AWS S3 location. * Added unit test to support above case. Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- data-prepper-plugins/peer-forwarder/README.md | 12 ++- .../peer-forwarder/build.gradle | 7 ++ .../prepper/peerforwarder/PeerClientPool.java | 15 ++-- .../peerforwarder/PeerForwarderConfig.java | 42 ++++++---- .../certificate/CertificateProvider.java | 7 ++ .../CertificateProviderConfig.java | 43 ++++++++++ .../CertificateProviderFactory.java | 51 ++++++++++++ .../acm/ACMCertificateProvider.java | 58 +++++++++++++ .../file/FileCertificateProvider.java | 33 ++++++++ .../certificate/model/Certificate.java | 18 ++++ .../certificate/s3/S3CertificateProvider.java | 44 ++++++++++ .../peerforwarder/PeerClientPoolTest.java | 12 ++- .../PeerForwarderConfigTest.java | 20 +++-- .../CertificateProviderFactoryTest.java | 82 +++++++++++++++++++ .../acm/ACMCertificateProviderTest.java | 67 +++++++++++++++ .../file/FileCertificateProviderTest.java | 42 ++++++++++ .../s3/S3CertificateProviderTest.java | 65 +++++++++++++++ 17 files changed, 588 insertions(+), 30 deletions(-) create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java diff --git a/data-prepper-plugins/peer-forwarder/README.md b/data-prepper-plugins/peer-forwarder/README.md index 64437a1417..5a4c811c1d 100644 --- a/data-prepper-plugins/peer-forwarder/README.md +++ b/data-prepper-plugins/peer-forwarder/README.md @@ -49,8 +49,16 @@ A DNS server (like [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html)) can * `discovery_mode`: peer discovery mode to be used. Allowable values are `static` and `dns`. Defaults to `static` * `static_endpoints`: list containing endpoints of all Data Prepper instances. * `domain_name`: single domain name to query DNS against. Typically used by creating multiple [DNS A Records](https://www.cloudflare.com/learning/dns/dns-records/dns-a-record/) for the same domain. -* `ssl` => Default is ```true```. -* `sslKeyCertChainFile` => Should be provided if ```ssl``` is set to ```true``` + +### SSL +The SSL configuration for setting up trust manager for peer forwarding client to connect to other Data Prepper instances. The SSL configuration should be same as the one used for OTel Trace Source. + +* `ssl(Optional)` => A boolean enables TLS/SSL. Default is ```true```. +* `sslKeyCertChainFile(Optional)` => A `String` represents the SSL certificate chain file path or AWS S3 path. S3 path example ```s3:///```. Required if ```ssl``` is set to ```true```. +* `useAcmCertForSSL(Optional)` => A boolean enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is ```false```. +* `acmCertificateArn(Optional)` => A `String` represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if ```useAcmCertForSSL``` is set to ```true```. +* `awsRegion(Optional)` => A `String` represents the AWS region to use ACM or S3. Required if ```useAcmCertForSSL``` is set to ```true``` or ```sslKeyCertChainFile``` and ```sslKeyFile``` is ```AWS S3 path```. + ## Metrics diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 42785f0643..00abed9f4f 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -15,9 +15,16 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" + implementation "commons-io:commons-io:2.10.0" + implementation "org.apache.commons:commons-lang3:3.12.0" implementation "commons-validator:commons-validator:1.7" testImplementation "junit:junit:4.13.2" + testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-inline:3.11.2" + testImplementation "org.mockito:mockito-core:3.11.2" + testImplementation "commons-io:commons-io:2.10.0" } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java index e5525bf7e3..5ea2503e10 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java @@ -1,11 +1,13 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.linecorp.armeria.client.ClientBuilder; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.Clients; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import java.io.File; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -18,7 +20,7 @@ public class PeerClientPool { private int clientTimeoutSeconds = 3; private boolean ssl; - private File sslKeyCertChainFile; + private Certificate certificate; private PeerClientPool() { peerClients = new ConcurrentHashMap<>(); @@ -34,8 +36,8 @@ public void setClientTimeoutSeconds(int clientTimeoutSeconds) { public void setSsl(boolean ssl) { this.ssl = ssl; } - public void setSslKeyCertChainFile(File sslKeyCertChainFile) { - this.sslKeyCertChainFile = sslKeyCertChainFile; + public void setCertificate(final Certificate certificate) { + this.certificate = certificate; } public TraceServiceGrpc.TraceServiceBlockingStub getClient(final String address) { @@ -50,7 +52,10 @@ private TraceServiceGrpc.TraceServiceBlockingStub createGRPCClient(final String clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTPS, ipAddress)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)) .factory(ClientFactory.builder() - .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager(sslKeyCertChainFile)).build()); + .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager( + new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)) + )).build() + ); } else { clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTP, ipAddress)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)); diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java index c8fb81ac3f..c1f7b21ff6 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java @@ -1,12 +1,11 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProviderConfig; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProviderFactory; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.PeerListProvider; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.PeerListProviderFactory; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Paths; +import org.apache.commons.lang3.StringUtils; import static com.google.common.base.Preconditions.checkNotNull; @@ -20,6 +19,12 @@ public class PeerForwarderConfig { public static final String SSL = "ssl"; public static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; private static final boolean DEFAULT_SSL = true; + private static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; + private static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; + private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; + private static final String ACM_CERT_ARN = "acmCertificateArn"; + private static final String AWS_REGION = "awsRegion"; private final HashRing hashRing; private final PeerClientPool peerClientPool; @@ -46,20 +51,27 @@ public static PeerForwarderConfig buildConfig(final PluginSetting pluginSetting) peerClientPool.setClientTimeoutSeconds(3); final boolean ssl = pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL); final String sslKeyCertChainFilePath = pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null); - final File sslKeyCertChainFile; - if (ssl) { - if (sslKeyCertChainFilePath == null || sslKeyCertChainFilePath.isEmpty()) { + final boolean useAcmCertForSsl = pluginSetting.getBooleanOrDefault(USE_ACM_CERT_FOR_SSL, DEFAULT_USE_ACM_CERT_FOR_SSL); + + if (ssl || useAcmCertForSsl) { + if (ssl && StringUtils.isEmpty(sslKeyCertChainFilePath)) { throw new IllegalArgumentException(String.format("%s is enabled, %s can not be empty or null", SSL, SSL_KEY_CERT_FILE)); - } else if (!Files.exists(Paths.get(sslKeyCertChainFilePath))) { - throw new IllegalArgumentException(String.format("%s is enabled, %s does not exist", SSL, SSL_KEY_CERT_FILE)); - } else { - sslKeyCertChainFile = new File(sslKeyCertChainFilePath); } - } else { - sslKeyCertChainFile = null; + peerClientPool.setSsl(true); + final String acmCertificateArn = pluginSetting.getStringOrDefault(ACM_CERT_ARN, null); + final long acmCertIssueTimeOutMillis = pluginSetting.getLongOrDefault(ACM_CERT_ISSUE_TIME_OUT_MILLIS, DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS); + final String awsRegion = pluginSetting.getStringOrDefault(AWS_REGION, null); + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSsl, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFilePath + ); + final CertificateProviderFactory certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + peerClientPool.setCertificate(certificateProviderFactory.getCertificateProvider().getCertificate()); + } - peerClientPool.setSsl(ssl); - peerClientPool.setSslKeyCertChainFile(sslKeyCertChainFile); return new PeerForwarderConfig( peerClientPool, diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java new file mode 100644 index 0000000000..a9ef3665d1 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProvider.java @@ -0,0 +1,7 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; + +public interface CertificateProvider { + Certificate getCertificate(); +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java new file mode 100644 index 0000000000..dc29ca9b24 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderConfig.java @@ -0,0 +1,43 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +public class CertificateProviderConfig { + private static final String S3_PREFIX = "s3://"; + + private final boolean useAcmCertForSSL; + private final String acmCertificateArn; + private final String awsRegion; + private final long acmCertIssueTimeOutMillis; + private final String sslKeyCertChainFile; + + public CertificateProviderConfig(final boolean useAcmCertForSSL, final String acmCertificateArn, final String awsRegion, final long acmCertIssueTimeOutMillis, final String sslKeyCertChainFile) { + this.useAcmCertForSSL = useAcmCertForSSL; + this.acmCertificateArn = acmCertificateArn; + this.awsRegion = awsRegion; + this.acmCertIssueTimeOutMillis = acmCertIssueTimeOutMillis; + this.sslKeyCertChainFile = sslKeyCertChainFile; + } + + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public String getAcmCertificateArn() { + return acmCertificateArn; + } + + public String getAwsRegion() { + return awsRegion; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis; + } + + public String getSslKeyCertChainFile() { + return sslKeyCertChainFile; + } + + public boolean isSslCertFileInS3() { + return sslKeyCertChainFile.toLowerCase().startsWith(S3_PREFIX); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java new file mode 100644 index 0000000000..c6e5c2b9b7 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactory.java @@ -0,0 +1,51 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3.S3CertificateProvider; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.AWSCertificateManagerClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CertificateProviderFactory { + private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); + + final CertificateProviderConfig certificateProviderConfig; + public CertificateProviderFactory(final CertificateProviderConfig certificateProviderConfig) { + this.certificateProviderConfig = certificateProviderConfig; + } + + public CertificateProvider getCertificateProvider() { + // ACM Cert for SSL takes preference + if (certificateProviderConfig.useAcmCertForSSL()) { + LOG.info("Using ACM certificate for SSL/TLS to setup trust store."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final ClientConfiguration clientConfig = new ClientConfiguration() + .withThrottledRetries(true); + final AWSCertificateManager awsCertificateManager = AWSCertificateManagerClientBuilder.standard() + .withRegion(certificateProviderConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig) + .build(); + return new ACMCertificateProvider(awsCertificateManager, certificateProviderConfig.getAcmCertificateArn(), + certificateProviderConfig.getAcmCertIssueTimeOutMillis()); + } else if (certificateProviderConfig.isSslCertFileInS3()) { + LOG.info("Using S3 to fetch certificate for SSL/TLS to setup trust store."); + final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); + final AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(certificateProviderConfig.getAwsRegion()) + .withCredentials(credentialsProvider) + .build(); + return new S3CertificateProvider(s3Client, certificateProviderConfig.getSslKeyCertChainFile()); + } else { + LOG.info("Using local file system to get certificate for SSL/TLS to setup trust store."); + return new FileCertificateProvider(certificateProviderConfig.getSslKeyCertChainFile()); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java new file mode 100644 index 0000000000..0281542912 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProvider.java @@ -0,0 +1,58 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.GetCertificateRequest; +import com.amazonaws.services.certificatemanager.model.GetCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +public class ACMCertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); + private static final long SLEEP_INTERVAL = 10000L; + private final AWSCertificateManager awsCertificateManager; + private final String acmArn; + private final long totalTimeout; + public ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, + final String acmArn, + final long totalTimeout) { + this.awsCertificateManager = Objects.requireNonNull(awsCertificateManager); + this.acmArn = Objects.requireNonNull(acmArn); + this.totalTimeout = totalTimeout; + } + + public Certificate getCertificate() { + GetCertificateResult getCertificateResult = null; + long timeSlept = 0L; + + while (getCertificateResult == null && timeSlept < totalTimeout) { + try { + final GetCertificateRequest getCertificateRequest = new GetCertificateRequest() + .withCertificateArn(acmArn); + getCertificateResult = awsCertificateManager.getCertificate(getCertificateRequest); + + } catch (final RequestInProgressException ex) { + try { + Thread.sleep(SLEEP_INTERVAL); + } catch (InterruptedException iex) { + throw new RuntimeException(iex); + } + } catch (final ResourceNotFoundException | InvalidArnException ex) { + LOG.error("Exception retrieving the certificate with arn: {}", acmArn, ex); + throw ex; + } + timeSlept += SLEEP_INTERVAL; + } + if(getCertificateResult != null) { + return new Certificate(getCertificateResult.getCertificate()); + } else { + throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java new file mode 100644 index 0000000000..ca9b867c34 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProvider.java @@ -0,0 +1,33 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +public class FileCertificateProvider implements CertificateProvider { + private final String certificateFilePath; + + public FileCertificateProvider(final String certificateFilePath) { + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + } + + private static final Logger LOG = LoggerFactory.getLogger(FileCertificateProvider.class); + + public Certificate getCertificate() { + try { + final Path certFilePath = Path.of(certificateFilePath); + + final String certAsString = Files.readString(certFilePath); + + return new Certificate(certAsString); + } catch (final Exception ex) { + LOG.error("Error encountered while reading the certificate.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java new file mode 100644 index 0000000000..68df09e3e8 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/model/Certificate.java @@ -0,0 +1,18 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model; + +import static java.util.Objects.requireNonNull; + +public class Certificate { + /** + * The base64 PEM-encoded certificate. + */ + private String certificate; + + public Certificate(final String certificate) { + this.certificate = requireNonNull(certificate, "certificate must not be null"); + } + + public String getCertificate() { + return certificate; + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java new file mode 100644 index 0000000000..0e7a1937f6 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProvider.java @@ -0,0 +1,44 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.CertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +public class S3CertificateProvider implements CertificateProvider { + private static final Logger LOG = LoggerFactory.getLogger(S3CertificateProvider.class); + private final AmazonS3 s3Client; + private final String certificateFilePath; + + public S3CertificateProvider(final AmazonS3 s3Client, + final String certificateFilePath) { + this.s3Client = Objects.requireNonNull(s3Client); + this.certificateFilePath = Objects.requireNonNull(certificateFilePath); + } + + public Certificate getCertificate() { + final AmazonS3URI certificateS3URI = new AmazonS3URI(certificateFilePath); + final String certificate = getObjectWithKey(certificateS3URI.getBucket(), certificateS3URI.getKey()); + + return new Certificate(certificate); + } + + private String getObjectWithKey(final String bucketName, final String key) { + + // Download the object + try (final S3Object s3Object = s3Client.getObject(bucketName, key)) { + LOG.info("Object with key \"{}\" downloaded.", key); + return IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8); + } catch (final Exception ex) { + LOG.error("Error encountered while processing the response from Amazon S3.", ex); + throw new RuntimeException(ex); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java index ac1ae35198..8f7d53d300 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java @@ -1,5 +1,6 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -10,6 +11,9 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import static org.junit.Assert.assertNotNull; @@ -32,7 +36,7 @@ public void testGetClientValidAddress() { } @Test - public void testGetClientWithSSL() { + public void testGetClientWithSSL() throws IOException { // Set up test server with SSL ServerBuilder sb = Server.builder(); sb.service(GrpcService.builder() @@ -46,7 +50,11 @@ public void testGetClientWithSSL() { // Configure client pool PeerClientPool pool = PeerClientPool.getInstance(); pool.setSsl(true); - pool.setSslKeyCertChainFile(SSL_CRT_FILE); + + final Path certFilePath = Path.of(PeerClientPoolTest.class.getClassLoader().getResource("test-crt.crt").getPath()); + final String certAsString = Files.readString(certFilePath); + final Certificate certificate = new Certificate(certAsString); + pool.setCertificate(certificate); TraceServiceGrpc.TraceServiceBlockingStub client = pool.getClient(LOCALHOST); assertNotNull(client); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java index 76599b7cd9..f3d39f2a6b 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfigTest.java @@ -1,22 +1,25 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; import com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.DiscoveryMode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.junit.MockitoJUnitRunner; -import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mockStatic; @@ -42,7 +45,6 @@ public void setUp() { mockedPeerClientPoolClass = mockStatic(PeerClientPool.class); mockedPeerClientPoolClass.when(PeerClientPool::getInstance).thenReturn(peerClientPool); doNothing().when(peerClientPool).setSsl(anyBoolean()); - doNothing().when(peerClientPool).setSslKeyCertChainFile(any(File.class)); } @After @@ -87,13 +89,13 @@ public void testBuildConfigInvalidSSL() { }); settings.put(PeerForwarderConfig.SSL_KEY_CERT_FILE, INVALID_SSL_KEY_CERT_FILE); - Assert.assertThrows(IllegalArgumentException.class, () -> { + Assert.assertThrows(RuntimeException.class, () -> { PeerForwarderConfig.buildConfig(new PluginSetting("peer_forwarder", settings){{ setPipelineName(PIPELINE_NAME); }}); }); } @Test - public void testBuildConfigValidSSL() { + public void testBuildConfigValidSSL() throws IOException { final HashMap settings = new HashMap<>(); settings.put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); settings.put(PeerForwarderConfig.STATIC_ENDPOINTS, TEST_ENDPOINTS); @@ -102,6 +104,12 @@ public void testBuildConfigValidSSL() { PeerForwarderConfig.buildConfig(new PluginSetting("peer_forwarder", settings){{ setPipelineName(PIPELINE_NAME); }}); verify(peerClientPool, times(1)).setSsl(true); - verify(peerClientPool, times(1)).setSslKeyCertChainFile(new File(VALID_SSL_KEY_CERT_FILE)); + final ArgumentCaptor certificateArgumentCaptor = ArgumentCaptor.forClass(Certificate.class); + verify(peerClientPool, times(1)).setCertificate(certificateArgumentCaptor.capture()); + final Certificate certificate = certificateArgumentCaptor.getValue(); + + final Path certFilePath = Path.of(VALID_SSL_KEY_CERT_FILE); + final String certAsString = Files.readString(certFilePath); + Assert.assertEquals(certificate.getCertificate(), certAsString); } } \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java new file mode 100644 index 0000000000..9f872b7e5e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/CertificateProviderFactoryTest.java @@ -0,0 +1,82 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm.ACMCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file.FileCertificateProvider; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3.S3CertificateProvider; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class CertificateProviderFactoryTest { + private CertificateProviderFactory certificateProviderFactory; + + @Test + public void getCertificateProviderAcmProviderSuccess() { + final boolean useAcmCertForSSL = true; + final String awsRegion = "us-east-1"; + final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + final String sslKeyCertChainFile = null; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); + } + + @Test + public void getCertificateProviderS3ProviderSuccess() { + final boolean useAcmCertForSSL = false; + final String awsRegion = "us-east-1"; + final String acmCertificateArn = null; + final String sslKeyCertChainFile = "s3://some_s3_bucket/certificate/test_cert.crt"; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider();; + + assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); + } + + @Test + public void getCertificateProviderFileProviderSuccess() { + final boolean useAcmCertForSSL = false; + final String awsRegion = null; + final String acmCertificateArn = null; + final String sslKeyCertChainFile = "path_to_certificate/test_cert.crt"; + final long acmCertIssueTimeOutMillis = 1000L; + + final CertificateProviderConfig certificateProviderConfig = new CertificateProviderConfig( + useAcmCertForSSL, + acmCertificateArn, + awsRegion, + acmCertIssueTimeOutMillis, + sslKeyCertChainFile + ); + + certificateProviderFactory = new CertificateProviderFactory(certificateProviderConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java new file mode 100644 index 0000000000..6150c7925c --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/acm/ACMCertificateProviderTest.java @@ -0,0 +1,67 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.acm; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.certificatemanager.AWSCertificateManager; +import com.amazonaws.services.certificatemanager.model.GetCertificateRequest; +import com.amazonaws.services.certificatemanager.model.GetCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.UUID; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class ACMCertificateProviderTest { + private static final String acmCertificateArn = "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"; + private static final long acmCertIssueTimeOutMillis = 2000L; + @Mock + private AWSCertificateManager awsCertificateManager; + + @Mock + private GetCertificateResult getCertificateResult; + + private ACMCertificateProvider acmCertificateProvider; + + @Before + public void beforeEach() { + acmCertificateProvider = new ACMCertificateProvider(awsCertificateManager, acmCertificateArn, acmCertIssueTimeOutMillis); + } + + @Test + public void getACMCertificateSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + when(getCertificateResult.getCertificate()).thenReturn(certificateContent); + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenReturn(getCertificateResult); + final Certificate certificate = acmCertificateProvider.getCertificate(); + assertThat(certificate.getCertificate(), is(certificateContent)); + } + + @Test + public void getACMCertificateRequestInProgressException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new RequestInProgressException("Request in progress.")); + assertThrows(IllegalStateException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateResourceNotFoundException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new ResourceNotFoundException("Resource not found.")); + assertThrows(ResourceNotFoundException.class, () -> acmCertificateProvider.getCertificate()); + } + + @Test + public void getACMCertificateInvalidArnException() { + when(awsCertificateManager.getCertificate(any(GetCertificateRequest.class))).thenThrow(new InvalidArnException("Invalid certificate arn.")); + assertThrows(InvalidArnException.class, () -> acmCertificateProvider.getCertificate()); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java new file mode 100644 index 0000000000..417a48721b --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/file/FileCertificateProviderTest.java @@ -0,0 +1,42 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.file; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class FileCertificateProviderTest { + + private FileCertificateProvider fileCertificateProvider; + + @Test + public void getCertificateValidPathSuccess() throws IOException { + final String certificateFilePath = FileCertificateProviderTest.class.getClassLoader().getResource("test-crt.crt").getPath(); + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath); + + final Certificate certificate = fileCertificateProvider.getCertificate(); + + final Path certFilePath = Path.of(certificateFilePath); + final String certAsString = Files.readString(certFilePath); + + assertThat(certificate.getCertificate(), is(certAsString)); + } + + @Test(expected = RuntimeException.class) + public void getCertificateInvalidPathSuccess() { + final String certificateFilePath = "path_does_not_exit/test_cert.crt"; + + fileCertificateProvider = new FileCertificateProvider(certificateFilePath); + + fileCertificateProvider.getCertificate(); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java new file mode 100644 index 0000000000..0bd0db5656 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/certificate/s3/S3CertificateProviderTest.java @@ -0,0 +1,65 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.s3; + +import com.amazon.dataprepper.plugins.prepper.peerforwarder.certificate.model.Certificate; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class S3CertificateProviderTest { + @Mock + private AmazonS3 amazonS3; + + @Mock + private S3Object certS3Object; + + private S3CertificateProvider s3CertificateProvider; + + @Test + public void getCertificateValidKeyPathSuccess() { + final String certificateContent = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String certificatePath = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + + final InputStream certObjectStream = IOUtils.toInputStream(certificateContent, StandardCharsets.UTF_8); + + when(certS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(certObjectStream,null)); + + when(amazonS3.getObject(bucketName, certificatePath)).thenReturn(certS3Object); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile); + + final Certificate certificate = s3CertificateProvider.getCertificate(); + + assertThat(certificate.getCertificate(), is(certificateContent)); + } + + @Test(expected = RuntimeException.class) + public void getCertificateValidKeyPathS3Exception() { + final String certificatePath = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + + final String s3SslKeyCertChainFile = String.format("s3://%s/%s",bucketName, certificatePath); + + s3CertificateProvider = new S3CertificateProvider(amazonS3, s3SslKeyCertChainFile); + when(amazonS3.getObject(anyString(), anyString())).thenThrow(new RuntimeException("S3 exception")); + + s3CertificateProvider.getCertificate(); + } +} From ad6cdf6d940c273a768eefe5a56f8054627e95c1 Mon Sep 17 00:00:00 2001 From: David Venable Date: Mon, 19 Jul 2021 13:23:06 -0500 Subject: [PATCH 128/192] Only register the Health gRPC service when it is configured to be used. Signed-off-by: David Venable --- .../source/oteltrace/OTelTraceSource.java | 1 - .../source/oteltrace/OTelTraceSourceTest.java | 86 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 26d09c6a62..14df7eb8ec 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -60,7 +60,6 @@ public void start(Buffer> buffer) { buffer, pluginMetrics )) - .addService(new HealthGrpcService()) .useClientTimeoutHeader(false); if (oTelTraceSourceConfig.hasHealthCheck()) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index c2123acdea..d5b33a9540 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -6,6 +6,7 @@ import com.amazon.dataprepper.plugins.certificate.CertificateProvider; import com.amazon.dataprepper.plugins.certificate.CertificateProviderFactory; import com.amazon.dataprepper.plugins.certificate.model.Certificate; +import com.amazon.dataprepper.plugins.health.HealthGrpcService; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.client.ClientFactory; @@ -18,6 +19,8 @@ import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import io.grpc.BindableService; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; @@ -52,8 +55,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -66,6 +72,12 @@ public class OTelTraceSourceTest { @Mock private Server server; + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + @Mock private CertificateProviderFactory certificateProviderFactory; @@ -106,6 +118,10 @@ public void beforeEach() { lenient().when(serverBuilder.build()).thenReturn(server); lenient().when(server.start()).thenReturn(completableFuture); + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + final Map settingsMap = new HashMap<>(); settingsMap.put("request_timeout", 5); settingsMap.put(SSL, false); @@ -285,6 +301,76 @@ public void testServerStartACMCertSuccess() throws IOException { } } + @Test + void start_with_Health_configured_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + } + + @Test + void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + final OTelTraceSource source = new OTelTraceSource(testPluginSetting, certificateProviderFactory); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + } + @Test public void testDoubleStart() { // starting server From a66f2a81de144b7a086e64a640882ddd445bb6ed Mon Sep 17 00:00:00 2001 From: Kowshik <57502753+kowshikn@users.noreply.github.com> Date: Thu, 22 Jul 2021 17:55:30 -0600 Subject: [PATCH 129/192] Add support of STS assume role. (#736) * Add STS Assume Role Support. * Fix code format of Getters. * Use Pipelinename on session name. --- data-prepper-plugins/elasticsearch/README.md | 2 + .../elasticsearch/build.gradle | 3 ++ .../ConnectionConfiguration.java | 53 +++++++++++++++++-- .../ConnectionConfigurationTests.java | 50 +++++++++++------ 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/README.md b/data-prepper-plugins/elasticsearch/README.md index 7f0bc798ef..197bf925ba 100644 --- a/data-prepper-plugins/elasticsearch/README.md +++ b/data-prepper-plugins/elasticsearch/README.md @@ -70,6 +70,8 @@ Default is null. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. +- `aws_sts_role`: A IAM role which the sink plugin will assume to sign request to Amazon Elasticsearch. If not provided the plugin will use the default credentials. + - `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to `false`. - `username`(optional): A String of username used in the [internal users](https://opendistro.github.io/for-elasticsearch-docs/docs/security/access-control/users-roles) of ODFE cluster. Default is null. diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 7e42e12121..a4a983976b 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -38,8 +38,11 @@ dependencies { implementation 'software.amazon.awssdk:auth:2.16.95' implementation 'software.amazon.awssdk:http-client-spi:2.16.95' implementation 'software.amazon.awssdk:sdk-core:2.16.95' + implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.16.95' implementation 'software.amazon.awssdk:utils:2.16.95' + implementation 'software.amazon.awssdk:sts:2.16.95' + implementation 'software.amazon.awssdk:url-connection-client:2.16.95' implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index 51d8d5ab08..15ce300f72 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -21,6 +21,9 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import javax.net.ssl.SSLContext; import java.io.InputStream; @@ -31,6 +34,7 @@ import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.util.List; +import java.util.UUID; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -50,6 +54,7 @@ public class ConnectionConfiguration { public static final String INSECURE = "insecure"; public static final String AWS_SIGV4 = "aws_sigv4"; public static final String AWS_REGION = "aws_region"; + public static final String AWS_STS_ROLE = "aws_sts_role"; private final List hosts; private final String username; @@ -60,6 +65,8 @@ public class ConnectionConfiguration { private final boolean insecure; private final boolean awsSigv4; private final String awsRegion; + private final String awsStsRole; + private final String pipelineName; public List getHosts() { return hosts; @@ -81,6 +88,14 @@ public String getAwsRegion() { return awsRegion; } + public String getAwsStsRole() { + return awsStsRole; + } + + public Path getCertPath() { + return certPath; + } + public Integer getSocketTimeout() { return socketTimeout; } @@ -99,6 +114,8 @@ private ConnectionConfiguration(final Builder builder) { this.insecure = builder.insecure; this.awsSigv4 = builder.awsSigv4; this.awsRegion = builder.awsRegion; + this.awsStsRole = builder.awsStsRole; + this.pipelineName = builder.pipelineName; } public static ConnectionConfiguration readConnectionConfiguration(final PluginSetting pluginSetting){ @@ -106,6 +123,7 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe final List hosts = (List) pluginSetting.getAttributeFromSettings(HOSTS); ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder(hosts); final String username = (String) pluginSetting.getAttributeFromSettings(USERNAME); + builder.withPipelineName(pluginSetting.getPipelineName()); if (username != null) { builder = builder.withUsername(username); } @@ -123,7 +141,10 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe } builder.withAwsSigv4(pluginSetting.getBooleanOrDefault(AWS_SIGV4, false)); - builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); + if (builder.awsSigv4) { + builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); + builder.withAWSStsRole(pluginSetting.getStringOrDefault(AWS_STS_ROLE, null)); + } final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); @@ -136,8 +157,8 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe return builder.build(); } - public Path getCertPath() { - return certPath; + public String getPipelineName() { + return pipelineName; } public RestHighLevelClient createClient() { @@ -175,7 +196,19 @@ private void attachSigV4(final RestClientBuilder restClientBuilder) { //if not follow regular credentials process LOG.info("{} is set, will sign requests using AWSRequestSigningApacheInterceptor", AWS_SIGV4); final Aws4Signer aws4Signer = Aws4Signer.create(); - final AwsCredentialsProvider credentialsProvider = DefaultCredentialsProvider.create(); + AwsCredentialsProvider credentialsProvider; + if (awsStsRole != null && !awsStsRole.isEmpty()) { + credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .stsClient(StsClient.create()) + .refreshRequest(AssumeRoleRequest.builder() + .roleSessionName(pipelineName + " Elasticsearch-Sink " + UUID.randomUUID() + .toString()) + .roleArn(awsStsRole) + .build()) + .build(); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } final HttpRequestInterceptor httpRequestInterceptor = new AwsRequestSigningApacheInterceptor(SERVICE_NAME, aws4Signer, credentialsProvider, awsRegion); restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> { @@ -248,6 +281,8 @@ public static class Builder { private boolean insecure; private boolean awsSigv4; private String awsRegion; + private String awsStsRole; + private String pipelineName; public Builder(final List hosts) { @@ -302,6 +337,16 @@ public Builder withAwsRegion(final String awsRegion) { return this; } + public Builder withAWSStsRole(final String awsStsRole) { + this.awsStsRole = awsStsRole; + return this; + } + + public Builder withPipelineName(final String pipelineName) { + this.pipelineName = pipelineName; + return this; + } + public ConnectionConfiguration build() { return new ConnectionConfiguration(this); } diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java index 70f8c8fba7..a8ddea43d2 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java @@ -21,6 +21,7 @@ public class ConnectionConfigurationTests { private final List TEST_HOSTS = Collections.singletonList("http://localhost:9200"); private final String TEST_USERNAME = "admin"; private final String TEST_PASSWORD = "admin"; + private final String TEST_PIPELINE_NAME = "Test-Pipeline"; private final Integer TEST_CONNECT_TIMEOUT = 5; private final Integer TEST_SOCKET_TIMEOUT = 10; private final String TEST_CERT_PATH = Objects.requireNonNull(getClass().getClassLoader().getResource("test-ca.pem")).getFile(); @@ -28,7 +29,7 @@ public class ConnectionConfigurationTests { @Test public void testReadConnectionConfigurationDefault() { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, false); + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -38,12 +39,13 @@ public void testReadConnectionConfigurationDefault() { assertNull(connectionConfiguration.getCertPath()); assertNull(connectionConfiguration.getConnectTimeout()); assertNull(connectionConfiguration.getSocketTimeout()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientDefault() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, false, null, null, false); + TEST_HOSTS, null, null, null, null, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -54,7 +56,7 @@ public void testCreateClientDefault() throws IOException { @Test public void testReadConnectionConfigurationNoCert() { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals(TEST_HOSTS, connectionConfiguration.getHosts()); @@ -63,12 +65,13 @@ public void testReadConnectionConfigurationNoCert() { assertEquals(TEST_CONNECT_TIMEOUT, connectionConfiguration.getConnectTimeout()); assertEquals(TEST_SOCKET_TIMEOUT, connectionConfiguration.getSocketTimeout()); assertFalse(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientNoCert() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -79,7 +82,7 @@ public void testCreateClientNoCert() throws IOException { @Test public void testCreateClientInsecure() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, true); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -90,7 +93,7 @@ public void testCreateClientInsecure() throws IOException { @Test public void testCreateClientWithCertPath() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, TEST_CERT_PATH, false); + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); final RestHighLevelClient client = connectionConfiguration.createClient(); @@ -101,7 +104,7 @@ public void testCreateClientWithCertPath() throws IOException { @Test public void testCreateClientWithAWSSigV4AndRegion() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, "us-west-2", null, false); + TEST_HOSTS, null, null, null, null, true, "us-west-2", null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-west-2", connectionConfiguration.getAwsRegion()); @@ -111,37 +114,52 @@ public void testCreateClientWithAWSSigV4AndRegion() throws IOException { @Test public void testCreateClientWithAWSSigV4DefaultRegion() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, false); + TEST_HOSTS, null, null, null, null, true, null, null, null, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); - assertTrue(connectionConfiguration.isAwsSigv4());; + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientWithAWSSigV4AndInsecure() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, null, true); + TEST_HOSTS, null, null, null, null, true, null, null, null, true); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } @Test public void testCreateClientWithAWSSigV4AndCertPath() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( - TEST_HOSTS, null, null, null, null, true, null, TEST_CERT_PATH, false); + TEST_HOSTS, null, null, null, null, true, null, null, TEST_CERT_PATH, false); final ConnectionConfiguration connectionConfiguration = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); - assertTrue(connectionConfiguration.isAwsSigv4());; + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); + } + + @Test + public void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { + final PluginSetting pluginSetting = generatePluginSetting( + TEST_HOSTS, null, null, null, null, true, null, "some-iam-role", TEST_CERT_PATH, false); + final ConnectionConfiguration connectionConfiguration = + ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); + assertTrue(connectionConfiguration.isAwsSigv4()); + assertEquals("some-iam-role", connectionConfiguration.getAwsStsRole()); + assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } private PluginSetting generatePluginSetting( final List hosts, final String username, final String password, final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String certPath, final boolean insecure) { + final String awsStsRole, final String certPath, final boolean insecure) { final Map metadata = new HashMap<>(); metadata.put("hosts", hosts); metadata.put("username", username); @@ -152,9 +170,11 @@ private PluginSetting generatePluginSetting( if (awsRegion != null) { metadata.put("aws_region", awsRegion); } + metadata.put("aws_sts_role", awsStsRole); metadata.put("cert", certPath); metadata.put("insecure", insecure); - - return new PluginSetting("elasticsearch", metadata); + final PluginSetting pluginSetting = new PluginSetting("elasticsearch", metadata); + pluginSetting.setPipelineName(TEST_PIPELINE_NAME); + return pluginSetting; } } From c8617932a627be21ec31a3e168a53ddf583883b1 Mon Sep 17 00:00:00 2001 From: David Venable Date: Fri, 23 Jul 2021 14:24:53 -0500 Subject: [PATCH 130/192] AWS CloudMap as a discovery mode for the peer forwarder (#734) Adding support for AWS CloudMap as a discovery mode for the peer forwarder plugin. Addionally, refactored the interactions within the PeerListProviderFactory and DiscoveryMode classes. #730 Signed-off-by: David Venable --- data-prepper-plugins/peer-forwarder/README.md | 51 ++- .../peer-forwarder/build.gradle | 14 +- .../peerforwarder/PeerForwarderConfig.java | 4 +- .../AwsCloudMapPeerListProvider.java | 205 ++++++++++ .../discovery/DiscoveryMode.java | 30 +- .../discovery/DiscoveryUtils.java | 10 + .../discovery/DnsPeerListProvider.java | 19 + .../discovery/PeerListProviderFactory.java | 34 +- .../discovery/StaticPeerListProvider.java | 14 + .../AwsCloudMapPeerListProviderTest.java | 387 ++++++++++++++++++ ...wsCloudMapPeerListProvider_CreateTest.java | 86 ++++ .../discovery/DiscoveryMode.java | 10 - .../DnsPeerListProvider_CreateTest.java | 88 ++++ .../PeerListProviderFactoryTest.java | 134 ++---- .../StaticPeerListProvider_CreateTest.java | 63 +++ .../src/test/resources/log4j2.properties | 10 + 16 files changed, 1016 insertions(+), 143 deletions(-) create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java create mode 100644 data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java delete mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java create mode 100644 data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties diff --git a/data-prepper-plugins/peer-forwarder/README.md b/data-prepper-plugins/peer-forwarder/README.md index 5a4c811c1d..e674110924 100644 --- a/data-prepper-plugins/peer-forwarder/README.md +++ b/data-prepper-plugins/peer-forwarder/README.md @@ -26,7 +26,8 @@ prepper: discovery_mode: "dns" domain_name: "data-prepper-cluster.my-domain.net" ``` -For DNS cluster setup, see [Operating a Cluster with DNS Discovery](#DNS_Discovery). +For DNS cluster setup, see [Operating a Cluster with DNS Discovery](#DNS_Discovery). To +setup for AWS Cloud Map, see [Using AWS CloudMap](#AWS_CloudMap_Discovery). ### Operating a Cluster with DNS Discovery DNS discovery is recommended when scaling out a Data Prepper cluster. The core concept is to configure a DNS provider to return a list of Data Prepper hosts when given a single domain name. @@ -42,13 +43,59 @@ A DNS server (like [dnsmasq](http://www.thekelleys.org.uk/dnsmasq/doc.html)) can #### With Amazon Route 53 Private Hosted Zones [Private hosted zones](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/hosted-zones-private.html) enable Amazon Route 53 to "respond to DNS queries for a domain and its subdomains within one or more VPCs that you create with the Amazon VPC service." Similar to the custom DNS server approach, except that Route 53 maintains the list of Data Prepper hosts. Suffers from the same drawback in that the list must be manually kept up-to-date. +### Using AWS CloudMap + +An alternative to DNS discovery is [AWS CloudMap](https://docs.aws.amazon.com/cloud-map/latest/dg/what-is-cloud-map.html). +CloudMap provides API-based service discovery as well as DNS-based service discovery. + +Peer forwarder can use the API-based service discovery. To support this you must have an existing +namespace configured for API instance discovery. You can create a new one following the instructions +provided by the [CloudMap documentation](https://docs.aws.amazon.com/cloud-map/latest/dg/working-with-namespaces.html). + +Your pipeline configuration needs to include: + +* `awsCloudMapNamespaceName` - Set to your CloudMap Namespace name +* `awsCloudMapServiceName` - Set to the service name within your specified Namespace +* `awsRegion` - The AWS region where your namespace exists. +* `discovery_mode` - Set to `aws_cloud_map` + +Example configuration: + +``` + prepper: + - peer_forwarder: + discovery_mode: aws_cloud_map + awsCloudMapNamespaceName: my-namespace + awsCloudMapServiceName: data-prepper-cluster + awsRegion: us-east-1 +``` + +The DataPrepper must also be running with the necessary permissions. The following +IAM policy shows the necessary permissions. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CloudMapPeerForwarder", + "Effect": "Allow", + "Action": "servicediscovery:DiscoverInstances", + "Resource": "*" + } + ] +} +``` + ## Configuration * `time_out`: timeout in seconds for sending `ExportTraceServiceRequest`. Defaults to 3 seconds. * `span_agg_count`: batch size for number of spans per `ExportTraceServiceRequest`. Defaults to 48. -* `discovery_mode`: peer discovery mode to be used. Allowable values are `static` and `dns`. Defaults to `static` +* `discovery_mode`: peer discovery mode to be used. Allowable values are `static`, `dns`, and `aws_cloud_map`. Defaults to `static` * `static_endpoints`: list containing endpoints of all Data Prepper instances. * `domain_name`: single domain name to query DNS against. Typically used by creating multiple [DNS A Records](https://www.cloudflare.com/learning/dns/dns-records/dns-a-record/) for the same domain. +* `awsCloudMapNamespaceName` - specifies the CloudMap namespace when using AWS CloudMap service discovery +* `awsCloudMapServiceName` - specifies the CloudMap service when using AWS CloudMap service discovery ### SSL The SSL configuration for setting up trust manager for peer forwarding client to connect to other Data Prepper instances. The SSL configuration should be same as the one used for OTel Trace Source. diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 00abed9f4f..eac28736b6 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -15,16 +15,26 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" + implementation(platform('software.amazon.awssdk:bom:2.16.104')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" - implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.15" + implementation 'software.amazon.awssdk:servicediscovery' implementation "commons-io:commons-io:2.10.0" implementation "org.apache.commons:commons-lang3:3.12.0" implementation "commons-validator:commons-validator:1.7" - testImplementation "junit:junit:4.13.2" + testImplementation(platform('org.junit:junit-bom:5.7.2')) + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.junit.vintage:junit-vintage-engine' testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-inline:3.11.2" testImplementation "org.mockito:mockito-core:3.11.2" + testImplementation 'org.mockito:mockito-junit-jupiter:3.11.2' testImplementation "commons-io:commons-io:2.10.0" + testImplementation 'org.awaitility:awaitility:4.0.3' +} + +test { + useJUnitPlatform() } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java index c1f7b21ff6..f7cf753d2d 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java @@ -24,7 +24,9 @@ public class PeerForwarderConfig { private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; private static final String ACM_CERT_ARN = "acmCertificateArn"; - private static final String AWS_REGION = "awsRegion"; + public static final String AWS_REGION = "awsRegion"; + public static final String AWS_CLOUD_MAP_NAMESPACE_NAME = "awsCloudMapNamespaceName"; + public static final String AWS_CLOUD_MAP_SERVICE_NAME = "awsCloudMapServiceName"; private final HashRing hashRing; private final PeerClientPool peerClientPool; diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java new file mode 100644 index 0000000000..e2b89af27e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider.java @@ -0,0 +1,205 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.linecorp.armeria.client.Endpoint; +import com.linecorp.armeria.client.endpoint.DynamicEndpointGroup; +import com.linecorp.armeria.client.retry.Backoff; +import com.linecorp.armeria.common.CommonPools; +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.ScheduledFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.servicediscovery.ServiceDiscoveryAsyncClient; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesRequest; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesResponse; +import software.amazon.awssdk.services.servicediscovery.model.HttpInstanceSummary; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Implementation of {@link PeerListProvider} which uses AWS CloudMap's + * service discovery capability to discover peers. + */ +class AwsCloudMapPeerListProvider implements PeerListProvider, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(AwsCloudMapPeerListProvider.class); + private static final int ONE_SECOND = 1000; + private static final int TWENTY_SECONDS = 32000; + private static final double TWENTY_PERCENT = 0.2; + private static final String INSTANCE_IP4_ATTRIBUTE_NAME = "AWS_INSTANCE_IPV4"; + + private final ServiceDiscoveryAsyncClient awsServiceDiscovery; + private final String namespaceName; + private final String serviceName; + private final AwsCloudMapDynamicEndpointGroup endpointGroup; + private final int timeToRefreshSeconds; + private final Backoff backoff; + private final EventLoop eventLoop; + private final String domainName; + + AwsCloudMapPeerListProvider( + final ServiceDiscoveryAsyncClient awsServiceDiscovery, + final String namespaceName, + final String serviceName, + final int timeToRefreshSeconds, + final Backoff backoff, + final PluginMetrics pluginMetrics) { + this.awsServiceDiscovery = Objects.requireNonNull(awsServiceDiscovery); + this.namespaceName = Objects.requireNonNull(namespaceName); + this.serviceName = Objects.requireNonNull(serviceName); + this.timeToRefreshSeconds = timeToRefreshSeconds; + this.backoff = Objects.requireNonNull(backoff); + + if (timeToRefreshSeconds < 1) + throw new IllegalArgumentException("timeToRefreshSeconds must be positive. Actual: " + timeToRefreshSeconds); + + eventLoop = CommonPools.workerGroup().next(); + LOG.info("Using AWS CloudMap for Peer Forwarding. namespace='{}', serviceName='{}'", + namespaceName, serviceName); + + endpointGroup = new AwsCloudMapDynamicEndpointGroup(); + + domainName = serviceName + "." + namespaceName; + + pluginMetrics.gauge(PEER_ENDPOINTS, endpointGroup, group -> group.endpoints().size()); + } + + static AwsCloudMapPeerListProvider createPeerListProvider(final PluginSetting pluginSetting, final PluginMetrics pluginMetrics) { + final String awsRegion = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_REGION); + final String namespace = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME); + final String serviceName = getRequiredSettingString(pluginSetting, PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME); + + final Backoff standardBackoff = Backoff.exponential(ONE_SECOND, TWENTY_SECONDS).withJitter(TWENTY_PERCENT); + final int timeToRefreshSeconds = 20; + + final ServiceDiscoveryAsyncClient serviceDiscoveryAsyncClient = ServiceDiscoveryAsyncClient + .builder() + .region(Region.of(awsRegion)) + .credentialsProvider(DefaultCredentialsProvider.create()) + .build(); + + return new AwsCloudMapPeerListProvider( + serviceDiscoveryAsyncClient, + namespace, + serviceName, + timeToRefreshSeconds, + standardBackoff, + pluginMetrics); + } + + private static String getRequiredSettingString(final PluginSetting pluginSetting, final String propertyName) { + final String propertyValue = pluginSetting.getStringOrDefault(propertyName, null); + return Objects.requireNonNull(propertyValue, String.format("Missing '%s' configuration value", propertyName)); + } + + @Override + public List getPeerList() { + return endpointGroup.endpoints() + .stream() + .map(Endpoint::ipAddr) + .collect(Collectors.toList()); + } + + @Override + public void addListener(final Consumer> listener) { + endpointGroup.addListener(listener); + } + + @Override + public void removeListener(final Consumer listener) { + endpointGroup.removeListener(listener); + } + + @Override + public void close() { + endpointGroup.close(); + } + + /** + * The {@link DynamicEndpointGroup} class serves as a useful base class for updating + * endpoints by supporting the endpoint observer pattern for us. We just need to + * periodically check for updates and call {@link DynamicEndpointGroup#setEndpoints(Iterable)}. + */ + private class AwsCloudMapDynamicEndpointGroup extends DynamicEndpointGroup { + + private int failedAttemptCount = 0; + private volatile ScheduledFuture scheduledDiscovery; + + private AwsCloudMapDynamicEndpointGroup() { + eventLoop.execute(this::discoverInstances); + } + + private void discoverInstances() { + if (isClosing()) { + return; + } + + final DiscoverInstancesRequest discoverInstancesRequest = DiscoverInstancesRequest + .builder() + .namespaceName(namespaceName) + .serviceName(serviceName) + .build(); + + LOG.info("Discovering instances."); + + awsServiceDiscovery.discoverInstances(discoverInstancesRequest).whenComplete( + (discoverInstancesResponse, throwable) -> { + if (discoverInstancesResponse != null) { + try { + failedAttemptCount = 0; + updateEndpointsWithDiscoveredInstances(discoverInstancesResponse); + } catch (final Throwable ex) { + LOG.warn("Failed to update endpoints.", ex); + } finally { + scheduledDiscovery = eventLoop.schedule(this::discoverInstances, + timeToRefreshSeconds, TimeUnit.SECONDS); + } + } + + if (throwable != null) { + failedAttemptCount++; + final long delayMillis = backoff.nextDelayMillis(failedAttemptCount); + LOG.error("Failed to discover instances for: namespace='{}', serviceName='{}'. Will retry in {} ms.", + namespaceName, serviceName, delayMillis, throwable); + + scheduledDiscovery = eventLoop.schedule(this::discoverInstances, + delayMillis, TimeUnit.MILLISECONDS); + } + }); + } + + private void updateEndpointsWithDiscoveredInstances(final DiscoverInstancesResponse discoverInstancesResponse) { + final List instances = discoverInstancesResponse.instances(); + + LOG.info("Discovered {} instances.", instances.size()); + + final List endpoints = instances + .stream() + .map(HttpInstanceSummary::attributes) + .map(attributes -> attributes.get(INSTANCE_IP4_ATTRIBUTE_NAME)) + .map(ip -> Endpoint.of(domainName).withIpAddr(ip)) + .collect(Collectors.toList()); + + setEndpoints(endpoints); + } + + @Override + protected void doCloseAsync(final CompletableFuture future) { + final ScheduledFuture scheduledDiscovery = this.scheduledDiscovery; + if (scheduledDiscovery != null) { + scheduledDiscovery.cancel(true); + } + + future.complete(null); + } + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java index 1f64c8c5d2..697cf9fa5f 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java @@ -1,6 +1,32 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; + +import java.util.Objects; +import java.util.function.BiFunction; + public enum DiscoveryMode { - STATIC, - DNS, + STATIC(StaticPeerListProvider::createPeerListProvider), + DNS(DnsPeerListProvider::createPeerListProvider), + AWS_CLOUD_MAP(AwsCloudMapPeerListProvider::createPeerListProvider); + + private final BiFunction creationFunction; + + DiscoveryMode(final BiFunction creationFunction) { + Objects.requireNonNull(creationFunction); + + this.creationFunction = creationFunction; + } + + /** + * Creates a new {@link PeerListProvider} for the this discovery mode. + * + * @param pluginSetting The plugin settings + * @param pluginMetrics The plugin metrics + * @return The new {@link PeerListProvider} for this discovery mode + */ + PeerListProvider create(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + return creationFunction.apply(pluginSetting, pluginMetrics); + } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java new file mode 100644 index 0000000000..c6c9c1d0dc --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java @@ -0,0 +1,10 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import org.apache.commons.validator.routines.DomainValidator; +import org.apache.commons.validator.routines.InetAddressValidator; + +class DiscoveryUtils { + static boolean validateEndpoint(final String endpoint) { + return DomainValidator.getInstance(true).isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); + } +} diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java index ff4fc91922..52f654bcad 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider.java @@ -1,6 +1,9 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.google.common.base.Preconditions; import com.linecorp.armeria.client.Endpoint; import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; import org.slf4j.Logger; @@ -12,9 +15,13 @@ import java.util.function.Consumer; import java.util.stream.Collectors; +import static com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery.DiscoveryUtils.validateEndpoint; + public class DnsPeerListProvider implements PeerListProvider { private static final Logger LOG = LoggerFactory.getLogger(DnsPeerListProvider.class); + private static final int MIN_TTL = 10; + private static final int MAX_TTL = 20; private final DnsAddressEndpointGroup endpointGroup; @@ -33,6 +40,18 @@ public DnsPeerListProvider(final DnsAddressEndpointGroup endpointGroup, final Pl pluginMetrics.gauge(PEER_ENDPOINTS, endpointGroup, group -> group.endpoints().size()); } + static DnsPeerListProvider createPeerListProvider(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + final String domainName = pluginSetting.getStringOrDefault(PeerForwarderConfig.DOMAIN_NAME, null); + Objects.requireNonNull(domainName, String.format("Missing '%s' configuration value",PeerForwarderConfig. DOMAIN_NAME)); + Preconditions.checkState(validateEndpoint(domainName), "Invalid domain name: %s", domainName); + + final DnsAddressEndpointGroup endpointGroup = DnsAddressEndpointGroup.builder(domainName) + .ttl(MIN_TTL, MAX_TTL) + .build(); + + return new DnsPeerListProvider(endpointGroup, pluginMetrics); + } + @Override public List getPeerList() { return endpointGroup.endpoints() diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java index ca3c1808ef..0b9557cd4a 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactory.java @@ -3,19 +3,10 @@ import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; -import com.google.common.base.Preconditions; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; -import org.apache.commons.validator.routines.DomainValidator; -import org.apache.commons.validator.routines.InetAddressValidator; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; public class PeerListProviderFactory { - // TODO: Make these configurable? - private static final int MIN_TTL = 10; - private static final int MAX_TTL = 20; public PeerListProvider createProvider(final PluginSetting pluginSetting) { Objects.requireNonNull(pluginSetting); @@ -27,30 +18,7 @@ public PeerListProvider createProvider(final PluginSetting pluginSetting) { final PluginMetrics pluginMetrics = PluginMetrics.fromPluginSetting(pluginSetting); - switch (discoveryMode) { - case DNS: - final String domainName = pluginSetting.getStringOrDefault(PeerForwarderConfig.DOMAIN_NAME, null); - Objects.requireNonNull(domainName, String.format("Missing '%s' configuration value",PeerForwarderConfig. DOMAIN_NAME)); - Preconditions.checkState(validateEndpoint(domainName), "Invalid domain name: %s", domainName); - - final DnsAddressEndpointGroup endpointGroup = DnsAddressEndpointGroup.builder(domainName) - .ttl(MIN_TTL, MAX_TTL) - .build(); - - return new DnsPeerListProvider(endpointGroup, pluginMetrics); - case STATIC: - final List endpoints = (List) pluginSetting.getAttributeOrDefault(PeerForwarderConfig.STATIC_ENDPOINTS, null); - Objects.requireNonNull(endpoints, String.format("Missing '%s' configuration value", PeerForwarderConfig.STATIC_ENDPOINTS)); - final List invalidEndpoints = endpoints.stream().filter(endpoint -> !this.validateEndpoint(endpoint)).collect(Collectors.toList()); - Preconditions.checkState(invalidEndpoints.isEmpty(), "Including invalid endpoints: %s", invalidEndpoints); - - return new StaticPeerListProvider(endpoints, pluginMetrics); - default: - throw new UnsupportedOperationException(String.format("Unsupported discovery mode: %s", discoveryMode)); - } + return discoveryMode.create(pluginSetting, pluginMetrics); } - private boolean validateEndpoint(final String endpoint) { - return DomainValidator.getInstance(true).isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); - } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java index e85ad5cfc8..94ecfc4197 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider.java @@ -1,13 +1,18 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.google.common.base.Preconditions; import com.linecorp.armeria.client.Endpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Consumer; +import java.util.stream.Collectors; public class StaticPeerListProvider implements PeerListProvider { @@ -29,6 +34,15 @@ public StaticPeerListProvider(final List dataPrepperEndpoints, final Plu pluginMetrics.gauge(PEER_ENDPOINTS, endpoints, List::size); } + static StaticPeerListProvider createPeerListProvider(PluginSetting pluginSetting, PluginMetrics pluginMetrics) { + final List endpoints = (List) pluginSetting.getAttributeOrDefault(PeerForwarderConfig.STATIC_ENDPOINTS, null); + Objects.requireNonNull(endpoints, String.format("Missing '%s' configuration value", PeerForwarderConfig.STATIC_ENDPOINTS)); + final List invalidEndpoints = endpoints.stream().filter(endpoint -> !DiscoveryUtils.validateEndpoint(endpoint)).collect(Collectors.toList()); + Preconditions.checkState(invalidEndpoints.isEmpty(), "Including invalid endpoints: %s", invalidEndpoints); + + return new StaticPeerListProvider(endpoints, pluginMetrics); + } + @Override public List getPeerList() { return endpoints; diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java new file mode 100644 index 0000000000..7ba3e16b94 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProviderTest.java @@ -0,0 +1,387 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.linecorp.armeria.client.Endpoint; +import com.linecorp.armeria.client.retry.Backoff; +import org.apache.commons.lang3.RandomStringUtils; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.InOrder; +import software.amazon.awssdk.services.servicediscovery.ServiceDiscoveryAsyncClient; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesRequest; +import software.amazon.awssdk.services.servicediscovery.model.DiscoverInstancesResponse; +import software.amazon.awssdk.services.servicediscovery.model.HttpInstanceSummary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; + +class AwsCloudMapPeerListProviderTest { + + public static final int WAIT_TIME_MULTIPLIER_MILLIS = 1200; + private ServiceDiscoveryAsyncClient awsServiceDiscovery; + private String namespaceName; + private String serviceName; + private int timeToRefreshSeconds; + private Backoff backoff; + private PluginMetrics pluginMetrics; + private List objectsToClose; + + @BeforeEach + void setUp() { + awsServiceDiscovery = mock(ServiceDiscoveryAsyncClient.class); + namespaceName = RandomStringUtils.randomAlphabetic(10); + serviceName = RandomStringUtils.randomAlphabetic(10); + + timeToRefreshSeconds = 1; + backoff = mock(Backoff.class); + pluginMetrics = mock(PluginMetrics.class); + + objectsToClose = new ArrayList<>(); + } + + @AfterEach + void tearDown() { + objectsToClose.forEach(AwsCloudMapPeerListProvider::close); + } + + private AwsCloudMapPeerListProvider createObjectUnderTest() { + final AwsCloudMapPeerListProvider objectUnderTest = new AwsCloudMapPeerListProvider(awsServiceDiscovery, namespaceName, serviceName, timeToRefreshSeconds, backoff, pluginMetrics); + objectsToClose.add(objectUnderTest); + return objectUnderTest; + } + + @Test + void constructor_throws_with_null_AWSServiceDiscovery() { + awsServiceDiscovery = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_Namespace() { + namespaceName = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_ServiceName() { + serviceName = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_throws_with_null_Backoff() { + backoff = null; + + assertThrows(NullPointerException.class, + this::createObjectUnderTest); + } + + @ParameterizedTest + @ValueSource(ints = {Integer.MIN_VALUE, -10, -1, 0}) + void constructor_throws_with_non_positive_timeToRefreshSeconds(final int badTimeToRefresh) { + timeToRefreshSeconds = badTimeToRefresh; + + assertThrows(IllegalArgumentException.class, + this::createObjectUnderTest); + } + + @Test + void constructor_should_DiscoverInstances_with_correct_request() { + createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final ArgumentCaptor requestArgumentCaptor = + ArgumentCaptor.forClass(DiscoverInstancesRequest.class); + + then(awsServiceDiscovery) + .should() + .discoverInstances(requestArgumentCaptor.capture()); + + final DiscoverInstancesRequest actualRequest = requestArgumentCaptor.getValue(); + + assertThat(actualRequest.namespaceName(), equalTo(namespaceName)); + assertThat(actualRequest.serviceName(), equalTo(serviceName)); + assertThat(actualRequest.healthStatusAsString(), nullValue()); + } + + @Test + void getPeerList_is_empty_before_populated() { + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List peerList = objectUnderTest.getPeerList(); + + assertThat(peerList, notNullValue()); + assertThat(peerList.size(), equalTo(0)); + } + + @Nested + class WithDiscoverInstances { + + private DiscoverInstancesResponse discoverInstancesResponse; + + @BeforeEach + void setUp() { + discoverInstancesResponse = mock(DiscoverInstancesResponse.class); + + final CompletableFuture discoverFuture = + CompletableFuture.completedFuture(discoverInstancesResponse); + + given(awsServiceDiscovery.discoverInstances(any(DiscoverInstancesRequest.class))) + .willReturn(discoverFuture); + } + + @Test + void getPeerList_returns_empty_when_DiscoverInstances_has_no_instances() { + given(discoverInstancesResponse.instances()).willReturn(Collections.emptyList()); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List peerList = objectUnderTest.getPeerList(); + assertThat(peerList, notNullValue()); + assertThat(peerList.size(), equalTo(0)); + } + + @Test + void getPeerList_returns_list_as_found() { + + final List knownIpPeers = IntStream.range(0, 3) + .mapToObj(i -> generateRandomIp()) + .collect(Collectors.toList()); + + final List instances = knownIpPeers + .stream() + .map(ip -> { + final HttpInstanceSummary instanceSummary = mock(HttpInstanceSummary.class); + given(instanceSummary.attributes()).willReturn( + Collections.singletonMap("AWS_INSTANCE_IPV4", ip)); + return instanceSummary; + }) + .collect(Collectors.toList()); + + given(discoverInstancesResponse.instances()).willReturn(instances); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilPeerListPopulated(objectUnderTest); + + final List actualPeers = objectUnderTest.getPeerList(); + assertThat(actualPeers, notNullValue()); + assertThat(actualPeers.size(), equalTo(instances.size())); + + assertThat(new HashSet<>(actualPeers), equalTo(new HashSet<>(knownIpPeers))); + } + + @Test + void constructor_continues_to_discover_instances() { + + createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeast(2); + + final ArgumentCaptor requestArgumentCaptor = + ArgumentCaptor.forClass(DiscoverInstancesRequest.class); + + then(awsServiceDiscovery) + .should(atLeast(2)) + .discoverInstances(requestArgumentCaptor.capture()); + + for (DiscoverInstancesRequest actualRequest : requestArgumentCaptor.getAllValues()) { + assertThat(actualRequest.namespaceName(), equalTo(namespaceName)); + assertThat(actualRequest.serviceName(), equalTo(serviceName)); + assertThat(actualRequest.healthStatusAsString(), nullValue()); + } + } + } + + @Nested + class WithSeveralFailedAttempts { + + private List knownIpPeers; + + @BeforeEach + void setUp() { + final DiscoverInstancesResponse discoverInstancesResponse = mock(DiscoverInstancesResponse.class); + + knownIpPeers = IntStream.range(0, 3) + .mapToObj(i -> generateRandomIp()) + .collect(Collectors.toList()); + + final List instances = knownIpPeers + .stream() + .map(ip -> { + final HttpInstanceSummary instanceSummary = mock(HttpInstanceSummary.class); + given(instanceSummary.attributes()).willReturn( + Collections.singletonMap("AWS_INSTANCE_IPV4", ip)); + return instanceSummary; + }) + .collect(Collectors.toList()); + + given(discoverInstancesResponse.instances()).willReturn(instances); + + final CompletableFuture failedFuture1 = new CompletableFuture<>(); + failedFuture1.completeExceptionally(mock(Throwable.class)); + final CompletableFuture failedFuture2 = new CompletableFuture<>(); + failedFuture2.completeExceptionally(mock(Throwable.class)); + final CompletableFuture successFuture = CompletableFuture.completedFuture(discoverInstancesResponse); + + given(awsServiceDiscovery.discoverInstances(any(DiscoverInstancesRequest.class))) + .willReturn(failedFuture1) + .willReturn(failedFuture2) + .willReturn(successFuture); + + given(backoff.nextDelayMillis(anyInt())) + .willReturn(100L); + + } + + @Test + void getPeerList_returns_value_after_several_failed_attempts() { + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + final List expectedEmpty = objectUnderTest.getPeerList(); + + assertThat(expectedEmpty, notNullValue()); + assertThat(expectedEmpty.size(), equalTo(0)); + + waitUntilPeerListPopulated(objectUnderTest); + + final List expectedPopulated = objectUnderTest.getPeerList(); + + assertThat(expectedPopulated, notNullValue()); + assertThat(expectedPopulated.size(), equalTo(knownIpPeers.size())); + + assertThat(new HashSet<>(expectedPopulated), equalTo(new HashSet<>(knownIpPeers))); + + final InOrder inOrder = inOrder(backoff); + then(backoff) + .should(inOrder) + .nextDelayMillis(1); + then(backoff) + .should(inOrder) + .nextDelayMillis(2); + then(backoff) + .shouldHaveNoMoreInteractions(); + } + + @Test + void listener_gets_list_after_several_failed_attempts() { + + final List listenerEndpoints = new ArrayList<>(); + + final AwsCloudMapPeerListProvider objectUnderTest = createObjectUnderTest(); + + objectUnderTest.addListener(listenerEndpoints::addAll); + + waitUntilDiscoverInstancesCalledAtLeastOnce(); + + assertThat(listenerEndpoints.size(), equalTo(0)); + + waitUntilPeerListPopulated(objectUnderTest); + + assertThat(listenerEndpoints.size(), equalTo(knownIpPeers.size())); + + final Set observedIps = listenerEndpoints.stream() + .map(Endpoint::ipAddr) + .collect(Collectors.toSet()); + + assertThat(observedIps, equalTo(new HashSet<>(knownIpPeers))); + + final InOrder inOrder = inOrder(backoff); + then(backoff) + .should(inOrder) + .nextDelayMillis(1); + then(backoff) + .should(inOrder) + .nextDelayMillis(2); + then(backoff) + .shouldHaveNoMoreInteractions(); + } + } + + private void waitUntilDiscoverInstancesCalledAtLeastOnce() { + waitUntilDiscoverInstancesCalledAtLeast(1); + } + + /** + * Waits for DiscoverInstances to be called at least a specified number + * of times. This method intentionally does not inspect the request. + * + * @param timesCalled The number of times to wait for it to be called. + */ + private void waitUntilDiscoverInstancesCalledAtLeast(final int timesCalled) { + final long waitTimeMillis = (long) timesCalled * WAIT_TIME_MULTIPLIER_MILLIS; + Awaitility.waitAtMost(waitTimeMillis, TimeUnit.MILLISECONDS) + .pollDelay(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> then(awsServiceDiscovery) + .should(atLeast(timesCalled)) + .discoverInstances(ArgumentMatchers.any(DiscoverInstancesRequest.class))); + } + + /** + * Waits until the give {@link AwsCloudMapPeerListProvider} has a peer list + * with a value greater than 0. + * + * @param objectUnderTest The object to wait for. + */ + private void waitUntilPeerListPopulated(final AwsCloudMapPeerListProvider objectUnderTest) { + Awaitility.waitAtMost(2, TimeUnit.SECONDS) + .pollDelay(100, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + final List actualPeers = objectUnderTest.getPeerList(); + assertThat(actualPeers, notNullValue()); + assertThat(actualPeers.size(), greaterThan(0)); + }); + } + + private static String generateRandomIp() { + final Random random = new Random(); + + return IntStream.range(0, 4) + .map(i -> random.nextInt(255)) + .mapToObj(Integer::toString) + .collect(Collectors.joining(".")); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..cbe2ed27e1 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/AwsCloudMapPeerListProvider_CreateTest.java @@ -0,0 +1,86 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.regions.Region; + +import java.util.HashMap; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + +class AwsCloudMapPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String ENDPOINT = "ENDPOINT"; + private static final String PIPELINE_NAME = "pipelineName"; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + @BeforeEach + void setUp() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; + + pluginMetrics = mock(PluginMetrics.class); + + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME, UUID.randomUUID().toString()); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME, UUID.randomUUID().toString()); + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_REGION, "us-east-1"); + + } + + @Test + void createPeerListProvider_with_valid_configurations() { + final PeerListProvider result = AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(AwsCloudMapPeerListProvider.class)); + } + + @ParameterizedTest + @ValueSource(strings = { + PeerForwarderConfig.AWS_CLOUD_MAP_NAMESPACE_NAME, + PeerForwarderConfig.AWS_CLOUD_MAP_SERVICE_NAME, + PeerForwarderConfig.AWS_REGION + }) + void createPeerListProvider_with_missing_required_property(final String propertyToRemove) { + pluginSetting.getSettings().remove(propertyToRemove); + + assertThrows(NullPointerException.class, + () -> AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + + } + + @ParameterizedTest + @ArgumentsSource(AllRegionsArgumentProvider.class) + void createPeerListProvider_with_all_current_regions(final Region region) { + pluginSetting.getSettings().put(PeerForwarderConfig.AWS_REGION, region.toString()); + + final PeerListProvider result = AwsCloudMapPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(AwsCloudMapPeerListProvider.class)); + } + + static class AllRegionsArgumentProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Region.regions().stream().map(Arguments::of); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java deleted file mode 100644 index 7b21f13db2..0000000000 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryMode.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; - -/** - * Test enum defined to provide an UNSUPPORTED value for use in unit testing. - */ -public enum DiscoveryMode { - STATIC, - DNS, - UNSUPPORTED -} diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..309d61df84 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java @@ -0,0 +1,88 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; +import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroupBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.concurrent.CompletableFuture; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DnsPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String ENDPOINT = "ENDPOINT"; + private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; + private static final String PIPELINE_NAME = "pipelineName"; + + @Mock + private DnsAddressEndpointGroupBuilder dnsAddressEndpointGroupBuilder; + @Mock + private DnsAddressEndpointGroup dnsAddressEndpointGroup; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + private CompletableFuture completableFuture; + + @BeforeEach + void setup() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; + completableFuture = CompletableFuture.completedFuture(null); + + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); + pluginMetrics = mock(PluginMetrics.class); + } + + @Test + void testCreateProviderDnsInstance() { + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + + when(dnsAddressEndpointGroupBuilder.build()).thenReturn(dnsAddressEndpointGroup); + when(dnsAddressEndpointGroupBuilder.ttl(anyInt(), anyInt())).thenReturn(dnsAddressEndpointGroupBuilder); + when(dnsAddressEndpointGroup.whenReady()).thenReturn(completableFuture); + + try (MockedStatic armeriaMock = Mockito.mockStatic(DnsAddressEndpointGroup.class)) { + armeriaMock.when(() -> DnsAddressEndpointGroup.builder(anyString())).thenReturn(dnsAddressEndpointGroupBuilder); + + PeerListProvider result = DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(DnsPeerListProvider.class)); + } + } + + @Test + void testCreateProviderDnsInstanceWithNoHostname() { + assertThrows(NullPointerException.class, + () -> DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + + } + + @Test + void testCreateProviderDnsInstanceWithInvalidDomainName() { + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, INVALID_ENDPOINT); + + assertThrows(IllegalStateException.class, + () -> DnsPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java index 3fea3febd1..48101e6583 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/PeerListProviderFactoryTest.java @@ -1,129 +1,77 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; +import com.amazon.dataprepper.metrics.PluginMetrics; import com.amazon.dataprepper.model.configuration.PluginSetting; import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroup; -import com.linecorp.armeria.client.endpoint.dns.DnsAddressEndpointGroupBuilder; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; -import java.util.concurrent.CompletableFuture; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) -public class PeerListProviderFactoryTest { +@ExtendWith(MockitoExtension.class) +class PeerListProviderFactoryTest { private static final String PLUGIN_NAME = "PLUGIN_NAME"; - private static final String ENDPOINT = "ENDPOINT"; - private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; private static final String PIPELINE_NAME = "pipelineName"; - @Mock - private DnsAddressEndpointGroupBuilder dnsAddressEndpointGroupBuilder; - @Mock - private DnsAddressEndpointGroup dnsAddressEndpointGroup; - - private CompletableFuture completableFuture; - private PluginSetting pluginSetting; private PeerListProviderFactory factory; - @Before - public void setup() { + @BeforeEach + void setup() { factory = new PeerListProviderFactory(); - pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()){{ setPipelineName(PIPELINE_NAME); }}; - completableFuture = CompletableFuture.completedFuture(null); - } - - @Test(expected = NullPointerException.class) - public void testUnsupportedNoDiscoveryMode() { - factory.createProvider(pluginSetting); - } - - @Test(expected = IllegalArgumentException.class) - public void testUndefinedDiscoveryModeEnum() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, "GARBAGE"); - - factory.createProvider(pluginSetting); - } - - @Test(expected = UnsupportedOperationException.class) - public void testUnsupportedDiscoveryModeEnum() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.UNSUPPORTED.toString()); - - factory.createProvider(pluginSetting); + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()) {{ + setPipelineName(PIPELINE_NAME); + }}; } - @Test(expected = NullPointerException.class) - public void testCreateProviderStaticInstanceNoEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - - factory.createProvider(pluginSetting); + @Test + void createProvider_throws_when_no_DiscoveryMode_is_provided() { + assertThrows(NullPointerException.class, + () -> factory.createProvider(pluginSetting)); } @Test - public void testCreateProviderStaticInstanceWithEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Collections.singletonList(ENDPOINT)); - - PeerListProvider result = factory.createProvider(pluginSetting); + void createProvider_throws_for_undefined_DiscoveryMode() { + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, "GARBAGE"); - assertTrue(result instanceof StaticPeerListProvider); - assertEquals(1, result.getPeerList().size()); - assertTrue(result.getPeerList().contains(ENDPOINT)); + assertThrows(IllegalArgumentException.class, + () -> factory.createProvider(pluginSetting)); } - @Test(expected = IllegalStateException.class) - public void testCreateProviderStaticInstanceWithInvalidEndpoints() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.STATIC.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Arrays.asList(ENDPOINT, INVALID_ENDPOINT)); + @ParameterizedTest + @EnumSource(DiscoveryMode.class) + void createProvider_returns_correct_provider_for_all_DiscoveryModes(final DiscoveryMode discoveryMode) { + final String discoveryModeString = discoveryMode.toString(); + pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, discoveryModeString.toLowerCase()); - factory.createProvider(pluginSetting); - } - - @Test - public void testCreateProviderDnsInstance() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + final PeerListProvider expectedProvider = mock(PeerListProvider.class); + final PeerListProvider actualProvider; - when(dnsAddressEndpointGroupBuilder.build()).thenReturn(dnsAddressEndpointGroup); - when(dnsAddressEndpointGroupBuilder.ttl(anyInt(), anyInt())).thenReturn(dnsAddressEndpointGroupBuilder); - when(dnsAddressEndpointGroup.whenReady()).thenReturn(completableFuture); + try (final MockedStatic enumMock = Mockito.mockStatic(DiscoveryMode.class)) { + final DiscoveryMode mockedModeEnum = mock(DiscoveryMode.class); + enumMock.when(() -> DiscoveryMode.valueOf(discoveryModeString)).thenReturn(mockedModeEnum); - try (MockedStatic armeriaMock = Mockito.mockStatic(DnsAddressEndpointGroup.class)) { - armeriaMock.when(() -> DnsAddressEndpointGroup.builder(anyString())).thenReturn(dnsAddressEndpointGroupBuilder); + when(mockedModeEnum.create(eq(pluginSetting), any(PluginMetrics.class))).thenReturn(expectedProvider); - PeerListProvider result = factory.createProvider(pluginSetting); - - assertTrue(result instanceof DnsPeerListProvider); + actualProvider = factory.createProvider(pluginSetting); } - } - - @Test(expected = NullPointerException.class) - public void testCreateProviderDnsInstanceWithNoHostname() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - factory.createProvider(pluginSetting); + assertThat(actualProvider, sameInstance(expectedProvider)); } - @Test(expected = IllegalStateException.class) - public void testCreateProviderDnsInstanceWithInvalidDomainName() { - pluginSetting.getSettings().put(PeerForwarderConfig.DISCOVERY_MODE, DiscoveryMode.DNS.toString()); - pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, INVALID_ENDPOINT); - - factory.createProvider(pluginSetting); - } } diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java new file mode 100644 index 0000000000..b4b5f0b4e6 --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/StaticPeerListProvider_CreateTest.java @@ -0,0 +1,63 @@ +package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; + +import com.amazon.dataprepper.metrics.PluginMetrics; +import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.plugins.prepper.peerforwarder.PeerForwarderConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +class StaticPeerListProvider_CreateTest { + + private static final String PLUGIN_NAME = "PLUGIN_NAME"; + private static final String ENDPOINT = "ENDPOINT"; + private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; + private static final String PIPELINE_NAME = "pipelineName"; + + private PluginSetting pluginSetting; + private PluginMetrics pluginMetrics; + + @BeforeEach + void setup() { + pluginSetting = new PluginSetting(PLUGIN_NAME, new HashMap<>()){{ setPipelineName(PIPELINE_NAME); }}; + + pluginMetrics = mock(PluginMetrics.class); + } + + @Test + void testCreateProviderStaticInstanceNoEndpoints() { + + assertThrows(NullPointerException.class, + () -> StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + + @Test + void testCreateProviderStaticInstanceWithEndpoints() { + pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Collections.singletonList(ENDPOINT)); + + PeerListProvider result = StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics); + + assertThat(result, instanceOf(StaticPeerListProvider.class)); + assertEquals(1, result.getPeerList().size()); + assertTrue(result.getPeerList().contains(ENDPOINT)); + } + + @Test + void testCreateProviderStaticInstanceWithInvalidEndpoints() { + pluginSetting.getSettings().put(PeerForwarderConfig.STATIC_ENDPOINTS, Arrays.asList(ENDPOINT, INVALID_ENDPOINT)); + + assertThrows(IllegalStateException.class, + () -> StaticPeerListProvider.createPeerListProvider(pluginSetting, pluginMetrics)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties b/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..9462a0739e --- /dev/null +++ b/data-prepper-plugins/peer-forwarder/src/test/resources/log4j2.properties @@ -0,0 +1,10 @@ +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n + +rootLogger.level = warn +rootLogger.appenderRef.stdout.ref = STDOUT + +logger.core.name = com.amazon.dataprepper +logger.core.level = debug From 043e8f665df7d4710b9bcd49b07b25c2859bd182 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Fri, 16 Jul 2021 21:13:17 +0000 Subject: [PATCH 131/192] Minor update to ACMCertificateProvider. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../acm/ACMCertificateProvider.java | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java index 935bab4229..06de8bc8d6 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/certificate/acm/ACMCertificateProvider.java @@ -3,7 +3,12 @@ import com.amazon.dataprepper.plugins.certificate.CertificateProvider; import com.amazon.dataprepper.plugins.certificate.model.Certificate; import com.amazonaws.services.certificatemanager.AWSCertificateManager; -import com.amazonaws.services.certificatemanager.model.*; +import com.amazonaws.services.certificatemanager.model.ExportCertificateRequest; +import com.amazonaws.services.certificatemanager.model.ExportCertificateResult; +import com.amazonaws.services.certificatemanager.model.InvalidArnException; +import com.amazonaws.services.certificatemanager.model.RequestInProgressException; +import com.amazonaws.services.certificatemanager.model.ResourceNotFoundException; +import org.apache.commons.lang3.RandomStringUtils; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openssl.PEMEncryptedKeyPair; @@ -24,19 +29,24 @@ import java.io.StringWriter; import java.nio.ByteBuffer; import java.security.PrivateKey; +import java.security.SecureRandom; import java.security.Security; import java.util.Objects; import java.util.Optional; -import java.util.UUID; +import java.util.Random; public class ACMCertificateProvider implements CertificateProvider { private static final Logger LOG = LoggerFactory.getLogger(ACMCertificateProvider.class); private static final long SLEEP_INTERVAL = 10000L; + private static final int PASSPHRASE_CHAR_COUNT = 36; private static final String BOUNCY_CASTLE_PROVIDER = "BC"; + private static final Random SECURE_RANDOM = new SecureRandom(); + private final AWSCertificateManager awsCertificateManager; private final String acmArn; private final long totalTimeout; private final String passphrase; + public ACMCertificateProvider(final AWSCertificateManager awsCertificateManager, final String acmArn, final long totalTimeout, @@ -54,9 +64,9 @@ public Certificate getCertificate() { long timeSlept = 0L; // The private key from ACM is encrypted. Passphrase is the privateKey password that will be used to decrypt the - // private key. If it's not provided, UUID is used generate a temporary password. The configured passphrase can + // private key. If it's not provided, generate a random password. The configured passphrase can // be used to decrypt the private key manually using openssl commands for any inspection or debugging. - final String pkPassphrase = Optional.ofNullable(passphrase).orElse(UUID.randomUUID().toString()); + final String pkPassphrase = Optional.ofNullable(passphrase).orElse(generatePassphrase(PASSPHRASE_CHAR_COUNT)); while (exportCertificateResult == null && timeSlept < totalTimeout) { try { final ExportCertificateRequest exportCertificateRequest = new ExportCertificateRequest() @@ -76,14 +86,28 @@ public Certificate getCertificate() { } timeSlept += SLEEP_INTERVAL; } - if(exportCertificateResult != null) { + if (exportCertificateResult != null) { final String decryptedPrivateKey = getDecryptedPrivateKey(exportCertificateResult.getPrivateKey(), pkPassphrase); return new Certificate(exportCertificateResult.getCertificate(), decryptedPrivateKey); } else { - throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); + throw new IllegalStateException(String.format("Exception retrieving certificate results. Time spent retrieving certificate is" + + " %d ms and total time out set is %d ms.", timeSlept, totalTimeout)); } } + private String generatePassphrase(final int characterCount) { + String passphrase = RandomStringUtils.random( + characterCount, + 0, + 0, + true, + true, + null, + SECURE_RANDOM); + + return passphrase; + } + private String getDecryptedPrivateKey(final String encryptedPrivateKey, final String keyPassword) { try { final PrivateKey rsaPrivateKey = encryptedPrivateKeyStringToPrivateKey(encryptedPrivateKey, keyPassword.toCharArray()); @@ -95,8 +119,7 @@ private String getDecryptedPrivateKey(final String encryptedPrivateKey, final St private String privateKeyAsString(final PrivateKey key) throws IOException { final StringWriter sw = new StringWriter(); - try (JcaPEMWriter pw = new JcaPEMWriter(sw)) - { + try (JcaPEMWriter pw = new JcaPEMWriter(sw)) { pw.writeObject(key); } return sw.toString(); From d66769884df5d542f4f85baff5d458ecda8f7c12 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 27 Jul 2021 20:31:44 +0000 Subject: [PATCH 132/192] Update to PeerClientPool client builder. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../plugins/prepper/peerforwarder/PeerClientPool.java | 7 ++++++- .../plugins/prepper/peerforwarder/PeerForwarder.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java index 5ea2503e10..cde4a7f3cc 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java @@ -33,9 +33,11 @@ public static PeerClientPool getInstance() { public void setClientTimeoutSeconds(int clientTimeoutSeconds) { this.clientTimeoutSeconds = clientTimeoutSeconds; } + public void setSsl(boolean ssl) { this.ssl = ssl; } + public void setCertificate(final Certificate certificate) { this.certificate = certificate; } @@ -54,12 +56,15 @@ private TraceServiceGrpc.TraceServiceBlockingStub createGRPCClient(final String .factory(ClientFactory.builder() .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager( new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)) - )).build() + ) + ).tlsNoVerifyHosts(ipAddress) + .build() ); } else { clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTP, ipAddress)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)); } + return clientBuilder.build(TraceServiceGrpc.TraceServiceBlockingStub.class); } } diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java index eb6169db8b..9ca62098bc 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java @@ -136,7 +136,7 @@ private void processRequest(final TraceServiceGrpc.TraceServiceBlockingStub clie forwardRequestTimer.record(() -> client.export(request)); forwardedRequestCounter.increment(); } catch (Exception e) { - LOG.error(String.format("Failed to forward the request:\n%s\n", request.toString())); + LOG.error(String.format("Failed to forward the request:\n%s\n", request.toString()), e); forwardRequestErrorCounter.increment(); localBuffer.add(new Record<>(request)); } From 58fa08153188ef7a6e9464f677194e10153de381 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 27 Jul 2021 21:12:10 +0000 Subject: [PATCH 133/192] Replacing hashmap with concurrent hashmap in PeerForwarder. --- .../plugins/prepper/peerforwarder/PeerForwarder.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java index 9ca62098bc..980c6fe535 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; @DataPrepperPlugin(name = "peer_forwarder", type = PluginType.PREPPER) public class PeerForwarder extends AbstractPrepper, Record> { @@ -48,9 +49,9 @@ public PeerForwarder(final PluginSetting pluginSetting, this.peerClientPool = peerClientPool; this.hashRing = hashRing; this.maxNumSpansPerRequest = maxNumSpansPerRequest; - forwardedRequestCounters = new HashMap<>(); - forwardRequestErrorCounters = new HashMap<>(); - forwardRequestTimers = new HashMap<>(); + forwardedRequestCounters = new ConcurrentHashMap<>(); + forwardRequestErrorCounters = new ConcurrentHashMap<>(); + forwardRequestTimers = new ConcurrentHashMap<>(); } public PeerForwarder(final PluginSetting pluginSetting) { From 0f04ab8beca90d26838f99747e63d0b7e4c6792c Mon Sep 17 00:00:00 2001 From: dinujoh <86094133+dinujoh@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:11:40 -0500 Subject: [PATCH 134/192] Update roleSessionName to be less that 64 character (#740) * The roleSessionName has length constraint max of 64 characters. * Removing the pipelineName as the length was exceeding 64 characters. Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- data-prepper-plugins/elasticsearch/README.md | 2 +- .../ConnectionConfiguration.java | 26 +++++++++---------- .../ConnectionConfigurationTests.java | 6 ++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/README.md b/data-prepper-plugins/elasticsearch/README.md index 197bf925ba..8d1f8d51ca 100644 --- a/data-prepper-plugins/elasticsearch/README.md +++ b/data-prepper-plugins/elasticsearch/README.md @@ -70,7 +70,7 @@ Default is null. - `aws_region`: A String represents the region of Amazon Elasticsearch Service domain, e.g. us-west-2. Only applies to Amazon Elasticsearch Service. Defaults to `us-east-1`. -- `aws_sts_role`: A IAM role which the sink plugin will assume to sign request to Amazon Elasticsearch. If not provided the plugin will use the default credentials. +- `aws_sts_role_arn`: A IAM role arn which the sink plugin will assume to sign request to Amazon Elasticsearch. If not provided the plugin will use the default credentials. - `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to `false`. diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java index 15ce300f72..0a4fc39c1e 100644 --- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfiguration.java @@ -54,7 +54,7 @@ public class ConnectionConfiguration { public static final String INSECURE = "insecure"; public static final String AWS_SIGV4 = "aws_sigv4"; public static final String AWS_REGION = "aws_region"; - public static final String AWS_STS_ROLE = "aws_sts_role"; + public static final String AWS_STS_ROLE_ARN = "aws_sts_role_arn"; private final List hosts; private final String username; @@ -65,7 +65,7 @@ public class ConnectionConfiguration { private final boolean insecure; private final boolean awsSigv4; private final String awsRegion; - private final String awsStsRole; + private final String awsStsRoleArn; private final String pipelineName; public List getHosts() { @@ -88,8 +88,8 @@ public String getAwsRegion() { return awsRegion; } - public String getAwsStsRole() { - return awsStsRole; + public String getAwsStsRoleArn() { + return awsStsRoleArn; } public Path getCertPath() { @@ -114,7 +114,7 @@ private ConnectionConfiguration(final Builder builder) { this.insecure = builder.insecure; this.awsSigv4 = builder.awsSigv4; this.awsRegion = builder.awsRegion; - this.awsStsRole = builder.awsStsRole; + this.awsStsRoleArn = builder.awsStsRoleArn; this.pipelineName = builder.pipelineName; } @@ -143,7 +143,7 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe builder.withAwsSigv4(pluginSetting.getBooleanOrDefault(AWS_SIGV4, false)); if (builder.awsSigv4) { builder.withAwsRegion(pluginSetting.getStringOrDefault(AWS_REGION, DEFAULT_AWS_REGION)); - builder.withAWSStsRole(pluginSetting.getStringOrDefault(AWS_STS_ROLE, null)); + builder.withAWSStsRoleArn(pluginSetting.getStringOrDefault(AWS_STS_ROLE_ARN, null)); } final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); @@ -169,7 +169,7 @@ public RestHighLevelClient createClient() { i++; } final RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); - /** + /* * Given that this is a patch release, we will support only the IAM based access policy AES domains. * We will not support FGAC and Custom endpoint domains. This will be followed in the next version. */ @@ -197,13 +197,13 @@ private void attachSigV4(final RestClientBuilder restClientBuilder) { LOG.info("{} is set, will sign requests using AWSRequestSigningApacheInterceptor", AWS_SIGV4); final Aws4Signer aws4Signer = Aws4Signer.create(); AwsCredentialsProvider credentialsProvider; - if (awsStsRole != null && !awsStsRole.isEmpty()) { + if (awsStsRoleArn != null && !awsStsRoleArn.isEmpty()) { credentialsProvider = StsAssumeRoleCredentialsProvider.builder() .stsClient(StsClient.create()) .refreshRequest(AssumeRoleRequest.builder() - .roleSessionName(pipelineName + " Elasticsearch-Sink " + UUID.randomUUID() + .roleSessionName("Elasticsearch-Sink-" + UUID.randomUUID() .toString()) - .roleArn(awsStsRole) + .roleArn(awsStsRoleArn) .build()) .build(); } else { @@ -281,7 +281,7 @@ public static class Builder { private boolean insecure; private boolean awsSigv4; private String awsRegion; - private String awsStsRole; + private String awsStsRoleArn; private String pipelineName; @@ -337,8 +337,8 @@ public Builder withAwsRegion(final String awsRegion) { return this; } - public Builder withAWSStsRole(final String awsStsRole) { - this.awsStsRole = awsStsRole; + public Builder withAWSStsRoleArn(final String awsStsRoleArn) { + this.awsStsRoleArn = awsStsRoleArn; return this; } diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java index a8ddea43d2..10459127a2 100644 --- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ConnectionConfigurationTests.java @@ -152,14 +152,14 @@ public void testCreateClientWithAWSSigV4AndSTSRole() throws IOException { ConnectionConfiguration.readConnectionConfiguration(pluginSetting); assertEquals("us-east-1", connectionConfiguration.getAwsRegion()); assertTrue(connectionConfiguration.isAwsSigv4()); - assertEquals("some-iam-role", connectionConfiguration.getAwsStsRole()); + assertEquals("some-iam-role", connectionConfiguration.getAwsStsRoleArn()); assertEquals(TEST_PIPELINE_NAME, connectionConfiguration.getPipelineName()); } private PluginSetting generatePluginSetting( final List hosts, final String username, final String password, final Integer connectTimeout, final Integer socketTimeout, final boolean awsSigv4, final String awsRegion, - final String awsStsRole, final String certPath, final boolean insecure) { + final String awsStsRoleArn, final String certPath, final boolean insecure) { final Map metadata = new HashMap<>(); metadata.put("hosts", hosts); metadata.put("username", username); @@ -170,7 +170,7 @@ private PluginSetting generatePluginSetting( if (awsRegion != null) { metadata.put("aws_region", awsRegion); } - metadata.put("aws_sts_role", awsStsRole); + metadata.put("aws_sts_role_arn", awsStsRoleArn); metadata.put("cert", certPath); metadata.put("insecure", insecure); final PluginSetting pluginSetting = new PluginSetting("elasticsearch", metadata); From 86fe6b5c4e7f357f628093798746b118cc815e34 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Thu, 29 Jul 2021 20:53:11 +0000 Subject: [PATCH 135/192] Allowing PeerForwarder port to be configurable. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- data-prepper-plugins/peer-forwarder/README.md | 1 + .../plugins/prepper/peerforwarder/PeerClientPool.java | 8 +++++++- .../prepper/peerforwarder/PeerForwarderConfig.java | 6 ++++++ .../plugins/prepper/peerforwarder/PeerClientPoolTest.java | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/README.md b/data-prepper-plugins/peer-forwarder/README.md index e674110924..895978d145 100644 --- a/data-prepper-plugins/peer-forwarder/README.md +++ b/data-prepper-plugins/peer-forwarder/README.md @@ -91,6 +91,7 @@ IAM policy shows the necessary permissions. * `time_out`: timeout in seconds for sending `ExportTraceServiceRequest`. Defaults to 3 seconds. * `span_agg_count`: batch size for number of spans per `ExportTraceServiceRequest`. Defaults to 48. +* `target_port`: the destination port to forward requests to. Defaults to `21890` * `discovery_mode`: peer discovery mode to be used. Allowable values are `static`, `dns`, and `aws_cloud_map`. Defaults to `static` * `static_endpoints`: list containing endpoints of all Data Prepper instances. * `domain_name`: single domain name to query DNS against. Typically used by creating multiple [DNS A Records](https://www.cloudflare.com/learning/dns/dns-records/dns-a-record/) for the same domain. diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java index cde4a7f3cc..3d6d282e8d 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPool.java @@ -18,6 +18,7 @@ public class PeerClientPool { private static final PeerClientPool INSTANCE = new PeerClientPool(); private final Map peerClients; + private int port; private int clientTimeoutSeconds = 3; private boolean ssl; private Certificate certificate; @@ -27,6 +28,7 @@ private PeerClientPool() { } public static PeerClientPool getInstance() { + // TODO: remove singleton now that port is configurable return INSTANCE; } @@ -38,6 +40,10 @@ public void setSsl(boolean ssl) { this.ssl = ssl; } + public void setPort(int port) { + this.port = port; + } + public void setCertificate(final Certificate certificate) { this.certificate = certificate; } @@ -51,7 +57,7 @@ private TraceServiceGrpc.TraceServiceBlockingStub createGRPCClient(final String // TODO: replace hardcoded port with customization final ClientBuilder clientBuilder; if (ssl) { - clientBuilder = Clients.builder(String.format("%s://%s:21890/", GRPC_HTTPS, ipAddress)) + clientBuilder = Clients.builder(String.format("%s://%s:%s/", GRPC_HTTPS, ipAddress, port)) .writeTimeout(Duration.ofSeconds(clientTimeoutSeconds)) .factory(ClientFactory.builder() .tlsCustomizer(sslContextBuilder -> sslContextBuilder.trustManager( diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java index f7cf753d2d..ac462290bf 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java @@ -13,6 +13,7 @@ public class PeerForwarderConfig { public static final String TIME_OUT = "time_out"; public static final String MAX_NUM_SPANS_PER_REQUEST = "span_agg_count"; public static final int NUM_VIRTUAL_NODES = 10; + public static final String TARGET_PORT = "target_port"; public static final String DISCOVERY_MODE = "discovery_mode"; public static final String DOMAIN_NAME = "domain_name"; public static final String STATIC_ENDPOINTS = "static_endpoints"; @@ -21,6 +22,7 @@ public class PeerForwarderConfig { private static final boolean DEFAULT_SSL = true; private static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; private static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + private static final int DEFAULT_TARGET_PORT = 21890; private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; private static final String ACM_CERT_ARN = "acmCertificateArn"; @@ -51,6 +53,10 @@ public static PeerForwarderConfig buildConfig(final PluginSetting pluginSetting) final HashRing hashRing = new HashRing(peerListProvider, NUM_VIRTUAL_NODES); final PeerClientPool peerClientPool = PeerClientPool.getInstance(); peerClientPool.setClientTimeoutSeconds(3); + + final int targetPort = pluginSetting.getIntegerOrDefault(TARGET_PORT, DEFAULT_TARGET_PORT); + peerClientPool.setPort(targetPort); + final boolean ssl = pluginSetting.getBooleanOrDefault(SSL, DEFAULT_SSL); final String sslKeyCertChainFilePath = pluginSetting.getStringOrDefault(SSL_KEY_CERT_FILE, null); final boolean useAcmCertForSsl = pluginSetting.getBooleanOrDefault(USE_ACM_CERT_FOR_SSL, DEFAULT_USE_ACM_CERT_FOR_SSL); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java index 8f7d53d300..6ce87c0a63 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerClientPoolTest.java @@ -29,6 +29,7 @@ public class PeerClientPoolTest { @Test public void testGetClientValidAddress() { PeerClientPool pool = PeerClientPool.getInstance(); + pool.setPort(PORT); TraceServiceGrpc.TraceServiceBlockingStub client = pool.getClient(VALID_ADDRESS); From ae047e26eee779f5f2e38b4f7be98417935cd093 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:04:59 +0000 Subject: [PATCH 136/192] Bump micrometer-registry-cloudwatch2 in /data-prepper-core Bumps [micrometer-registry-cloudwatch2](https://github.com/micrometer-metrics/micrometer) from 1.7.1 to 1.7.2. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.1...v1.7.2) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-registry-cloudwatch2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 917e9de9ac..ed9e1a02eb 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation "org.reflections:reflections:0.9.12" implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" - implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.1" + implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.2" implementation "software.amazon.awssdk:cloudwatch:2.16.93" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.11.2" From 1b1d22c9e39765a650023c8764328b7834af4a03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:03 +0000 Subject: [PATCH 137/192] Bump jackson-dataformat-yaml from 2.12.3 to 2.12.4 in /data-prepper-core Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 917e9de9ac..f57b16d422 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation project(':data-prepper-plugins:common') testImplementation project(':data-prepper-plugins:common').sourceSets.test.output implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "javax.validation:validation-api:2.0.1.Final" implementation "org.apache.bval:bval-extras:2.0.5" implementation "org.apache.bval:bval-jsr:2.0.5" From 3c8e2a79ca962beba1725f23f0f1d4d455c05242 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:12 +0000 Subject: [PATCH 138/192] Bump aws-java-sdk-s3 in /data-prepper-plugins/otel-trace-source Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.12.15 to 1.12.37. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.15...1.12.37) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 7dbadc0430..5753fc2818 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "commons-io:commons-io:2.10.0" - implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.37" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" From e4272ce9306d24837c2c12e2994b551ae0b2e49d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:19 +0000 Subject: [PATCH 139/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-source Bumps [armeria-grpc](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 7dbadc0430..a9310092cf 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" - implementation "com.linecorp.armeria:armeria-grpc:1.9.1" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.apache.commons:commons-lang3:3.12.0" From 3bfd5b33dfacdc272164e88f07e8285ee7a4f6b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:27 +0000 Subject: [PATCH 140/192] Bump jackson-dataformat-yaml Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 5845fbcda9..8f06f02da3 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' testImplementation 'org.assertj:assertj-core:3.20.2' testImplementation "org.mockito:mockito-inline:3.11.2" From 4c8108b5154f088b2db3177ea44bb1350ff8981a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:41 +0000 Subject: [PATCH 141/192] Bump armeria in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 5845fbcda9..c027554dc7 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' - implementation "com.linecorp.armeria:armeria:1.9.1" + implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" From 126eb96febeb1a552e74dc0ab39dc443ecec0a24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:05:57 +0000 Subject: [PATCH 142/192] Bump kotlin-stdlib in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib](https://github.com/JetBrains/kotlin) from 1.5.20 to 1.5.21. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.20...v1.5.21) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin:kotlin-stdlib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index 909edcd5f9..41362a0303 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.20' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.21' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.20' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output From 7b0c13be5b7aab30652e94c01f66c1f92130dc1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:06:33 +0000 Subject: [PATCH 143/192] Bump jackson-databind in /data-prepper-plugins/common Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index a671fc6daa..818ad6712b 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -3,7 +3,7 @@ plugins { } dependencies { api project(':data-prepper-api') - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.reflections:reflections:0.9.12" testImplementation "junit:junit:4.13.2" From 4dced6bc32ceef8c6f454028df55fa0fa1e15b65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:06:38 +0000 Subject: [PATCH 144/192] Bump commons-io from 2.10.0 to 2.11.0 in /data-prepper-plugins/common Bumps commons-io from 2.10.0 to 2.11.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index a671fc6daa..eaef85f617 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -7,7 +7,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.reflections:reflections:0.9.12" testImplementation "junit:junit:4.13.2" - testImplementation "commons-io:commons-io:2.10.0" + testImplementation "commons-io:commons-io:2.11.0" } jacocoTestCoverageVerification { From 5f3a7d37255e4ba3580a97eb04e59d84c105dc9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:00 +0000 Subject: [PATCH 145/192] Bump commons-io in /data-prepper-plugins/peer-forwarder Bumps commons-io from 2.10.0 to 2.11.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index eac28736b6..789bea9f3e 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -19,7 +19,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" implementation "com.amazonaws:aws-java-sdk-acm:1.12.15" implementation 'software.amazon.awssdk:servicediscovery' - implementation "commons-io:commons-io:2.10.0" + implementation "commons-io:commons-io:2.11.0" implementation "org.apache.commons:commons-lang3:3.12.0" implementation "commons-validator:commons-validator:1.7" testImplementation(platform('org.junit:junit-bom:5.7.2')) From 5c75e9b90ab9eea457347fa1d87854d470a53aeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:08 +0000 Subject: [PATCH 146/192] Bump micrometer-core in /data-prepper-plugins/service-map-stateful Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.1 to 1.7.2. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.1...v1.7.2) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index 07e35db802..f070b10c2b 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation project(':data-prepper-plugins:mapdb-prepper-state') testImplementation project(':data-prepper-api').sourceSets.test.output - implementation "io.micrometer:micrometer-core:1.7.1" + implementation "io.micrometer:micrometer-core:1.7.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" From 416988df64ba25f17e30f1ddb3f71620a110f780 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:08 +0000 Subject: [PATCH 147/192] Bump aws-java-sdk-acm in /data-prepper-plugins/peer-forwarder Bumps [aws-java-sdk-acm](https://github.com/aws/aws-sdk-java) from 1.12.15 to 1.12.37. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.15...1.12.37) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-acm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index eac28736b6..05694e9696 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.9.1" implementation(platform('software.amazon.awssdk:bom:2.16.104')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" - implementation "com.amazonaws:aws-java-sdk-acm:1.12.15" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.37" implementation 'software.amazon.awssdk:servicediscovery' implementation "commons-io:commons-io:2.10.0" implementation "org.apache.commons:commons-lang3:3.12.0" From 0fcadc83b0b7dd0478a3980af1ef50d5d8f1b947 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:10 +0000 Subject: [PATCH 148/192] Bump jackson-core in /data-prepper-plugins/elasticsearch Bumps [jackson-core](https://github.com/FasterXML/jackson-core) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-core/releases) - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.12.3...jackson-core-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index a4a983976b..4f56949656 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -62,7 +62,7 @@ configurations.all { force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' - force 'com.fasterxml.jackson.core:jackson-core:2.12.3' + force 'com.fasterxml.jackson.core:jackson-core:2.12.4' force 'com.fasterxml.jackson:jackson-bom:2.12.3' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' force 'commons-logging:commons-logging:1.2' From f5fee1b73e17ecd8b38c16c12cd1d7118b721855 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:13 +0000 Subject: [PATCH 149/192] Bump armeria-grpc in /data-prepper-plugins/peer-forwarder Bumps [armeria-grpc](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index eac28736b6..aef06e9772 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -14,7 +14,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.9.1" - implementation "com.linecorp.armeria:armeria-grpc:1.9.1" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation(platform('software.amazon.awssdk:bom:2.16.104')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" implementation "com.amazonaws:aws-java-sdk-acm:1.12.15" From 1c34758962ffce20bb1cbf75a1ddce914e051bc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:15 +0000 Subject: [PATCH 150/192] Bump awaitility in /data-prepper-plugins/peer-forwarder Bumps [awaitility](https://github.com/awaitility/awaitility) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/awaitility/awaitility/releases) - [Changelog](https://github.com/awaitility/awaitility/blob/master/changelog.txt) - [Commits](https://github.com/awaitility/awaitility/commits/awaitility-4.1.0) --- updated-dependencies: - dependency-name: org.awaitility:awaitility dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index eac28736b6..ed44a44d9b 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -30,7 +30,7 @@ dependencies { testImplementation "org.mockito:mockito-core:3.11.2" testImplementation 'org.mockito:mockito-junit-jupiter:3.11.2' testImplementation "commons-io:commons-io:2.10.0" - testImplementation 'org.awaitility:awaitility:4.0.3' + testImplementation 'org.awaitility:awaitility:4.1.0' } test { From 0d6c2dc66ebc40e2150756a170b12efee727f71d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:07:18 +0000 Subject: [PATCH 151/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.12.15 to 1.12.37. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.15...1.12.37) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index a4a983976b..2cf4aa6f36 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -58,7 +58,7 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.15' + force 'com.amazonaws:aws-java-sdk-core:1.12.37' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' From a8d8a656a086847b64690e9c59cf3ca53f15dd45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:08:37 +0000 Subject: [PATCH 152/192] Bump jackson-databind in /data-prepper-plugins/otel-trace-group-prepper Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index b41798a363..166cd2348e 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation project(':data-prepper-plugins:elasticsearch') testImplementation project(':data-prepper-api').sourceSets.test.output implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "io.micrometer:micrometer-core:1.7.1" testImplementation "org.mockito:mockito-core:3.11.2" From 25b54f5e3e68c1edc83efd3993c130506ab8b1d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Aug 2021 04:08:41 +0000 Subject: [PATCH 153/192] Bump micrometer-core in /data-prepper-plugins/otel-trace-group-prepper Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.1 to 1.7.2. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.1...v1.7.2) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index b41798a363..05ea8f5ba9 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" - implementation "io.micrometer:micrometer-core:1.7.1" + implementation "io.micrometer:micrometer-core:1.7.2" testImplementation "org.mockito:mockito-core:3.11.2" testImplementation "junit:junit:4.13.2" } \ No newline at end of file From 58f73e6a1e7463fa59a2e426517cfbf575887395 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Mon, 9 Aug 2021 17:05:16 -0500 Subject: [PATCH 154/192] =?UTF-8?q?Added=20dedicated=20threadpool=20for=20?= =?UTF-8?q?peer=20forward=20requests.=20Improved=20buffer=E2=80=A6=20(#777?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added dedicated threadpool for peer forward requests. Improved buffer and forwarder metrics. Fixed DNS domain name validator being too strict. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- .../dataprepper/metrics/MetricNames.java | 5 ++ .../dataprepper/metrics/PluginMetrics.java | 8 ++ .../model/buffer/AbstractBuffer.java | 4 + .../metrics/PluginMetricsTest.java | 37 ++++++-- .../oteltrace/OTelTraceGrpcService.java | 4 + data-prepper-plugins/peer-forwarder/README.md | 6 +- .../prepper/peerforwarder/PeerForwarder.java | 88 +++++++++++++------ .../discovery/DiscoveryUtils.java | 4 +- .../peerforwarder/PeerForwarderTest.java | 14 +-- .../DnsPeerListProvider_CreateTest.java | 4 +- 10 files changed, 128 insertions(+), 46 deletions(-) diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java index cba16007c2..8c0798b2c0 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/MetricNames.java @@ -42,6 +42,11 @@ private MetricNames() {} */ public static final String RECORDS_INFLIGHT = "recordsInFlight"; + /** + * Metric representing the number of records currently in the buffer. + */ + public static final String RECORDS_IN_BUFFER = "recordsInBuffer"; + /** * Metric representing the number of records read from a buffer and processed by the pipeline. */ diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java index 15f5259a4e..652e739c9f 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/metrics/PluginMetrics.java @@ -38,6 +38,10 @@ public Counter counter(final String name) { return Metrics.counter(getMeterName(name)); } + public Counter counterWithTags(final String name, final String... tags) { + return Metrics.counter(getMeterName(name), tags); + } + public Counter counter(final String name, final String metricsPrefix) { return Metrics.counter(new StringJoiner(MetricNames.DELIMITER).add(metricsPrefix).add(name).toString()); } @@ -46,6 +50,10 @@ public Timer timer(final String name) { return Metrics.timer(getMeterName(name)); } + public Timer timerWithTags(final String name, final String... tags) { + return Metrics.timer(getMeterName(name), tags); + } + public DistributionSummary summary(final String name) { return Metrics.summary(getMeterName(name)); } diff --git a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java index 46cf6a9090..679f0c5c21 100644 --- a/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java +++ b/data-prepper-api/src/main/java/com/amazon/dataprepper/model/buffer/AbstractBuffer.java @@ -22,6 +22,7 @@ public abstract class AbstractBuffer> implements Buffer { private final Counter recordsWrittenCounter; private final Counter recordsReadCounter; private final AtomicLong recordsInFlight; + private final AtomicLong recordsInBuffer; private final Counter recordsProcessedCounter; private final Counter writeTimeoutCounter; private final Timer writeTimer; @@ -41,6 +42,7 @@ private AbstractBuffer(final PluginMetrics pluginMetrics, final String pipelineN this.recordsWrittenCounter = pluginMetrics.counter(MetricNames.RECORDS_WRITTEN); this.recordsReadCounter = pluginMetrics.counter(MetricNames.RECORDS_READ); this.recordsInFlight = pluginMetrics.gauge(MetricNames.RECORDS_INFLIGHT, new AtomicLong()); + this.recordsInBuffer = pluginMetrics.gauge(MetricNames.RECORDS_IN_BUFFER, new AtomicLong()); this.recordsProcessedCounter = pluginMetrics.counter(MetricNames.RECORDS_PROCESSED, pipelineName); this.writeTimeoutCounter = pluginMetrics.counter(MetricNames.WRITE_TIMEOUTS); this.writeTimer = pluginMetrics.timer(MetricNames.WRITE_TIME_ELAPSED); @@ -63,6 +65,7 @@ public void write(T record, int timeoutInMillis) throws TimeoutException { try { doWrite(record, timeoutInMillis); recordsWrittenCounter.increment(); + recordsInBuffer.incrementAndGet(); } catch (TimeoutException e) { writeTimeoutCounter.increment(); throw e; @@ -83,6 +86,7 @@ public Map.Entry, CheckpointState> read(int timeoutInMillis) { final Map.Entry, CheckpointState> readResult = readTimer.record(() -> doRead(timeoutInMillis)); recordsReadCounter.increment(readResult.getKey().size() * 1.0); recordsInFlight.addAndGet(readResult.getValue().getNumRecordsToBeChecked()); + recordsInBuffer.addAndGet(-1 * readResult.getValue().getNumRecordsToBeChecked()); return readResult; } diff --git a/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java b/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java index f20f242178..8720aa83a7 100644 --- a/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java +++ b/data-prepper-api/src/test/java/com/amazon/dataprepper/metrics/PluginMetricsTest.java @@ -1,9 +1,6 @@ package com.amazon.dataprepper.metrics; import com.amazon.dataprepper.model.configuration.PluginSetting; -import java.util.Collections; -import java.util.StringJoiner; -import java.util.concurrent.atomic.AtomicInteger; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Metrics; @@ -11,9 +8,15 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Collections; +import java.util.StringJoiner; +import java.util.concurrent.atomic.AtomicInteger; + public class PluginMetricsTest { private static final String PLUGIN_NAME = "testPlugin"; private static final String PIPELINE_NAME = "pipelineName"; + private static final String TAG_KEY = "tagKey"; + private static final String TAG_VALUE = "tagValue"; private static final PluginSetting PLUGIN_SETTING = new PluginSetting(PLUGIN_NAME, Collections.emptyMap()) {{ setPipelineName(PIPELINE_NAME); }}; @@ -29,6 +32,18 @@ public void testCounter() { counter.getId().getName()); } + @Test + public void testCounterWithTags() { + final Counter counter = PLUGIN_METRICS.counterWithTags("counter", TAG_KEY, TAG_VALUE); + Assert.assertEquals( + new StringJoiner(MetricNames.DELIMITER) + .add(PIPELINE_NAME).add(PLUGIN_NAME) + .add("counter").toString(), + counter.getId().getName()); + + Assert.assertEquals(TAG_VALUE, counter.getId().getTag(TAG_KEY)); + } + @Test public void testCustomMetricsPrefixCounter() { final Counter counter = PLUGIN_METRICS.counter("counter", PIPELINE_NAME); @@ -40,12 +55,24 @@ public void testCustomMetricsPrefixCounter() { @Test public void testTimer() { - final Timer counter = PLUGIN_METRICS.timer("timer"); + final Timer timer = PLUGIN_METRICS.timer("timer"); Assert.assertEquals( new StringJoiner(MetricNames.DELIMITER) .add(PIPELINE_NAME).add(PLUGIN_NAME) .add("timer").toString(), - counter.getId().getName()); + timer.getId().getName()); + } + + @Test + public void testTimerWithTags() { + final Timer timer = PLUGIN_METRICS.timerWithTags("timer", TAG_KEY, TAG_VALUE); + Assert.assertEquals( + new StringJoiner(MetricNames.DELIMITER) + .add(PIPELINE_NAME).add(PLUGIN_NAME) + .add("timer").toString(), + timer.getId().getName()); + + Assert.assertEquals(TAG_VALUE, timer.getId().getTag(TAG_KEY)); } @Test diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index 43d4e31caa..d79b4b5717 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -10,10 +10,13 @@ import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.TimeoutException; public class OTelTraceGrpcService extends TraceServiceGrpc.TraceServiceImplBase { + private static final Logger LOG = LoggerFactory.getLogger(OTelTraceGrpcService.class); public static final String REQUEST_TIMEOUTS = "requestTimeouts"; public static final String REQUESTS_RECEIVED = "requestsReceived"; @@ -51,6 +54,7 @@ public void export(ExportTraceServiceRequest request, StreamObserver, Record> { - public static final String FORWARD_REQUEST_LATENCY_PREFIX = "forwardRequestLatency"; - public static final String FORWARD_REQUEST_SUCCESS_PREFIX = "forwardRequestSuccess"; - public static final String FORWARD_REQUEST_ERRORS_PREFIX = "forwardRequestErrors"; + public static final String REQUESTS = "requests"; + public static final String LATENCY = "latency"; + public static final String ERRORS = "errors"; + public static final String DESTINATION = "destination"; + + public static final int ASYNC_REQUEST_THREAD_COUNT = 200; private static final Logger LOG = LoggerFactory.getLogger(PeerForwarder.class); @@ -41,6 +48,8 @@ public class PeerForwarder extends AbstractPrepper forwardedRequestCounters; private final Map forwardRequestErrorCounters; + private final ExecutorService executorService; + public PeerForwarder(final PluginSetting pluginSetting, final PeerClientPool peerClientPool, final HashRing hashRing, @@ -52,6 +61,8 @@ public PeerForwarder(final PluginSetting pluginSetting, forwardedRequestCounters = new ConcurrentHashMap<>(); forwardRequestErrorCounters = new ConcurrentHashMap<>(); forwardRequestTimers = new ConcurrentHashMap<>(); + + executorService = Executors.newFixedThreadPool(ASYNC_REQUEST_THREAD_COUNT); } public PeerForwarder(final PluginSetting pluginSetting) { @@ -84,8 +95,8 @@ public List> doExecute(final Collection> results = new ArrayList<>(); + final List> recordsToProcessLocally = new ArrayList<>(); + final List> forwardedRequestFutures = new ArrayList<>(); for (final Map.Entry> entry : groupedRS.entrySet()) { final TraceServiceGrpc.TraceServiceBlockingStub client; @@ -102,8 +113,11 @@ public List> doExecute(final Collection= maxNumSpansPerRequest) { final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - // Send the batch request to designated remote peer or ingest into localhost - processRequest(client, currRequest, results); + if (client == null) { + recordsToProcessLocally.add(new Record<>(currRequest)); + } else { + forwardedRequestFutures.add(processRequest(client, currRequest)); + } currRequestBuilder = ExportTraceServiceRequest.newBuilder(); currSpansCount = 0; } @@ -113,37 +127,57 @@ public List> doExecute(final Collection 0) { final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - processRequest(client, currRequest, results); + if (client == null) { + recordsToProcessLocally.add(new Record<>(currRequest)); + } else { + forwardedRequestFutures.add(processRequest(client, currRequest)); + } } } - return results; + + for (final CompletableFuture future : forwardedRequestFutures) { + try { + final Record record = future.get(); + if (record != null) { + recordsToProcessLocally.add(record); + } + } catch (InterruptedException | ExecutionException e) { + LOG.error("Problem with asynchronous peer forwarding", e); + } + } + + return recordsToProcessLocally; } /** - * Forward request to the peer address if client is given, otherwise push the request to local buffer. + * Asynchronously forwards a request to the peer address. Returns a record with an empty payload if + * the request succeeds, otherwise the payload will contain the failed ExportTraceServiceRequest to + * be processed locally. */ - private void processRequest(final TraceServiceGrpc.TraceServiceBlockingStub client, - final ExportTraceServiceRequest request, - final List> localBuffer) { - if (client != null) { - final String peerIp = client.getChannel().authority(); - final Timer forwardRequestTimer = forwardRequestTimers.computeIfAbsent( - peerIp, ip -> pluginMetrics.timer(String.format("%s:%s", FORWARD_REQUEST_LATENCY_PREFIX, ip))); - final Counter forwardedRequestCounter = forwardedRequestCounters.computeIfAbsent( - peerIp, ip -> pluginMetrics.counter(String.format("%s:%s", FORWARD_REQUEST_SUCCESS_PREFIX, ip))); - final Counter forwardRequestErrorCounter = forwardRequestErrorCounters.computeIfAbsent( - peerIp, ip -> pluginMetrics.counter(String.format("%s:%s", FORWARD_REQUEST_ERRORS_PREFIX, ip))); + private CompletableFuture processRequest(final TraceServiceGrpc.TraceServiceBlockingStub client, + final ExportTraceServiceRequest request) { + final String peerIp = client.getChannel().authority(); + final Timer forwardRequestTimer = forwardRequestTimers.computeIfAbsent( + peerIp, ip -> pluginMetrics.timerWithTags(LATENCY, DESTINATION, ip)); + final Counter forwardedRequestCounter = forwardedRequestCounters.computeIfAbsent( + peerIp, ip -> pluginMetrics.counterWithTags(REQUESTS, DESTINATION, ip)); + final Counter forwardRequestErrorCounter = forwardRequestErrorCounters.computeIfAbsent( + peerIp, ip -> pluginMetrics.counterWithTags(ERRORS, DESTINATION, ip)); + + final CompletableFuture callFuture = CompletableFuture.supplyAsync(() -> + { + forwardedRequestCounter.increment(); try { forwardRequestTimer.record(() -> client.export(request)); - forwardedRequestCounter.increment(); + return null; } catch (Exception e) { - LOG.error(String.format("Failed to forward the request:\n%s\n", request.toString()), e); + LOG.error("Failed to forward request to address: {}", peerIp, e); forwardRequestErrorCounter.increment(); - localBuffer.add(new Record<>(request)); + return new Record<>(request); } - } else { - localBuffer.add(new Record<>(request)); - } + }, executorService); + + return callFuture; } private boolean isAddressDefinedLocally(final String address) { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java index c6c9c1d0dc..b7bc0c870a 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DiscoveryUtils.java @@ -1,10 +1,10 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder.discovery; -import org.apache.commons.validator.routines.DomainValidator; +import com.google.common.net.InternetDomainName; import org.apache.commons.validator.routines.InetAddressValidator; class DiscoveryUtils { static boolean validateEndpoint(final String endpoint) { - return DomainValidator.getInstance(true).isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); + return InternetDomainName.isValid(endpoint) || InetAddressValidator.getInstance().isValid(endpoint); } } diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java index 13e03a7b21..9080e2d891 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java @@ -213,17 +213,17 @@ public void testSingleRemoteIpForwardedRequestOnly() throws Exception { // Verify metrics final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_ERRORS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.ERRORS).toString()); Assert.assertEquals(1, forwardRequestErrorMeasurements.size()); Assert.assertEquals(0.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_SUCCESS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.REQUESTS).toString()); Assert.assertEquals(1, forwardRequestSuccessMeasurements.size()); Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_LATENCY_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.LATENCY).toString()); Assert.assertEquals(3, forwardRequestLatencyMeasurements.size()); // COUNT Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); @@ -261,17 +261,17 @@ public void testSingleRemoteIpForwardRequestError() { // Verify metrics final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_ERRORS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.ERRORS).toString()); Assert.assertEquals(1, forwardRequestErrorMeasurements.size()); Assert.assertEquals(1.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_SUCCESS_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.REQUESTS).toString()); Assert.assertEquals(1, forwardRequestSuccessMeasurements.size()); - Assert.assertEquals(0.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); + Assert.assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(String.format("%s:%s", PeerForwarder.FORWARD_REQUEST_LATENCY_PREFIX, fullPeerIp)).toString()); + .add(PeerForwarder.LATENCY).toString()); Assert.assertEquals(3, forwardRequestLatencyMeasurements.size()); // COUNT Assert.assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java index 309d61df84..739d94a89e 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/discovery/DnsPeerListProvider_CreateTest.java @@ -28,7 +28,7 @@ class DnsPeerListProvider_CreateTest { private static final String PLUGIN_NAME = "PLUGIN_NAME"; - private static final String ENDPOINT = "ENDPOINT"; + private static final String VALID_ENDPOINT = "VALID.ENDPOINT"; private static final String INVALID_ENDPOINT = "INVALID_ENDPOINT_"; private static final String PIPELINE_NAME = "pipelineName"; @@ -55,7 +55,7 @@ void setup() { @Test void testCreateProviderDnsInstance() { - pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, ENDPOINT); + pluginSetting.getSettings().put(PeerForwarderConfig.DOMAIN_NAME, VALID_ENDPOINT); when(dnsAddressEndpointGroupBuilder.build()).thenReturn(dnsAddressEndpointGroup); when(dnsAddressEndpointGroupBuilder.ttl(anyInt(), anyInt())).thenReturn(dnsAddressEndpointGroupBuilder); From 6cd48190eb2eb530c08496eadb0f245c97e59403 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:00:05 +0000 Subject: [PATCH 155/192] Bump armeria-grpc in /data-prepper-plugins/otel-trace-raw-prepper Bumps [armeria-grpc](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria-grpc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index 7b54fa1d46..ca7b2f7f92 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.2" - implementation "com.linecorp.armeria:armeria-grpc:1.9.1" + implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' From f1c18e369b88a03dcf33806eb9e29b81d039bb93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:03:11 +0000 Subject: [PATCH 156/192] Bump jackson-databind from 2.12.3 to 2.12.4 in /data-prepper-core Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index 80ab2c7a22..ffd71bd17e 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':data-prepper-plugins') implementation project(':data-prepper-plugins:common') testImplementation project(':data-prepper-plugins:common').sourceSets.test.output - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "javax.validation:validation-api:2.0.1.Final" implementation "org.apache.bval:bval-extras:2.0.5" From 084caa8a10d9b9f68f4dbf2834ecb5ebec0c75d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:03:12 +0000 Subject: [PATCH 157/192] Bump kotlin-stdlib-common in /data-prepper-plugins/mapdb-prepper-state Bumps [kotlin-stdlib-common](https://github.com/JetBrains/kotlin) from 1.5.20 to 1.5.21. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.5.20...v1.5.21) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin:kotlin-stdlib-common dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle index 41362a0303..af930916bc 100644 --- a/data-prepper-plugins/mapdb-prepper-state/build.gradle +++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation project(':data-prepper-plugins:common') implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8' implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.21' - implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.20' + implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.21' testImplementation project(':data-prepper-plugins:common').sourceSets.test.output testImplementation group: 'junit', name: 'junit', version: '4.13.2' From 9f25fdefff52d13703ef46fe7ed55e1ef2a88513 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:03:16 +0000 Subject: [PATCH 158/192] Bump jackson-databind in /data-prepper-plugins/otel-trace-source Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index fd59da696a..ac8c2901e9 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation "org.apache.commons:commons-lang3:3.12.0" implementation "org.bouncycastle:bcprov-jdk15on:1.69" From cff20311d34e8faab8fe18f46cd31a6e645e0f75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 10:03:20 -0700 Subject: [PATCH 159/192] Bump jackson-annotations in /data-prepper-plugins/elasticsearch Bumps [jackson-annotations](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-annotations dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index a4a983976b..2507536ead 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -58,11 +58,11 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.15' + force 'com.amazonaws:aws-java-sdk-core:1.12.37' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' - force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' + force 'com.fasterxml.jackson.core:jackson-annotations:2.12.4' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' - force 'com.fasterxml.jackson.core:jackson-core:2.12.3' + force 'com.fasterxml.jackson.core:jackson-core:2.12.4' force 'com.fasterxml.jackson:jackson-bom:2.12.3' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' force 'commons-logging:commons-logging:1.2' From 7b18856c31f3d50caff6a66cd2e88eac7ef6f160 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:03:23 +0000 Subject: [PATCH 160/192] Bump jackson-databind in /data-prepper-plugins/elasticsearch Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index a4a983976b..4ff74f19bb 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -32,7 +32,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output api project(':data-prepper-plugins:common') implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.16.95' @@ -58,11 +58,11 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.15' + force 'com.amazonaws:aws-java-sdk-core:1.12.37' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' - force 'com.fasterxml.jackson.core:jackson-core:2.12.3' + force 'com.fasterxml.jackson.core:jackson-core:2.12.4' force 'com.fasterxml.jackson:jackson-bom:2.12.3' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' force 'commons-logging:commons-logging:1.2' From fd3a386858a0632b80eb07e94d449db81e150a11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:05:40 +0000 Subject: [PATCH 161/192] Bump jackson-dataformat-yaml in /data-prepper-plugins/common Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle index 2cfd2cf69e..57863a02b2 100644 --- a/data-prepper-plugins/common/build.gradle +++ b/data-prepper-plugins/common/build.gradle @@ -4,7 +4,7 @@ plugins { dependencies { api project(':data-prepper-api') implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "org.reflections:reflections:0.9.12" testImplementation "junit:junit:4.13.2" testImplementation "commons-io:commons-io:2.11.0" From 21fce764135c84ee96d31e4eff759e586854b8ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:05:41 +0000 Subject: [PATCH 162/192] Bump jackson-databind in /data-prepper-plugins/service-map-stateful Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/service-map-stateful/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/service-map-stateful/build.gradle b/data-prepper-plugins/service-map-stateful/build.gradle index f070b10c2b..5c0b2c2f87 100644 --- a/data-prepper-plugins/service-map-stateful/build.gradle +++ b/data-prepper-plugins/service-map-stateful/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation project(':data-prepper-plugins:mapdb-prepper-state') testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.micrometer:micrometer-core:1.7.2" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-inline:3.11.2" From bff63f0f6ec7f9827eea0b3021218d0bf8b27ba0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:05:57 +0000 Subject: [PATCH 163/192] Bump jackson-databind in /data-prepper-plugins/otel-trace-raw-prepper Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle index ca7b2f7f92..207656a09e 100644 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre' testImplementation 'org.assertj:assertj-core:3.20.2' From 42c57331146fc41e97726c3a439cd30755837d42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:05:58 +0000 Subject: [PATCH 164/192] Bump jackson-dataformat-yaml Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-group-prepper/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle index 7c7cabf763..3ddb13a9f7 100644 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ b/data-prepper-plugins/otel-trace-group-prepper/build.gradle @@ -12,7 +12,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "io.micrometer:micrometer-core:1.7.2" testImplementation "org.mockito:mockito-core:3.11.2" testImplementation "junit:junit:4.13.2" From b682b7c15f22176c605415533caddc292d6e0162 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:05:58 +0000 Subject: [PATCH 165/192] Bump software.amazon.awssdk:bom in /data-prepper-plugins/peer-forwarder Bumps [software.amazon.awssdk:bom](https://github.com/aws/aws-sdk-java-v2) from 2.16.104 to 2.17.15. - [Release notes](https://github.com/aws/aws-sdk-java-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java-v2/compare/2.16.104...2.17.15) --- updated-dependencies: - dependency-name: software.amazon.awssdk:bom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 003f78f0fc..7c4ef51d6e 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" - implementation(platform('software.amazon.awssdk:bom:2.16.104')) + implementation(platform('software.amazon.awssdk:bom:2.17.15')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" implementation "com.amazonaws:aws-java-sdk-acm:1.12.37" implementation 'software.amazon.awssdk:servicediscovery' From be3983f845e6f3b4e2aa17cc0e0120ac8deb4e05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:06:29 +0000 Subject: [PATCH 166/192] Bump jackson-dataformat-yaml in /data-prepper-plugins/otel-trace-source Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index ac8c2901e9..0d6138bff9 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -15,7 +15,7 @@ dependencies { implementation "com.linecorp.armeria:armeria:1.9.1" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation "org.apache.commons:commons-lang3:3.12.0" implementation "org.bouncycastle:bcprov-jdk15on:1.69" implementation "org.bouncycastle:bcpkix-jdk15on:1.69" From 4a434594dcb653ab8069b1eb97a45432f2598460 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:06:34 +0000 Subject: [PATCH 167/192] Bump aws-java-sdk-acm in /data-prepper-plugins/peer-forwarder Bumps [aws-java-sdk-acm](https://github.com/aws/aws-sdk-java) from 1.12.37 to 1.12.43. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.37...1.12.43) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-acm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 003f78f0fc..56db274c97 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation(platform('software.amazon.awssdk:bom:2.16.104')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" - implementation "com.amazonaws:aws-java-sdk-acm:1.12.37" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" implementation 'software.amazon.awssdk:servicediscovery' implementation "commons-io:commons-io:2.11.0" implementation "org.apache.commons:commons-lang3:3.12.0" From 2fd2541464cd8256aa8c6267fe725dfd2d1e60b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:06:34 +0000 Subject: [PATCH 168/192] Bump aws-java-sdk-core in /data-prepper-plugins/elasticsearch Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.12.37 to 1.12.43. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.37...1.12.43) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index e5626174fa..67da7a57e8 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -32,7 +32,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output api project(':data-prepper-plugins:common') implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" - implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3" + implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.16.95' @@ -58,9 +58,9 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.37' + force 'com.amazonaws:aws-java-sdk-core:1.12.43' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' - force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' + force 'com.fasterxml.jackson.core:jackson-annotations:2.12.4' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' force 'com.fasterxml.jackson.core:jackson-core:2.12.4' force 'com.fasterxml.jackson:jackson-bom:2.12.3' From bea16e2a555d87813f7cab25db467a4f8a38556b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:06:54 +0000 Subject: [PATCH 169/192] Bump micrometer-core from 1.7.1 to 1.7.2 in /data-prepper-api Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.1 to 1.7.2. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.1...v1.7.2) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle index f680e9e11f..de198a3d91 100644 --- a/data-prepper-api/build.gradle +++ b/data-prepper-api/build.gradle @@ -2,7 +2,7 @@ plugins { } dependencies { - implementation "io.micrometer:micrometer-core:1.7.1" + implementation "io.micrometer:micrometer-core:1.7.2" testImplementation "org.hamcrest:hamcrest:2.2" } From 5b8a2407bf70542f4f21b68bb7419a26b8734458 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:07:14 +0000 Subject: [PATCH 170/192] Bump aws-java-sdk-s3 in /data-prepper-plugins/otel-trace-source Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.12.37 to 1.12.43. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.37...1.12.43) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 0d6138bff9..0adcbae7c5 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -9,7 +9,7 @@ dependencies { testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "commons-io:commons-io:2.10.0" - implementation "com.amazonaws:aws-java-sdk-s3:1.12.37" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.1" From 032703f12a7e106afe1a5ab50668310e4f4a3fff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:07:31 +0000 Subject: [PATCH 171/192] Bump commons-io in /data-prepper-plugins/otel-trace-source Bumps commons-io from 2.10.0 to 2.11.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 0adcbae7c5..b5de999ff0 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.15' testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "commons-io:commons-io:2.10.0" + implementation "commons-io:commons-io:2.11.0" implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' From 2777bc7a1bea0a0d0a0e0aef119c11d128e5b80b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:07:33 +0000 Subject: [PATCH 172/192] Bump armeria from 1.9.1 to 1.9.2 in /data-prepper-plugins/peer-forwarder Bumps [armeria](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 3304fee3fa..4291f833ae 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':data-prepper-api') testImplementation project(':data-prepper-api').sourceSets.test.output implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" - implementation "com.linecorp.armeria:armeria:1.9.1" + implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation(platform('software.amazon.awssdk:bom:2.17.15')) implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" From f3c639caafe6c71705b56a8583ad8af678258ad6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:07:46 +0000 Subject: [PATCH 173/192] Bump jackson-bom in /data-prepper-plugins/elasticsearch Bumps [jackson-bom](https://github.com/FasterXML/jackson-bom) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-bom/releases) - [Commits](https://github.com/FasterXML/jackson-bom/compare/jackson-bom-2.12.3...jackson-bom-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson:jackson-bom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 58b139d572..f89873dea9 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -58,12 +58,12 @@ compileJava.options.warnings = false // Resolve dependency conflict between ES sink and main project configurations.all { resolutionStrategy { - force 'com.amazonaws:aws-java-sdk-core:1.12.37' + force 'com.amazonaws:aws-java-sdk-core:1.12.43' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3' force 'com.fasterxml.jackson.core:jackson-annotations:2.12.4' force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' force 'com.fasterxml.jackson.core:jackson-core:2.12.4' - force 'com.fasterxml.jackson:jackson-bom:2.12.3' + force 'com.fasterxml.jackson:jackson-bom:2.12.4' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' force 'commons-logging:commons-logging:1.2' force 'org.apache.httpcomponents:httpclient:4.5.10' From e24d23e3ced8daf2f6b0e4ef944af6f7fdb937e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:08:13 +0000 Subject: [PATCH 174/192] Bump sdk-core in /data-prepper-plugins/elasticsearch Bumps sdk-core from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:sdk-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 67da7a57e8..e2834e3d80 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.16.95' implementation 'software.amazon.awssdk:http-client-spi:2.16.95' - implementation 'software.amazon.awssdk:sdk-core:2.16.95' + implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.16.95' implementation 'software.amazon.awssdk:utils:2.16.95' From a209c2e8be98a0ce6300d27aec59c2c3322e349f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:08:15 +0000 Subject: [PATCH 175/192] Bump url-connection-client in /data-prepper-plugins/elasticsearch Bumps url-connection-client from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:url-connection-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 67da7a57e8..ecb1fc0e92 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -42,7 +42,7 @@ dependencies { implementation 'software.amazon.awssdk:regions:2.16.95' implementation 'software.amazon.awssdk:utils:2.16.95' implementation 'software.amazon.awssdk:sts:2.16.95' - implementation 'software.amazon.awssdk:url-connection-client:2.16.95' + implementation 'software.amazon.awssdk:url-connection-client:2.17.15' implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell From e936fffb80b450cebbc6814dd13e40e75c9e2f2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:09:03 +0000 Subject: [PATCH 176/192] Bump armeria in /data-prepper-plugins/otel-trace-source Bumps [armeria](https://github.com/line/armeria) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/line/armeria/releases) - [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg) - [Commits](https://github.com/line/armeria/compare/armeria-1.9.1...armeria-1.9.2) --- updated-dependencies: - dependency-name: com.linecorp.armeria:armeria dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index b5de999ff0..2013128de2 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' - implementation "com.linecorp.armeria:armeria:1.9.1" + implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" From 8fd5debeef35a138a94277e50d60f914aa52024e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:09:21 +0000 Subject: [PATCH 177/192] Bump auth from 2.16.95 to 2.17.15 in /data-prepper-plugins/elasticsearch Bumps auth from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:auth dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index e2834e3d80..97fa3d1e51 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation 'software.amazon.awssdk:auth:2.16.95' + implementation 'software.amazon.awssdk:auth:2.17.15' implementation 'software.amazon.awssdk:http-client-spi:2.16.95' implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' From ac81633b111bac34bdaf81819e366b60f06bdd6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:09:33 +0000 Subject: [PATCH 178/192] Bump aws-java-sdk-acm in /data-prepper-plugins/otel-trace-source Bumps [aws-java-sdk-acm](https://github.com/aws/aws-sdk-java) from 1.12.12 to 1.12.43. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.12...1.12.43) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-acm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 2013128de2..03fbd27df1 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -10,7 +10,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}" implementation "commons-io:commons-io:2.11.0" implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" - implementation "com.amazonaws:aws-java-sdk-acm:1.12.12" + implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" implementation 'com.google.protobuf:protobuf-java-util:3.17.3' implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" From 2714eb214d1c601b764ce5af5301f7eca6f1c405 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:09:34 +0000 Subject: [PATCH 179/192] Bump aws-java-sdk-s3 in /data-prepper-plugins/peer-forwarder Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.12.15 to 1.12.43. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.15...1.12.43) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/peer-forwarder/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle index 4291f833ae..30c2801aec 100644 --- a/data-prepper-plugins/peer-forwarder/build.gradle +++ b/data-prepper-plugins/peer-forwarder/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation "com.linecorp.armeria:armeria:1.9.2" implementation "com.linecorp.armeria:armeria-grpc:1.9.2" implementation(platform('software.amazon.awssdk:bom:2.17.15')) - implementation "com.amazonaws:aws-java-sdk-s3:1.12.15" + implementation "com.amazonaws:aws-java-sdk-s3:1.12.43" implementation "com.amazonaws:aws-java-sdk-acm:1.12.43" implementation 'software.amazon.awssdk:servicediscovery' implementation "commons-io:commons-io:2.11.0" From ab88451040e21373c1897d2408bf0b30a5e13066 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:09:47 +0000 Subject: [PATCH 180/192] Bump jackson-dataformat-smile in /data-prepper-plugins/elasticsearch Bumps [jackson-dataformat-smile](https://github.com/FasterXML/jackson-dataformats-binary) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-binary/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-binary/compare/jackson-dataformats-binary-2.12.3...jackson-dataformats-binary-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-smile dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index e2834e3d80..e5fc9925d1 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -35,7 +35,7 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' - implementation 'software.amazon.awssdk:auth:2.16.95' + implementation 'software.amazon.awssdk:auth:2.17.15' implementation 'software.amazon.awssdk:http-client-spi:2.16.95' implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' @@ -70,7 +70,7 @@ configurations.all { force "org.hdrhistogram:HdrHistogram:2.1.12" force 'joda-time:joda-time:2.10.10' force 'org.yaml:snakeyaml:1.29' - force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.3' + force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.4' force 'junit:junit:4.13' force "org.slf4j:slf4j-api:1.7.30" force "org.apache.logging.log4j:log4j-api:2.14.1" From 6dd9f730ca1be84a395df3fe240565ece1ed75bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:10:07 +0000 Subject: [PATCH 181/192] Bump cloudwatch from 2.16.93 to 2.17.15 in /data-prepper-core Bumps cloudwatch from 2.16.93 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:cloudwatch dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index ffd71bd17e..bc9fbc2287 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation "io.micrometer:micrometer-core:1.6.5" implementation "io.micrometer:micrometer-registry-prometheus:1.6.5" implementation "io.micrometer:micrometer-registry-cloudwatch2:1.7.2" - implementation "software.amazon.awssdk:cloudwatch:2.16.93" + implementation "software.amazon.awssdk:cloudwatch:2.17.15" testImplementation "org.hamcrest:hamcrest:2.2" testImplementation "org.mockito:mockito-core:3.11.2" } From 09bb9ce4afcf08c9c5cbd76012415cc3198c3799 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:10:19 +0000 Subject: [PATCH 182/192] Bump http-client-spi in /data-prepper-plugins/elasticsearch Bumps [http-client-spi](https://github.com/aws/aws-sdk-java-v2) from 2.16.95 to 2.17.15. - [Release notes](https://github.com/aws/aws-sdk-java-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java-v2/compare/2.16.95...2.17.15) --- updated-dependencies: - dependency-name: software.amazon.awssdk:http-client-spi dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index e5fc9925d1..c8f08af05a 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.17.15' - implementation 'software.amazon.awssdk:http-client-spi:2.16.95' + implementation 'software.amazon.awssdk:http-client-spi:2.17.15' implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.16.95' From 56df9be83b79857c629daed668e017a87a4590d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:13:30 +0000 Subject: [PATCH 183/192] Bump regions in /data-prepper-plugins/elasticsearch Bumps regions from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:regions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 84ff1dfcbe..3cc9ad835e 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -39,7 +39,7 @@ dependencies { implementation 'software.amazon.awssdk:http-client-spi:2.16.95' implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' - implementation 'software.amazon.awssdk:regions:2.16.95' + implementation 'software.amazon.awssdk:regions:2.17.15' implementation 'software.amazon.awssdk:utils:2.16.95' implementation 'software.amazon.awssdk:sts:2.16.95' implementation 'software.amazon.awssdk:url-connection-client:2.17.15' From c5410759862801d489b2afad97becbc31f6bb63c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:13:34 +0000 Subject: [PATCH 184/192] Bump sts from 2.16.95 to 2.17.15 in /data-prepper-plugins/elasticsearch Bumps sts from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:sts dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 618ae698bf..7b3af19182 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -41,7 +41,7 @@ dependencies { implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.16.95' implementation 'software.amazon.awssdk:utils:2.16.95' - implementation 'software.amazon.awssdk:sts:2.16.95' + implementation 'software.amazon.awssdk:sts:2.17.15' implementation 'software.amazon.awssdk:url-connection-client:2.17.15' implementation "io.micrometer:micrometer-core:1.7.1" testImplementation("junit:junit:4.13.2") { From 57fd7abe858ffab9a00796134b537a341e97a273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:14:31 +0000 Subject: [PATCH 185/192] Bump jackson-dataformat-yaml in /data-prepper-plugins/elasticsearch Bumps [jackson-dataformat-yaml](https://github.com/FasterXML/jackson-dataformats-text) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-text/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-text/compare/jackson-dataformats-text-2.12.3...jackson-dataformats-text-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 84ff1dfcbe..3378926b19 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -33,7 +33,7 @@ dependencies { api project(':data-prepper-plugins:common') implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.17.15' implementation 'software.amazon.awssdk:http-client-spi:2.16.95' From dfcb847f287f5aa0b9cd01d7fcf931fce9d3662a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:15:46 +0000 Subject: [PATCH 186/192] Bump slf4j-api in /data-prepper-plugins/elasticsearch Bumps [slf4j-api](https://github.com/qos-ch/slf4j) from 1.7.30 to 1.7.32. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.30...v_1.7.32) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-api dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 074a9366c7..6f8c42bf5c 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -33,7 +33,7 @@ dependencies { api project(':data-prepper-plugins:common') implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${es_version}" implementation "com.fasterxml.jackson.core:jackson-databind:2.12.4" - implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3" + implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4" implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:auth:2.17.15' implementation 'software.amazon.awssdk:http-client-spi:2.17.15' @@ -72,7 +72,7 @@ configurations.all { force 'org.yaml:snakeyaml:1.29' force 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.12.4' force 'junit:junit:4.13' - force "org.slf4j:slf4j-api:1.7.30" + force "org.slf4j:slf4j-api:1.7.32" force "org.apache.logging.log4j:log4j-api:2.14.1" force "org.apache.logging.log4j:log4j-core:2.14.1" } From 57c4aa19fba0b76b54f9a7132727d0d4e62a3121 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:15:50 +0000 Subject: [PATCH 187/192] Bump utils in /data-prepper-plugins/elasticsearch Bumps [utils](https://github.com/aws/aws-sdk-java-v2) from 2.16.95 to 2.17.15. - [Release notes](https://github.com/aws/aws-sdk-java-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java-v2/compare/2.16.95...2.17.15) --- updated-dependencies: - dependency-name: software.amazon.awssdk:utils dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 20fd17583c..b7d774754f 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -40,7 +40,7 @@ dependencies { implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.17.15' - implementation 'software.amazon.awssdk:utils:2.16.95' + implementation 'software.amazon.awssdk:utils:2.17.15' implementation 'software.amazon.awssdk:sts:2.17.15' implementation 'software.amazon.awssdk:url-connection-client:2.17.15' implementation "io.micrometer:micrometer-core:1.7.1" From d3fefe0ef86d78c8e5dd882122886f7d919ecb22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:18:48 +0000 Subject: [PATCH 188/192] Bump micrometer-core in /data-prepper-plugins/elasticsearch Bumps [micrometer-core](https://github.com/micrometer-metrics/micrometer) from 1.7.1 to 1.7.2. - [Release notes](https://github.com/micrometer-metrics/micrometer/releases) - [Commits](https://github.com/micrometer-metrics/micrometer/compare/v1.7.1...v1.7.2) --- updated-dependencies: - dependency-name: io.micrometer:micrometer-core dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 20fd17583c..34e0cd06be 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -40,10 +40,10 @@ dependencies { implementation 'software.amazon.awssdk:sdk-core:2.17.15' implementation 'software.amazon.awssdk:aws-core:2.16.95' implementation 'software.amazon.awssdk:regions:2.17.15' - implementation 'software.amazon.awssdk:utils:2.16.95' + implementation 'software.amazon.awssdk:utils:2.17.15' implementation 'software.amazon.awssdk:sts:2.17.15' implementation 'software.amazon.awssdk:url-connection-client:2.17.15' - implementation "io.micrometer:micrometer-core:1.7.1" + implementation "io.micrometer:micrometer-core:1.7.2" testImplementation("junit:junit:4.13.2") { exclude group:'org.hamcrest' // workaround for jarHell } From 6d3ba4813080689fbd35582688d469c0b49caa74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:19:27 +0000 Subject: [PATCH 189/192] Bump jackson-dataformat-cbor in /data-prepper-plugins/elasticsearch Bumps [jackson-dataformat-cbor](https://github.com/FasterXML/jackson-dataformats-binary) from 2.12.3 to 2.12.4. - [Release notes](https://github.com/FasterXML/jackson-dataformats-binary/releases) - [Commits](https://github.com/FasterXML/jackson-dataformats-binary/compare/jackson-dataformats-binary-2.12.3...jackson-dataformats-binary-2.12.4) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index cf0719f100..47ea97925a 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -64,7 +64,7 @@ configurations.all { force 'com.fasterxml.jackson.core:jackson-databind:2.12.3' force 'com.fasterxml.jackson.core:jackson-core:2.12.4' force 'com.fasterxml.jackson:jackson-bom:2.12.4' - force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3' + force 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.4' force 'commons-logging:commons-logging:1.2' force 'org.apache.httpcomponents:httpclient:4.5.10' force "org.hdrhistogram:HdrHistogram:2.1.12" From d82c3c01c03a8f842f89a84050c335ff0b65eb28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:20:36 +0000 Subject: [PATCH 190/192] Bump aws-core in /data-prepper-plugins/elasticsearch Bumps aws-core from 2.16.95 to 2.17.15. --- updated-dependencies: - dependency-name: software.amazon.awssdk:aws-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index ddc895e828..3b2641ab44 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -38,7 +38,7 @@ dependencies { implementation 'software.amazon.awssdk:auth:2.17.15' implementation 'software.amazon.awssdk:http-client-spi:2.17.15' implementation 'software.amazon.awssdk:sdk-core:2.17.15' - implementation 'software.amazon.awssdk:aws-core:2.16.95' + implementation 'software.amazon.awssdk:aws-core:2.17.15' implementation 'software.amazon.awssdk:regions:2.17.15' implementation 'software.amazon.awssdk:utils:2.17.15' implementation 'software.amazon.awssdk:sts:2.17.15' From 073d6be6c5dc7e084e37083fbf26f1821f5da3dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Aug 2021 18:37:51 +0000 Subject: [PATCH 191/192] Bump commons-io in /data-prepper-plugins/elasticsearch Bumps commons-io from 2.10.0 to 2.11.0. --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- data-prepper-plugins/elasticsearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle index 3b2641ab44..5b6853d4b3 100644 --- a/data-prepper-plugins/elasticsearch/build.gradle +++ b/data-prepper-plugins/elasticsearch/build.gradle @@ -49,7 +49,7 @@ dependencies { } testImplementation "org.awaitility:awaitility:4.1.0" testImplementation "org.elasticsearch.test:framework:${es_version}" - testImplementation "commons-io:commons-io:2.10.0" + testImplementation "commons-io:commons-io:2.11.0" } // Workaround for Werror From 9457c8d22b3aff93ee9113bc830e58268065d784 Mon Sep 17 00:00:00 2001 From: Jeff Wright <74204404+wrijeff@users.noreply.github.com> Date: Tue, 10 Aug 2021 19:24:55 +0000 Subject: [PATCH 192/192] Updated hash ring vnode count to more evenly distribute load. Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com> --- data-prepper-core/integrationTest.gradle | 12 ++++++------ .../raw-span-e2e-pipeline-latest-release.yml | 2 +- .../resources/service-map-e2e-pipeline.yml | 2 +- .../prepper/peerforwarder/PeerForwarderConfig.java | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle index 631ec30c04..2e1c3d3571 100644 --- a/data-prepper-core/integrationTest.gradle +++ b/data-prepper-core/integrationTest.gradle @@ -170,9 +170,9 @@ task rawSpanEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer def createDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") + "rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") def createDataPrepper2Task = createDataPrepperDockerContainer( - "rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") + "rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task @@ -207,9 +207,9 @@ task rawSpanCompatibilityEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer def createDataPrepper1Task = createDataPrepperDockerContainer( - "rawSpanDataPrepperFromBuild", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + "rawSpanDataPrepperFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage( - "rawSpanDataPrepperFromPull", "data-prepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") + "rawSpanDataPrepperFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task @@ -244,9 +244,9 @@ task serviceMapEndToEndTest(type: Test) { dependsOn build dependsOn startOdfeDockerContainer def createDataPrepper1Task = createDataPrepperDockerContainer( - "serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") + "serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") def createDataPrepper2Task = createDataPrepperDockerContainer( - "serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") + "serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer) def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer) dependsOn startDataPrepper1Task diff --git a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml index a21c367f24..5f8cf74d13 100644 --- a/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml +++ b/data-prepper-core/src/integrationTest/resources/raw-span-e2e-pipeline-latest-release.yml @@ -5,7 +5,7 @@ entry-pipeline: prepper: - peer_forwarder: discovery_mode: "static" - static_endpoints: ["data-prepper1", "data-prepper2"] + static_endpoints: ["dataprepper1", "dataprepper2"] ssl: false sink: - pipeline: diff --git a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml index 9850b20a8d..8e97cb711b 100644 --- a/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml +++ b/data-prepper-core/src/integrationTest/resources/service-map-e2e-pipeline.yml @@ -5,7 +5,7 @@ entry-pipeline: prepper: - peer_forwarder: discovery_mode: "static" - static_endpoints: ["data-prepper1", "data-prepper2"] + static_endpoints: ["dataprepper1", "dataprepper2"] ssl: false sink: - pipeline: diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java index ac462290bf..215270a3fc 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderConfig.java @@ -12,7 +12,7 @@ public class PeerForwarderConfig { public static final String TIME_OUT = "time_out"; public static final String MAX_NUM_SPANS_PER_REQUEST = "span_agg_count"; - public static final int NUM_VIRTUAL_NODES = 10; + public static final int NUM_VIRTUAL_NODES = 128; public static final String TARGET_PORT = "target_port"; public static final String DISCOVERY_MODE = "discovery_mode"; public static final String DOMAIN_NAME = "domain_name"; @@ -23,6 +23,7 @@ public class PeerForwarderConfig { private static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL"; private static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; private static final int DEFAULT_TARGET_PORT = 21890; + private static final int DEFAULT_TIMEOUT_SECONDS = 2; private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis"; private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; private static final String ACM_CERT_ARN = "acmCertificateArn"; @@ -84,7 +85,7 @@ public static PeerForwarderConfig buildConfig(final PluginSetting pluginSetting) return new PeerForwarderConfig( peerClientPool, hashRing, - pluginSetting.getIntegerOrDefault(TIME_OUT, 3), + pluginSetting.getIntegerOrDefault(TIME_OUT, DEFAULT_TIMEOUT_SECONDS), pluginSetting.getIntegerOrDefault(MAX_NUM_SPANS_PER_REQUEST, 48)); }