From f87adf4000d96148b6f07c5045f403fc0cfac0fc Mon Sep 17 00:00:00 2001
From: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
Date: Mon, 26 Apr 2021 15:49:21 +0000
Subject: [PATCH 001/192] Updating THIRD-PARTY attributions.
Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
---
THIRD-PARTY | 62 +++++++++++++++--------------------------------------
1 file changed, 17 insertions(+), 45 deletions(-)
diff --git a/THIRD-PARTY b/THIRD-PARTY
index 6a988b22fa..ab4e9d5984 100644
--- a/THIRD-PARTY
+++ b/THIRD-PARTY
@@ -2,10 +2,9 @@
** Amazon ION Java; version 1.0.2 -- https://github.com/amzn/ion-java
** android annotations; version 4.1.1.4 --
https://github.com/androidannotations/androidannotations
-** Apache Lucene; version 8.5.1 -- http://lucene.apache.org/
-** armeria; version 1.5.0 -- https://github.com/line/armeria
-** armeria-grpc; version 1.5.0 -- https://github.com/line/armeria
-** aws-java-sdk-core; version 1.11.965 -- https://aws.amazon.com/sdk-for-java/
+** Apache Lucene; version 8.7.0 -- http://lucene.apache.org/
+** armeria-grpc; version 1.6.0 -- https://github.com/line/armeria
+** aws-java-sdk-core; version 1.11.1001 -- https://aws.amazon.com/sdk-for-java/
** aws-request-signing-apache-interceptor; version b3772780da --
https://github.com/awslabs/aws-request-signing-apache-interceptor
** bval; version 2.0.4 -- https://github.com/apache/bval
@@ -13,46 +12,45 @@ https://github.com/awslabs/aws-request-signing-apache-interceptor
** commons-codec; version 1.15 -- https://github.com/apache/commons-codec
** commons-logging; version 1.2 --
https://github.com/apache/commons-logging/tree/LOGGING_1_1_3
-** Elasticsearch; version 7.8.0 -- https://github.com/elastic/elasticsearch
-** Elasticsearch Java High Level Rest Client; version 7.8.1 --
+** Elasticsearch; version 7.10.2 -- https://github.com/elastic/elasticsearch
+** Elasticsearch Java High Level Rest Client; version 7.10.2 --
https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high.html
** Error Prone; version 2.3.4 -- https://github.com/google/error-prone
** Findbugs Jsr305; version 3.0.2 --
https://search.maven.org/artifact/com.google.code.findbugs/jsr305/3.0.2/jar
** Google Gson; version 2.8.6 -- https://github.com/google/gson
-** gRPC; version 1.35.0 -- https://github.com/grpc/grpc
-** guava; version 29.0 -- https://github.com/google/guava
+** gRPC; version 1.36.1 -- https://github.com/grpc/grpc
+** guava; version 30.1.1-jre -- https://github.com/google/guava
** Hppc; version 0.8.1 --
https://github.com/carrotsearch/hppc/blob/master/LICENSE.txt
** httpcomponents-core; version 4.4.12 --
https://github.com/apache/httpcomponents-core/blob/rel/v4.4.1
** J2ObjC; version 1.3 -- https://developers.google.com/j2objc/
-** Jackson; version 2.12.1 -- http://github.com/FasterXML/jackson
+** Jackson; version 2.12.3 -- http://github.com/FasterXML/jackson
** JavaAssist; version 3.26.0-GA --
https://github.com/jboss-javassist/javassist
** JavaxValidationApi; version 2.0.1.Final -- http://beanvalidation.org
** jetbrains-annotations; version 13.0 --
https://github.com/JetBrains/java-annotations/blob/master/LICENSE.txt
** jffi; version 1.2.23 -- https://github.com/jnr/jffi/blob/jffi-1.2.23
-** jna; version 4.5.1 -- https://github.com/java-native-access/jna
+** jna; version 5.5.0 -- https://github.com/java-native-access/jna
** jnr a64asm; version 1.0.0 --
https://github.com/jnr/jnr-a64asm/tree/jnr-a64asm-1.0.0
** jnr constants; version 0.9.15 --
https://github.com/jnr/jnr-constants/tree/jnr-constants-0.9.15
** joda-time; version 2.10.4 -- http://www.joda.org/joda-time/
-** Kotlin; version 1.2.71 -- https://github.com/JetBrains/kotlin
-** lmdbjava; version 0.8.1 -- https://github.com/lmdbjava/lmdbjava
+** Kotlin; version 1.4.32 -- https://github.com/JetBrains/kotlin
** log4j-slf4j-impl; version 2.14.0 -- https://logging.apache.org/log4j/2.x/
** lz4-java; version 1.3.0 -- https://github.com/lz4/lz4-java/tree/1.3.0
** MapDB; version 3.0.8 -- http://www.mapdb.org/
-** micrometer; version 1.5.5 --
+** micrometer; version 1.6.6 --
https://github.com/micrometer-metrics/micrometer
-** micrometer-registry-prometheus; version 1.5.5 --
+** micrometer-registry-prometheus; version 1.6.5 --
https://github.com/micrometer-metrics/micrometer
** MustacheJava; version 0.9.6 -- https://github.com/spullara/mustache.java
-** netty 4.1.51; version 4.1.58.Final --
+** netty 4.1.51; version 4.1.63.Final --
https://github.com/netty/netty/tree/netty-4.1.50.Final
-** OpenTelemetry 0.10.0; version 0.10.0 --
+** OpenTelemetry 0.10.0; version 1.0.1-alpha --
https://github.com/open-telemetry/opentelemetry-java
** perfmark-api; version 0.23.0 -- https://github.com/perfmark/perfmark
** SnakeYAML; version 1.28 -- http://www.snakeyaml.org
@@ -306,13 +304,6 @@ limitations under the License.
ICU4J, (under contrib/collation) is licensed under an MIT styles license
(contrib/collation/lib/ICU-LICENSE.txt) and Copyright (c) 1995-2008
International Business Machines Corporation and others
-* For armeria see also this required NOTICE:
- Armeria
- =======
-
- Please visit the official web site for more information:
-
- * https://armeria.dev/
* For armeria-grpc see also this required NOTICE:
Armeria
=======
@@ -540,26 +531,6 @@ limitations under the License.
limitations under the License.
* For Kotlin see also this required NOTICE:
Copyright 2010-2020 JetBrains s.r.o and respective authors and developers
-* For lmdbjava see also this required NOTICE:
- /*-
- * #%L
- * LmdbJava
- * %%
- * Copyright (C) 2016 - 2020 The LmdbJava Open Source Project
- * %%
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * #L%
- */
* For log4j-slf4j-impl see also this required NOTICE:
Apache Log4j SLF4J Binding
Copyright 1999-2020 The Apache Software Foundation
@@ -830,7 +801,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
------
-** google-protobuf; version 3.15.3 --
+** google-protobuf; version 3.15.8 --
https://github.com/protocolbuffers/protobuf
Copyright 2008 Google Inc. All rights reserved.
@@ -882,7 +853,7 @@ work. If not, see .
------
-** eclipse-collections; version 11.0.0.M1 --
+** eclipse-collections; version 11.0.0.M2 --
https://github.com/eclipse/eclipse-collections
Copyright (c) 2016 Shotaro Sano.
@@ -892,6 +863,7 @@ Copyright (c) 2017 Goldman Sachs and others.
Copyright (c) 2018 Goldman Sachs and others.
Copyright (c) 2019 Goldman Sachs and others.
Copyright (c) 2020 Goldman Sachs and others.
+Copyright (c) 2021 Goldman Sachs and others.
Eclipse Public License - v 1.0
From 5e32109c1564360c049403b23b4ec4c3a7832402 Mon Sep 17 00:00:00 2001
From: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
Date: Tue, 27 Apr 2021 11:18:43 -0500
Subject: [PATCH 002/192] FIX: ISM API breaking change (#546)
* FIX: ISM API breaking change
Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
---
.github/workflows/gradle.yml | 11 ---
...k-odfe-before-1_13_0-integration-tests.yml | 34 ++++++++
...nk-odfe-since-1_13_0-integration-tests.yml | 34 ++++++++
.../elasticsearch/build.gradle | 1 +
.../sink/elasticsearch/ElasticsearchSink.java | 10 ++-
.../sink/elasticsearch/IndexConstants.java | 3 +-
.../elasticsearch/IndexStateManagement.java | 81 ++++++++++++-------
...n => raw-span-policy-no-ism-template.json} | 0
.../raw-span-policy-with-ism-template.json | 22 +++++
.../elasticsearch/ElasticsearchSinkIT.java | 6 +-
10 files changed, 155 insertions(+), 47 deletions(-)
create mode 100644 .github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
create mode 100644 .github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
rename data-prepper-plugins/elasticsearch/src/main/resources/{raw-span-policy.json => raw-span-policy-no-ism-template.json} (100%)
create mode 100644 data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index f647c9002c..baab044d85 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -23,14 +23,3 @@ jobs:
run: chmod +x gradlew
- name: Build with Gradle
run: ./gradlew build
- - name: Run ODFE docker
- run: |
- export version=1.9.0
- docker pull amazon/opendistro-for-elasticsearch:$version
- docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version
- sleep 90
- - name: Run ODFE tests
- run: |
- ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
- ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
-
diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
new file mode 100644
index 0000000000..d046b7908d
--- /dev/null
+++ b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
@@ -0,0 +1,34 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+ integration_tests:
+ strategy:
+ matrix:
+ java: [14]
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+ - name: Checkout Data-Prepper
+ uses: actions/checkout@v2
+ - name: Grant execute permission for gradlew
+ run: chmod +x gradlew
+ - name: Run ODFE docker
+ run: |
+ export version=1.12.0
+ docker pull amazon/opendistro-for-elasticsearch:$version
+ docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version
+ sleep 90
+ - name: Run ODFE tests
+ run: |
+ ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
+ ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
\ No newline at end of file
diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
new file mode 100644
index 0000000000..77dfa83ab5
--- /dev/null
+++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -0,0 +1,34 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+ integration_tests:
+ strategy:
+ matrix:
+ java: [14]
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v1
+ with:
+ java-version: ${{ matrix.java }}
+ - name: Checkout Data-Prepper
+ uses: actions/checkout@v2
+ - name: Grant execute permission for gradlew
+ run: chmod +x gradlew
+ - name: Run ODFE docker
+ run: |
+ export version=1.13.2
+ docker pull amazon/opendistro-for-elasticsearch:$version
+ docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d amazon/opendistro-for-elasticsearch:$version
+ sleep 90
+ - name: Run ODFE tests
+ run: |
+ ./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
+ ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
\ No newline at end of file
diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle
index c7cc16146d..525c5f29cc 100644
--- a/data-prepper-plugins/elasticsearch/build.gradle
+++ b/data-prepper-plugins/elasticsearch/build.gradle
@@ -41,6 +41,7 @@ dependencies {
testImplementation("junit:junit:4.13.2") {
exclude group:'org.hamcrest' // workaround for jarHell
}
+ testImplementation "org.awaitility:awaitility:4.0.3"
testImplementation "org.elasticsearch.test:framework:${es_version}"
testImplementation "commons-io:commons-io:2.8.0"
}
diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java
index 3f934b6767..2c24ae41a2 100644
--- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java
+++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSink.java
@@ -35,6 +35,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.Optional;
import java.util.function.Supplier;
@DataPrepperPlugin(name = "elasticsearch", type = PluginType.SINK)
@@ -77,9 +78,10 @@ public ElasticsearchSink(final PluginSetting pluginSetting) {
public void start() throws IOException {
LOG.info("Starting Elasticsearch sink");
restHighLevelClient = esSinkConfig.getConnectionConfiguration().createClient();
- final String ismPolicyId = IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType);
+ final boolean isISMEnabled = IndexStateManagement.checkISMEnabled(restHighLevelClient);
+ final Optional policyIdOptional = isISMEnabled? IndexStateManagement.checkAndCreatePolicy(restHighLevelClient, indexType) : Optional.empty();
if (!esSinkConfig.getIndexConfiguration().getIndexTemplate().isEmpty()) {
- createIndexTemplate(ismPolicyId);
+ createIndexTemplate(isISMEnabled, policyIdOptional.orElse(null));
}
final String dlqFile = esSinkConfig.getRetryConfiguration().getDlqFile();
if ( dlqFile != null) {
@@ -144,7 +146,7 @@ private void flushBatch(final BulkRequest bulkRequest) {
});
}
- private void createIndexTemplate(final String ismPolicyId) throws IOException {
+ private void createIndexTemplate(final boolean isISMEnabled, final String ismPolicyId) throws IOException {
final String indexAlias = esSinkConfig.getIndexConfiguration().getIndexAlias();
final PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(indexAlias + "-index-template");
final boolean isRaw = indexType.equals(IndexConstants.RAW);
@@ -153,7 +155,7 @@ private void createIndexTemplate(final String ismPolicyId) throws IOException {
} else {
putIndexTemplateRequest.patterns(Collections.singletonList(indexAlias));
}
- if (ismPolicyId != null) {
+ if (isISMEnabled) {
IndexStateManagement.attachPolicy(esSinkConfig.getIndexConfiguration(), ismPolicyId, indexAlias);
}
putIndexTemplateRequest.source(esSinkConfig.getIndexConfiguration().getIndexTemplate());
diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java
index 083bd07468..ac2c903343 100644
--- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java
+++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexConstants.java
@@ -14,7 +14,8 @@ public class IndexConstants {
// TODO: extract out version number into version enum
public static final String RAW_DEFAULT_TEMPLATE_FILE = "otel-v1-apm-span-index-template.json";
public static final String RAW_ISM_POLICY = "raw-span-policy";
- public static final String RAW_ISM_FILE = "raw-span-policy.json";
+ public static final String RAW_ISM_FILE_NO_ISM_TEMPLATE = "raw-span-policy-no-ism-template.json";
+ public static final String RAW_ISM_FILE_WITH_ISM_TEMPLATE = "raw-span-policy-with-ism-template.json";
public static final String ISM_ENABLED_SETTING = "opendistro.index_state_management.enabled";
public static final String ISM_POLICY_ID_SETTING = "opendistro.index_state_management.policy_id";
public static final String ISM_ROLLOVER_ALIAS_SETTING = "opendistro.index_state_management.rollover_alias";
diff --git a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java
index 096e7a0c79..53382de43f 100644
--- a/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java
+++ b/data-prepper-plugins/elasticsearch/src/main/java/com/amazon/dataprepper/plugins/sink/elasticsearch/IndexStateManagement.java
@@ -1,13 +1,12 @@
package com.amazon.dataprepper.plugins.sink.elasticsearch;
+import com.google.common.base.Preconditions;
import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.rest.RestStatus;
import javax.ws.rs.HttpMethod;
import java.io.BufferedReader;
@@ -16,6 +15,8 @@
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
public class IndexStateManagement {
public static boolean checkISMEnabled(final RestHighLevelClient restHighLevelClient) throws IOException {
@@ -26,46 +27,66 @@ public static boolean checkISMEnabled(final RestHighLevelClient restHighLevelCli
return enabled != null && enabled.equals("true");
}
- public static String checkAndCreatePolicy(
+ /**
+ * @return ISM policy_id optional that needs to be attached to index settings.
+ */
+ public static Optional checkAndCreatePolicy(
final RestHighLevelClient restHighLevelClient, final String indexType) throws IOException {
- if (checkISMEnabled(restHighLevelClient) && indexType.equals(IndexConstants.RAW)) {
+ if (indexType.equals(IndexConstants.RAW)) {
final String endPoint = "/_opendistro/_ism/policies/" + IndexConstants.RAW_ISM_POLICY;
- Request request = new Request(HttpMethod.HEAD, endPoint);
- final Response response = restHighLevelClient.getLowLevelClient().performRequest(request);
- if (response.getStatusLine().getStatusCode() != RestStatus.OK.getStatus()) {
- final InputStream is = IndexStateManagement.class.getClassLoader().getResourceAsStream(IndexConstants.RAW_ISM_FILE);
- assert is != null;
- final StringBuilder policyJsonBuffer = new StringBuilder();
- try (final BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
- reader.lines().forEach(line -> policyJsonBuffer.append(line).append("\n"));
- }
- is.close();
- request = new Request(HttpMethod.PUT, endPoint);
- request.setJsonEntity(policyJsonBuffer.toString());
- try {
- restHighLevelClient.getLowLevelClient().performRequest(request);
- } catch (ResponseException e) {
- if (e.getMessage().contains("version_conflict_engine_exception")
- || e.getMessage().contains("resource_already_exists_exception")) {
- // Do nothing - likely caused by a race condition where the resource was created
- // by another host before this host's restClient made its request
- } else {
- throw e;
+ Request request = createPolicyRequestFromFile(endPoint, IndexConstants.RAW_ISM_FILE_WITH_ISM_TEMPLATE);
+ try {
+ restHighLevelClient.getLowLevelClient().performRequest(request);
+ } catch (ResponseException e1) {
+ final String msg = e1.getMessage();
+ if (msg.contains("Invalid field: [ism_template]")) {
+ request = createPolicyRequestFromFile(endPoint, IndexConstants.RAW_ISM_FILE_NO_ISM_TEMPLATE);
+ try {
+ restHighLevelClient.getLowLevelClient().performRequest(request);
+ } catch (ResponseException e2) {
+ if (e2.getMessage().contains("version_conflict_engine_exception")
+ || e2.getMessage().contains("resource_already_exists_exception")) {
+ // Do nothing - likely caused by
+ // (1) a race condition where the resource was created by another host before this host's
+ // restClient made its request;
+ // (2) policy already exists in the cluster
+ } else {
+ throw e2;
+ }
}
+ return Optional.of(IndexConstants.RAW_ISM_POLICY);
+ } else if (e1.getMessage().contains("version_conflict_engine_exception")
+ || e1.getMessage().contains("resource_already_exists_exception")) {
+ // Do nothing - likely caused by
+ // (1) a race condition where the resource was created by another host before this host's
+ // restClient made its request;
+ // (2) policy already exists in the cluster
+ } else {
+ throw e1;
}
}
- return IndexConstants.RAW_ISM_POLICY;
- } else {
- return null;
}
+ return Optional.empty();
}
@SuppressWarnings("unchecked")
public static void attachPolicy(
final IndexConfiguration configuration, final String ismPolicyId, final String rolloverAlias) {
configuration.getIndexTemplate().putIfAbsent("settings", new HashMap<>());
- // Attach policy_id and rollover_alias
- ((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId);
+ if (ismPolicyId != null) {
+ ((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_POLICY_ID_SETTING, ismPolicyId);
+ }
((Map) configuration.getIndexTemplate().get("settings")).put(IndexConstants.ISM_ROLLOVER_ALIAS_SETTING, rolloverAlias);
}
+
+ private static Request createPolicyRequestFromFile(final String endPoint, final String fileName) throws IOException {
+ final StringBuilder policyJsonBuffer = new StringBuilder();
+ try (final InputStream inputStream = IndexStateManagement.class.getClassLoader().getResourceAsStream(fileName);
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(inputStream)))) {
+ reader.lines().forEach(line -> policyJsonBuffer.append(line).append("\n"));
+ }
+ final Request request = new Request(HttpMethod.PUT, endPoint);
+ request.setJsonEntity(policyJsonBuffer.toString());
+ return request;
+ }
}
diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy.json b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-no-ism-template.json
similarity index 100%
rename from data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy.json
rename to data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-no-ism-template.json
diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json
new file mode 100644
index 0000000000..f70ce3f7b8
--- /dev/null
+++ b/data-prepper-plugins/elasticsearch/src/main/resources/raw-span-policy-with-ism-template.json
@@ -0,0 +1,22 @@
+{
+ "policy": {
+ "description": "Managing raw spans for trace analytics",
+ "default_state": "current_write_index",
+ "states": [
+ {
+ "name": "current_write_index",
+ "actions": [
+ {
+ "rollover": {
+ "min_size": "50gb",
+ "min_index_age": "24h"
+ }
+ }
+ ]
+ }
+ ],
+ "ism_template": {
+ "index_patterns": ["otel-v1-apm-span-*"]
+ }
+ }
+}
\ No newline at end of file
diff --git a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java
index 0e391745df..38056c0fdd 100644
--- a/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java
+++ b/data-prepper-plugins/elasticsearch/src/test/java/com/amazon/dataprepper/plugins/sink/elasticsearch/ElasticsearchSinkIT.java
@@ -51,9 +51,11 @@
import java.util.Objects;
import java.util.Optional;
import java.util.StringJoiner;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.apache.http.HttpStatus.SC_OK;
+import static org.awaitility.Awaitility.await;
public class ElasticsearchSinkIT extends ESRestTestCase {
private static final String PLUGIN_NAME = "elasticsearch";
@@ -85,7 +87,9 @@ public void testInstantiateSinkRawSpanDefault() throws IOException {
if (isODFE()) {
// Check managed index
- assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index));
+ await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> {
+ assertEquals(IndexConstants.RAW_ISM_POLICY, getIndexPolicyId(index)); }
+ );
}
// roll over initial index
From afd2acae35e32d93a56c68e92b85f4fbe704c6f5 Mon Sep 17 00:00:00 2001
From: Qi Chen
Date: Thu, 29 Apr 2021 17:39:46 -0500
Subject: [PATCH 003/192] MAINT: update cfn template (#548)
* MAINT: update cfn template
Signed-off-by: qchea
---
.../ec2/data-prepper-ec2-deployment-cfn.yaml | 121 ++++++++++++------
1 file changed, 83 insertions(+), 38 deletions(-)
diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml
index 58bee69ad8..b69c750493 100644
--- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml
+++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml
@@ -16,6 +16,14 @@ Parameters:
AWSElasticsearchSubnetId:
Description: The subnet ID of the AWS Elasticsearch Service domain (Leave blank if the domain is not in a VPC)
Type: String
+ Username:
+ Description: The username of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role)
+ Type: String
+ Default: ""
+ Password:
+ Description: The password of the AWS Elasticsearch Service domain (Leave blank if the domain is configured with IAM role)
+ Type: String
+ Default: ""
DataPrepperVersion:
Description: Version of Data Prepper to download and run
Type: String
@@ -24,8 +32,9 @@ Parameters:
ConstraintDescription: must be a valid release number
IAMRole:
Description: Pre-existing IAM Role to associate with the EC2 instance, to be used for authentication when calling Elasticsearch
+ (Leave blank if the domain is configured with HTTP basic authentication in Fine-grained access control)
Type: String
- AllowedPattern: "[a-zA-Z0-9+=,\\.@\\-_]+"
+ AllowedPattern: "^$|[a-zA-Z0-9+=,\\.@\\-_]+"
Default: DataPrepperRole
ConstraintDescription: must be a valid IAM role name
InstanceType:
@@ -50,9 +59,39 @@ Parameters:
Default: 0.0.0.0/0
AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
+Metadata:
+ AWS::CloudFormation::Interface:
+ ParameterGroups:
+ - Label:
+ default: Common
+ Parameters:
+ - IAMRole
+ - Label:
+ default: Amazon Elasticsearch Service Domain
+ Parameters:
+ - AWSElasticsearchEndpoint
+ - AWSElasticsearchRegion
+ - AWSElasticsearchSubnetId
+ - Username
+ - Password
+ - Label:
+ default: Data-Prepper Configuration
+ Parameters:
+ - DataPrepperVersion
+ - InstanceType
+ - KeyName
+ - LatestAmi
+ - SSHLocation
Conditions:
DomainIsPublic: !Equals [!Ref AWSElasticsearchSubnetId, ""]
DomainIsInVPC: !Not [Condition: DomainIsPublic]
+ NoMasterUser: !And
+ - !Equals
+ - !Ref Username
+ - ""
+ - !Equals
+ - !Ref Password
+ - ""
Resources:
EC2Instance:
Type: AWS::EC2::Instance
@@ -70,43 +109,49 @@ Resources:
owner: root
group: root
"/etc/data-prepper/pipelines.yaml":
- content: !Sub |
- entry-pipeline:
- delay: "100"
- source:
- otel_trace_source:
- ssl: false
- health_check_service: true
- sink:
- - pipeline:
- name: "raw-pipeline"
- - pipeline:
- name: "service-map-pipeline"
- raw-pipeline:
- source:
- pipeline:
- name: "entry-pipeline"
- prepper:
- - otel_trace_raw_prepper:
- sink:
- - elasticsearch:
- hosts: [ "${AWSElasticsearchEndpoint}" ]
- aws_sigv4: true
- aws_region: "${AWSElasticsearchRegion}"
- trace_analytics_raw: true
- service-map-pipeline:
- delay: "100"
- source:
- pipeline:
- name: "entry-pipeline"
- prepper:
- - service_map_stateful:
- sink:
- - elasticsearch:
- hosts: [ "${AWSElasticsearchEndpoint}" ]
- aws_sigv4: true
- aws_region: "${AWSElasticsearchRegion}"
- trace_analytics_service_map: true
+ content: !Sub
+ - |
+ entry-pipeline:
+ delay: "100"
+ source:
+ otel_trace_source:
+ ssl: false
+ health_check_service: true
+ sink:
+ - pipeline:
+ name: "raw-pipeline"
+ - pipeline:
+ name: "service-map-pipeline"
+ raw-pipeline:
+ source:
+ pipeline:
+ name: "entry-pipeline"
+ prepper:
+ - otel_trace_raw_prepper:
+ sink:
+ - elasticsearch: ${elasticsearchConfig}
+ service-map-pipeline:
+ delay: "100"
+ source:
+ pipeline:
+ name: "entry-pipeline"
+ prepper:
+ - service_map_stateful:
+ sink:
+ - elasticsearch: ${elasticsearchConfig}
+ - elasticsearchConfig: !If
+ - NoMasterUser
+ - !Sub "\n
+ \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n
+ \ aws_sigv4: true\n
+ \ aws_region: \"${AWSElasticsearchRegion}\"\n
+ \ trace_analytics_service_map: true"
+ - !Sub "\n
+ \ hosts: [ \"${AWSElasticsearchEndpoint}\" ]\n
+ \ aws_sigv4: false\n
+ \ username: \"${Username}\"\n
+ \ password: \"${Password}\"\n
+ \ trace_analytics_service_map: true"
mode: "000400"
owner: root
group: root
From f35ea6d0e34efe59c1183e6a1dd98c137190383a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 May 2021 07:30:57 +0000
Subject: [PATCH 004/192] Bump armeria-grpc in
/data-prepper-plugins/otel-trace-source
Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/otel-trace-source/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle
index 94b23a97b7..e8fc2b3afb 100644
--- a/data-prepper-plugins/otel-trace-source/build.gradle
+++ b/data-prepper-plugins/otel-trace-source/build.gradle
@@ -10,7 +10,7 @@ dependencies {
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
implementation 'com.google.protobuf:protobuf-java-util:3.15.8'
implementation "com.linecorp.armeria:armeria:1.6.0"
- implementation "com.linecorp.armeria:armeria-grpc:1.6.0"
+ implementation "com.linecorp.armeria:armeria-grpc:1.7.2"
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3"
testImplementation 'org.assertj:assertj-core:3.19.0'
From e3c43ac8fa99ae42bdda4f844cde92bf062792e2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 May 2021 07:32:53 +0000
Subject: [PATCH 005/192] Bump armeria-grpc in
/data-prepper-plugins/otel-trace-raw-prepper
Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
index 6f883dfc7f..3bb286d074 100644
--- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
+++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
@@ -10,7 +10,7 @@ dependencies {
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
implementation 'com.google.protobuf:protobuf-java-util:3.15.8'
implementation "com.linecorp.armeria:armeria:1.6.0"
- implementation "com.linecorp.armeria:armeria-grpc:1.6.0"
+ implementation "com.linecorp.armeria:armeria-grpc:1.7.2"
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3"
implementation group: 'com.google.guava', name: 'guava', version: '30.1.1-jre'
From 2c657a334301e0dd7155f9b96d154b966588596b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 May 2021 07:33:58 +0000
Subject: [PATCH 006/192] Bump armeria from 1.6.0 to 1.7.2 in
/data-prepper-plugins/peer-forwarder
Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/peer-forwarder/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle
index a8143de01c..df36b8e267 100644
--- a/data-prepper-plugins/peer-forwarder/build.gradle
+++ b/data-prepper-plugins/peer-forwarder/build.gradle
@@ -13,7 +13,7 @@ dependencies {
compile project(':data-prepper-api')
testCompile project(':data-prepper-api').sourceSets.test.output
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
- implementation "com.linecorp.armeria:armeria:1.6.0"
+ implementation "com.linecorp.armeria:armeria:1.7.2"
implementation "com.linecorp.armeria:armeria-grpc:1.6.0"
implementation "commons-validator:commons-validator:1.7"
testImplementation "junit:junit:4.13.2"
From 2048421cd844f55e910858caa80b1cb941137a4f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 3 May 2021 07:34:03 +0000
Subject: [PATCH 007/192] Bump aws-java-sdk-core in
/data-prepper-plugins/elasticsearch
Bumps [aws-java-sdk-core](https://github.com/aws/aws-sdk-java) from 1.11.1001 to 1.11.1009.
- [Release notes](https://github.com/aws/aws-sdk-java/releases)
- [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.1001...1.11.1009)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/elasticsearch/build.gradle | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/data-prepper-plugins/elasticsearch/build.gradle b/data-prepper-plugins/elasticsearch/build.gradle
index 525c5f29cc..d6e5e0faed 100644
--- a/data-prepper-plugins/elasticsearch/build.gradle
+++ b/data-prepper-plugins/elasticsearch/build.gradle
@@ -35,7 +35,7 @@ dependencies {
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3"
implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1'
- implementation "com.amazonaws:aws-java-sdk-core:1.11.1001"
+ implementation "com.amazonaws:aws-java-sdk-core:1.11.1009"
implementation "com.github.awslabs:aws-request-signing-apache-interceptor:b3772780da"
implementation "io.micrometer:micrometer-core:1.6.5"
testImplementation("junit:junit:4.13.2") {
@@ -52,7 +52,7 @@ compileJava.options.warnings = false
// Resolve dependency conflict between ES sink and main project
configurations.all {
resolutionStrategy {
- force 'com.amazonaws:aws-java-sdk-core:1.11.1000'
+ force 'com.amazonaws:aws-java-sdk-core:1.11.1009'
force 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3'
force 'com.fasterxml.jackson.core:jackson-annotations:2.12.3'
force 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
From 10ce391eb66bb26506e5fa9510cdd865a9b36827 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 May 2021 18:58:19 +0000
Subject: [PATCH 008/192] Bump armeria in
/data-prepper-plugins/otel-trace-raw-prepper
Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/otel-trace-raw-prepper/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
index 3bb286d074..801e15abcf 100644
--- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
+++ b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle
@@ -9,7 +9,7 @@ dependencies {
testCompile project(':data-prepper-api').sourceSets.test.output
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
implementation 'com.google.protobuf:protobuf-java-util:3.15.8'
- implementation "com.linecorp.armeria:armeria:1.6.0"
+ implementation "com.linecorp.armeria:armeria:1.7.2"
implementation "com.linecorp.armeria:armeria-grpc:1.7.2"
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3"
From ecb8a176cf7399bfdce145b67cdd80f19ae0fc42 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 May 2021 18:58:22 +0000
Subject: [PATCH 009/192] Bump armeria-grpc in
/data-prepper-plugins/peer-forwarder
Bumps [armeria-grpc](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/peer-forwarder/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/peer-forwarder/build.gradle b/data-prepper-plugins/peer-forwarder/build.gradle
index df36b8e267..59c78f335b 100644
--- a/data-prepper-plugins/peer-forwarder/build.gradle
+++ b/data-prepper-plugins/peer-forwarder/build.gradle
@@ -14,7 +14,7 @@ dependencies {
testCompile project(':data-prepper-api').sourceSets.test.output
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
implementation "com.linecorp.armeria:armeria:1.7.2"
- implementation "com.linecorp.armeria:armeria-grpc:1.6.0"
+ implementation "com.linecorp.armeria:armeria-grpc:1.7.2"
implementation "commons-validator:commons-validator:1.7"
testImplementation "junit:junit:4.13.2"
testImplementation "org.mockito:mockito-inline:3.9.0"
From 51e4362323cd25a4ea51eb9e35f34ece93b84189 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 May 2021 18:58:40 +0000
Subject: [PATCH 010/192] Bump armeria in
/data-prepper-plugins/otel-trace-source
Bumps [armeria](https://github.com/line/armeria) from 1.6.0 to 1.7.2.
- [Release notes](https://github.com/line/armeria/releases)
- [Changelog](https://github.com/line/armeria/blob/master/.post-release-msg)
- [Commits](https://github.com/line/armeria/compare/armeria-1.6.0...armeria-1.7.2)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/otel-trace-source/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle
index e8fc2b3afb..9e0e645cbe 100644
--- a/data-prepper-plugins/otel-trace-source/build.gradle
+++ b/data-prepper-plugins/otel-trace-source/build.gradle
@@ -9,7 +9,7 @@ dependencies {
testCompile project(':data-prepper-api').sourceSets.test.output
implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetry_proto}"
implementation 'com.google.protobuf:protobuf-java-util:3.15.8'
- implementation "com.linecorp.armeria:armeria:1.6.0"
+ implementation "com.linecorp.armeria:armeria:1.7.2"
implementation "com.linecorp.armeria:armeria-grpc:1.7.2"
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.3"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.3"
From 701c2ea0f3ea48fb2438d32273dd557bb9a6923c Mon Sep 17 00:00:00 2001
From: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
Date: Tue, 4 May 2021 14:01:04 -0500
Subject: [PATCH 011/192] Fixed minor typo in github integ test name
---
.../opensearch-sink-odfe-since-1_13_0-integration-tests.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
index 77dfa83ab5..eb08451867 100644
--- a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
+++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -1,7 +1,7 @@
# This workflow will build a Java project with Gradle
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
-name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0
+name: Data Prepper elasticsearch sink integration tests with ODFE >= 1.13.0
on: [push, pull_request, workflow_dispatch]
@@ -31,4 +31,4 @@ jobs:
- name: Run ODFE tests
run: |
./gradlew :data-prepper-plugins:elasticsearch:test --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
- ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
\ No newline at end of file
+ ./gradlew :data-prepper-plugins:elasticsearch:integTest --tests "com.amazon.dataprepper.plugins.sink.elasticsearch.ElasticsearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
From 02502f7cc0131f444a19cf46c42bec81b6b1a17f Mon Sep 17 00:00:00 2001
From: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
Date: Tue, 4 May 2021 14:27:32 -0500
Subject: [PATCH 012/192] Updated workflows to potentially stop duplicate
actions from firing. (#563)
* Updated workflows to potentially stop duplicate actions from firing.
Signed-off-by: Jeff Wright <74204404+wrijeff@users.noreply.github.com>
---
.../workflows/data-prepper-trace-analytics-e2e-tests.yml | 1 -
.github/workflows/gradle.yml | 6 +++++-
...opensearch-sink-odfe-before-1_13_0-integration-tests.yml | 6 +++++-
.../opensearch-sink-odfe-since-1_13_0-integration-tests.yml | 6 +++++-
4 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml
index bdfab23884..ea39bdc594 100644
--- a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml
+++ b/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml
@@ -7,7 +7,6 @@ on:
push:
branches: [ main ]
pull_request:
- branches: [ main ]
workflow_dispatch:
jobs:
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
index baab044d85..3dad348eb5 100644
--- a/.github/workflows/gradle.yml
+++ b/.github/workflows/gradle.yml
@@ -3,7 +3,11 @@
name: Data Prepper Java CI with Gradle
-on: [push, pull_request, workflow_dispatch]
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ workflow_dispatch:
jobs:
build:
diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
index d046b7908d..5295214c2d 100644
--- a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
+++ b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
@@ -3,7 +3,11 @@
name: Data Prepper elasticsearch sink integration tests with ODFE < 1.13.0
-on: [push, pull_request, workflow_dispatch]
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ workflow_dispatch:
jobs:
integration_tests:
diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
index eb08451867..623672eb2c 100644
--- a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
+++ b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -3,7 +3,11 @@
name: Data Prepper elasticsearch sink integration tests with ODFE >= 1.13.0
-on: [push, pull_request, workflow_dispatch]
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ workflow_dispatch:
jobs:
integration_tests:
From 78ad931102e92582edf4fe8f89e4b9296bf5aa2b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 May 2021 19:28:09 +0000
Subject: [PATCH 013/192] Bump kotlin-stdlib in
/data-prepper-plugins/mapdb-prepper-state
Bumps [kotlin-stdlib](https://github.com/JetBrains/kotlin) from 1.4.32 to 1.5.0.
- [Release notes](https://github.com/JetBrains/kotlin/releases)
- [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md)
- [Commits](https://github.com/JetBrains/kotlin/commits)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle
index 120c3553c2..b469f33676 100644
--- a/data-prepper-plugins/mapdb-prepper-state/build.gradle
+++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle
@@ -16,7 +16,7 @@ dependencies {
compile project(':data-prepper-api')
compile project(':data-prepper-plugins:common')
implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8'
- implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.4.32'
+ implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0'
implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.4.32'
testCompile project(':data-prepper-plugins:common').sourceSets.test.output
From 211c7a4f2290ed94c1c18cb89d9251adfee3b833 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 4 May 2021 19:30:23 +0000
Subject: [PATCH 014/192] Bump kotlin-stdlib-common in
/data-prepper-plugins/mapdb-prepper-state
Bumps [kotlin-stdlib-common](https://github.com/JetBrains/kotlin) from 1.4.32 to 1.5.0.
- [Release notes](https://github.com/JetBrains/kotlin/releases)
- [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md)
- [Commits](https://github.com/JetBrains/kotlin/commits)
Signed-off-by: dependabot[bot]
---
data-prepper-plugins/mapdb-prepper-state/build.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-prepper-plugins/mapdb-prepper-state/build.gradle b/data-prepper-plugins/mapdb-prepper-state/build.gradle
index b469f33676..a2184fbad0 100644
--- a/data-prepper-plugins/mapdb-prepper-state/build.gradle
+++ b/data-prepper-plugins/mapdb-prepper-state/build.gradle
@@ -17,7 +17,7 @@ dependencies {
compile project(':data-prepper-plugins:common')
implementation group: 'org.mapdb', name: 'mapdb', version: '3.0.8'
implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: '1.5.0'
- implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.4.32'
+ implementation group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-common', version: '1.5.0'
testCompile project(':data-prepper-plugins:common').sourceSets.test.output
testImplementation "org.hamcrest:hamcrest:2.2"
From ff834afe08826d8b63c424226725a9de1325a1b3 Mon Sep 17 00:00:00 2001
From: Qi Chen
Date: Wed, 5 May 2021 14:31:22 -0500
Subject: [PATCH 015/192] MAINT: restore traceGroup backward compatibility
(#564)
* MAINT: restore traceGroup backward compatibility
Signed-off-by: qchea
---
.../main/resources/otel-v1-apm-span-index-template.json | 8 ++++----
.../plugins/prepper/oteltracegroup/model/TraceGroup.java | 8 ++++----
.../src/test/resources/raw-span-complete-1.json | 8 ++++----
.../src/test/resources/raw-span-complete-2.json | 8 ++++----
.../test/resources/raw-span-missing-trace-group-1.json | 8 ++++----
.../test/resources/raw-span-missing-trace-group-2.json | 8 ++++----
.../plugins/prepper/oteltrace/model/RawSpan.java | 2 +-
.../plugins/prepper/oteltrace/model/TraceGroup.java | 4 ++++
.../prepper/oteltrace/OTelTraceRawPrepperTest.java | 8 ++++----
9 files changed, 33 insertions(+), 29 deletions(-)
diff --git a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json
index 252688d207..25df9ff8e7 100644
--- a/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json
+++ b/data-prepper-plugins/elasticsearch/src/main/resources/otel-v1-apm-span-index-template.json
@@ -40,11 +40,11 @@
"type": "keyword"
},
"traceGroup": {
+ "ignore_above": 1024,
+ "type": "keyword"
+ },
+ "traceGroupFields": {
"properties": {
- "name": {
- "ignore_above": 1024,
- "type": "keyword"
- },
"endTime": {
"type": "date_nanos"
},
diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java
index 1af2a9bb93..8695e19ee8 100644
--- a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java
+++ b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java
@@ -5,10 +5,10 @@
import java.util.Objects;
public class TraceGroup {
- public static final String TRACE_GROUP_NAME_FIELD = "traceGroup.name";
- public static final String TRACE_GROUP_END_TIME_FIELD = "traceGroup.endTime";
- public static final String TRACE_GROUP_STATUS_CODE_FIELD = "traceGroup.statusCode";
- public static final String TRACE_GROUP_DURATION_IN_NANOS_FIELD = "traceGroup.durationInNanos";
+ public static final String TRACE_GROUP_NAME_FIELD = "traceGroup";
+ public static final String TRACE_GROUP_END_TIME_FIELD = "traceGroupFields.endTime";
+ public static final String TRACE_GROUP_STATUS_CODE_FIELD = "traceGroupFields.statusCode";
+ public static final String TRACE_GROUP_DURATION_IN_NANOS_FIELD = "traceGroupFields.durationInNanos";
@JsonProperty(TRACE_GROUP_NAME_FIELD)
private final String name;
diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json
index a03acb51ff..1226514180 100644
--- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json
+++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json
@@ -14,10 +14,10 @@
"droppedAttributesCount": 0,
"droppedEventsCount": 0,
"droppedLinksCount": 0,
- "traceGroup.name": "/logs",
- "traceGroup.endTime": "2020-08-20T05:40:46.089556800Z",
- "traceGroup.statusCode": 1,
- "traceGroup.durationInNanos": 48545200,
+ "traceGroup": "/logs",
+ "traceGroupFields.endTime": "2020-08-20T05:40:46.089556800Z",
+ "traceGroupFields.statusCode": 1,
+ "traceGroupFields.durationInNanos": 48545200,
"span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m",
"span.attributes.http@method": "PUT",
"resource.attributes.telemetry@sdk@name": "opentelemetry",
diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json
index a32d96e565..7c119322c4 100644
--- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json
+++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json
@@ -14,10 +14,10 @@
"droppedAttributesCount": 0,
"droppedEventsCount": 0,
"droppedLinksCount": 0,
- "traceGroup.name": "/logs",
- "traceGroup.endTime": "2020-08-20T05:40:43.217170200Z",
- "traceGroup.statusCode": 1,
- "traceGroup.durationInNanos": 49160000,
+ "traceGroup": "/logs",
+ "traceGroupFields.endTime": "2020-08-20T05:40:43.217170200Z",
+ "traceGroupFields.statusCode": 1,
+ "traceGroupFields.durationInNanos": 49160000,
"span.attributes.http@url": "http://0.0.0.0:8087/logs",
"span.attributes.net@peer@ip": "172.29.0.1",
"resource.attributes.telemetry@sdk@language": "java",
diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json
index 6fe5304b3b..239508132a 100644
--- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json
+++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json
@@ -14,10 +14,10 @@
"droppedAttributesCount": 0,
"droppedEventsCount": 0,
"droppedLinksCount": 0,
- "traceGroup.name": null,
- "traceGroup.endTime": null,
- "traceGroup.statusCode": null,
- "traceGroup.durationInNanos": null,
+ "traceGroup": null,
+ "traceGroupFields.endTime": null,
+ "traceGroupFields.statusCode": null,
+ "traceGroupFields.durationInNanos": null,
"span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m",
"span.attributes.http@method": "PUT",
"resource.attributes.telemetry@sdk@name": "opentelemetry",
diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json
index 4348d7340f..cb4ae8c110 100644
--- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json
+++ b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json
@@ -14,10 +14,10 @@
"droppedAttributesCount": 0,
"droppedEventsCount": 0,
"droppedLinksCount": 0,
- "traceGroup.name": null,
- "traceGroup.endTime": null,
- "traceGroup.statusCode": null,
- "traceGroup.durationInNanos": null,
+ "traceGroup": null,
+ "traceGroupFields.endTime": null,
+ "traceGroupFields.statusCode": null,
+ "traceGroupFields.durationInNanos": null,
"span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m",
"span.attributes.http@method": "PUT",
"resource.attributes.telemetry@sdk@name": "opentelemetry",
diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java
index 29d399c410..25d03dd19b 100644
--- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java
+++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java
@@ -185,7 +185,7 @@ public Map getAttributes() {
return attributes;
}
- @JsonUnwrapped(prefix="traceGroup.")
+ @JsonUnwrapped
public TraceGroup getTraceGroup() {
return traceGroup;
}
diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java
index 3059a676e8..d183c51fe5 100644
--- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java
+++ b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java
@@ -6,9 +6,13 @@
import java.util.Objects;
public class TraceGroup {
+ @JsonProperty("traceGroup")
private final String name;
+ @JsonProperty("traceGroupFields.endTime")
private final String endTime;
+ @JsonProperty("traceGroupFields.statusCode")
private final Integer statusCode;
+ @JsonProperty("traceGroupFields.durationInNanos")
private final Long durationInNanos;
public String getName() {
diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java
index 9f85f59010..cf4b846e8e 100644
--- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java
+++ b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java
@@ -236,10 +236,10 @@ private int getMissingTraceGroupFieldsSpanCount(List> records) th
for (Record record: records) {
final String spanJson = record.getData();
Map spanMap = OBJECT_MAPPER.readValue(spanJson, new TypeReference