From 7bc0d677fbffc934ec08a86c5bc874e968833c73 Mon Sep 17 00:00:00 2001 From: Qi Chen <19492223+chenqi0805@users.noreply.github.com> Date: Fri, 2 Sep 2022 09:33:03 -0500 Subject: [PATCH] Maintenance: remove support for non event data model (#1689) * MAINT: remove support for OTLP data transport Signed-off-by: Qi Chen <19492223+chenqi0805@users.noreply.github.com> --- ...ytics-raw-span-compatibility-e2e-tests.yml | 7 +- ...per-trace-analytics-raw-span-e2e-tests.yml | 7 +- ...-trace-analytics-service-map-e2e-tests.yml | 7 +- .../sink/opensearch/OpenSearchSinkIT.java | 85 ++--- .../sink/opensearch/OpenSearchSink.java | 33 +- .../otel-trace-group-prepper/README.md | 67 ---- .../otel-trace-group-prepper/build.gradle | 22 -- .../oteltracegroup/OTelTraceGroupPrepper.java | 200 ---------- .../OTelTraceGroupPrepperConfig.java | 34 -- .../oteltracegroup/model/TraceGroup.java | 68 ---- .../OTelTraceGroupPrepperConfigTests.java | 38 -- .../OTelTraceGroupPrepperTests.java | 300 --------------- .../org.mockito.plugins.MockMaker | 3 - .../test/resources/raw-span-complete-1.json | 30 -- .../test/resources/raw-span-complete-2.json | 38 -- .../raw-span-missing-trace-group-1.json | 30 -- .../raw-span-missing-trace-group-2.json | 30 -- .../otel-trace-group-processor/README.md | 2 +- .../otel-trace-raw-prepper/README.md | 27 -- .../otel-trace-raw-prepper/build.gradle | 35 -- .../oteltrace/OTelTraceRawPrepper.java | 299 --------------- .../oteltrace/OtelTraceRawPrepperConfig.java | 13 - .../oteltrace/model/OTelProtoHelper.java | 161 -------- .../prepper/oteltrace/model/RawEvent.java | 49 --- .../prepper/oteltrace/model/RawLink.java | 68 ---- .../prepper/oteltrace/model/RawSpan.java | 200 ---------- .../oteltrace/model/RawSpanBuilder.java | 164 -------- .../prepper/oteltrace/model/RawSpanSet.java | 33 -- .../prepper/oteltrace/model/TraceGroup.java | 100 ----- .../oteltrace/OTelTraceRawPrepperTest.java | 253 ------------- .../oteltrace/model/OTelProtoHelperTest.java | 199 ---------- .../oteltrace/model/RawBuilderTest.java | 91 ----- .../prepper/oteltrace/model/RawEventTest.java | 55 --- .../prepper/oteltrace/model/RawLinkTest.java | 51 --- .../prepper/oteltrace/model/TestUtils.java | 20 - .../sample-request-one-full-trace-group.json | 207 ---------- ...-request-one-trace-group-missing-root.json | 112 ------ .../sample-request-two-full-trace-group.json | 353 ------------------ ...request-two-trace-group-interleaved-1.json | 207 ---------- ...request-two-trace-group-interleaved-2.json | 207 ---------- ...request-two-trace-group-missing-roots.json | 171 --------- .../otel-trace-source/README.md | 3 - .../oteltrace/OTelTraceGrpcService.java | 34 +- .../source/oteltrace/OTelTraceSource.java | 1 - .../oteltrace/OTelTraceSourceConfig.java | 9 - .../plugins/source/oteltrace/RecordType.java | 40 -- .../oteltrace/OTelTraceGrpcServiceTest.java | 58 +-- .../source/oteltrace/OTelTraceSourceTest.java | 33 +- .../oteltrace/OtelTraceSourceConfigTests.java | 49 --- .../prepper/peerforwarder/PeerForwarder.java | 82 +--- .../peerforwarder/PeerForwarderUtils.java | 32 -- .../peerforwarder/PeerForwarderTest.java | 266 +------------ .../prepper/ServiceMapStatefulPrepper.java | 53 +-- .../ServiceMapStatefulPrepperTest.java | 166 -------- .../plugins/prepper/ServiceMapTestUtils.java | 60 +-- .../ec2/data-prepper-ec2-deployment-cfn.yaml | 2 +- deployment-template/k8s/data-prepper-k8s.yaml | 2 +- docs/configuration.md | 2 +- docs/trace_analytics.md | 17 +- e2e-test/trace/README.md | 51 +-- e2e-test/trace/build.gradle | 60 +-- .../raw-span-e2e-pipeline-event-type.yml | 25 -- ... raw-span-e2e-pipeline-peer-forwarder.yml} | 1 - .../resources/raw-span-e2e-pipeline.yml | 6 +- .../service-map-e2e-pipeline-event-type.yml | 26 -- .../resources/service-map-e2e-pipeline.yml | 4 +- examples/config/example-pipelines.yaml | 6 +- .../data-prepper-emf-demo-cfn.yaml | 4 +- .../pipelines-raw-trace-stdout.yaml | 4 +- examples/dev/k8s/data-prepper.yaml | 8 +- .../dev/trace-analytics-sample-app/Dockerfile | 2 +- .../resources/pipelines.yaml | 8 +- examples/trace_analytics.yml | 6 +- examples/trace_analytics_no_ssl.yml | 6 +- .../data-prepper/config/pipelines.yaml | 6 +- research/build.gradle | 5 - research/zipkin-opensearch-to-otel/README.md | 22 -- .../zipkin-opensearch-to-otel/build.gradle | 41 -- .../research/zipkin/OpenSearchReader.java | 79 ---- .../zipkin/ZipkinOpenSearchToOtel.java | 98 ----- .../zipkin/ZipkinOpenSearchToOtelPrepper.java | 170 --------- settings.gradle | 4 - 82 files changed, 157 insertions(+), 5470 deletions(-) delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/README.md delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/build.gradle delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepper.java delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfig.java delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfigTests.java delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperTests.java delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json delete mode 100644 data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/README.md delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/build.gradle delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEvent.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLink.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanBuilder.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanSet.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEventTest.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLinkTest.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TestUtils.java delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-full-trace-group.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-trace-group-missing-root.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-full-trace-group.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-1.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-2.json delete mode 100644 data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-missing-roots.json delete mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/RecordType.java delete mode 100644 e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-event-type.yml rename e2e-test/trace/src/integrationTest/resources/{raw-span-e2e-pipeline-peer-forwarder-event-type.yml => raw-span-e2e-pipeline-peer-forwarder.yml} (95%) delete mode 100644 e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline-event-type.yml delete mode 100644 research/build.gradle delete mode 100644 research/zipkin-opensearch-to-otel/README.md delete mode 100644 research/zipkin-opensearch-to-otel/build.gradle delete mode 100644 research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/OpenSearchReader.java delete mode 100644 research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtel.java delete mode 100644 research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtelPrepper.java diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml index 62998d52d7..8150d7beaf 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-compatibility-e2e-tests.yml @@ -24,8 +24,5 @@ jobs: java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 - # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - - name: Run raw-span OTLP record type latest release compatibility end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest - - name: Run raw-span Event record type latest release compatibility end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest + - name: Run raw-span latest release compatibility end-to-end tests with Gradle + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanLatestReleaseCompatibilityEndToEndTest diff --git a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml index fb3bc1b716..d945141f22 100644 --- a/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-raw-span-e2e-tests.yml @@ -24,8 +24,5 @@ jobs: java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 - # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - - name: Run raw-span OTLP record type end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPEndToEndTest - - name: Run raw-span OTLP and Event end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest \ No newline at end of file + - name: Run raw-span end-to-end tests with Gradle + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:rawSpanEndToEndTest \ No newline at end of file diff --git a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml index 3e558f0eaf..2523751385 100644 --- a/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml +++ b/.github/workflows/data-prepper-trace-analytics-service-map-e2e-tests.yml @@ -24,8 +24,5 @@ jobs: java-version: 11 - name: Checkout Data-Prepper uses: actions/checkout@v2 - # TODO: Event record type only in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) - - name: Run service-map OTLP record type end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPEndToEndTest - - name: Run service-map OTLP and Event end-to-end tests with Gradle - run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest \ No newline at end of file + - name: Run service-map end-to-end tests with Gradle + run: ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:trace:serviceMapEndToEndTest \ No newline at end of file diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 8cc0241dbf..b3602bc072 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -29,11 +29,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; @@ -60,9 +55,7 @@ import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createContentParser; import static com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchIntegrationHelper.createOpenSearchClient; @@ -158,16 +151,15 @@ public void testInstantiateSinkRawSpanReservedAliasAlreadyUsedAsIndex() throws I RuntimeException.class, () -> new OpenSearchSink(pluginSetting)); } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) - public void testOutputRawSpanDefault(final Function stringToRecord) throws IOException, InterruptedException { + @Test + public void testOutputRawSpanDefault() throws IOException, InterruptedException { final String testDoc1 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1); final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_2); final ObjectMapper mapper = new ObjectMapper(); @SuppressWarnings("unchecked") final Map expData1 = mapper.readValue(testDoc1, Map.class); @SuppressWarnings("unchecked") final Map expData2 = mapper.readValue(testDoc2, Map.class); - final List> testRecords = Arrays.asList(stringToRecord.apply(testDoc1), stringToRecord.apply(testDoc2)); + final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); final OpenSearchSink sink = new OpenSearchSink(pluginSetting); sink.output(testRecords); @@ -223,15 +215,15 @@ public void testOutputRawSpanDefault(final Function stringToReco MatcherAssert.assertThat(bulkRequestSizeBytesMetrics.get(2).getValue(), closeTo(2058.0, 0)); } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) - public void testOutputRawSpanWithDLQ(final Function stringToRecord) throws IOException, InterruptedException { + @Test + public void testOutputRawSpanWithDLQ() throws IOException, InterruptedException { // TODO: write test case final String testDoc1 = readDocFromFile("raw-span-error.json"); final String testDoc2 = readDocFromFile(DEFAULT_RAW_SPAN_FILE_1); final ObjectMapper mapper = new ObjectMapper(); @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc2, Map.class); - final List> testRecords = Arrays.asList(stringToRecord.apply(testDoc1), stringToRecord.apply(testDoc2)); + + final List> testRecords = Arrays.asList(jsonStringToRecord(testDoc1), jsonStringToRecord(testDoc2)); final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); // generate temporary directory for dlq file final File tempDirectory = Files.createTempDirectory("").toFile(); @@ -299,14 +291,13 @@ public void testInstantiateSinkServiceMapDefault() throws IOException { } } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) - public void testOutputServiceMapDefault(final Function stringToRecord) throws IOException, InterruptedException { + @Test + public void testOutputServiceMapDefault() throws IOException, InterruptedException { final String testDoc = readDocFromFile(DEFAULT_SERVICE_MAP_FILE); final ObjectMapper mapper = new ObjectMapper(); @SuppressWarnings("unchecked") final Map expData = mapper.readValue(testDoc, Map.class); - final List> testRecords = Collections.singletonList(stringToRecord.apply(testDoc)); + final List> testRecords = Collections.singletonList(jsonStringToRecord(testDoc)); final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_SERVICE_MAP.getValue(), null, null); OpenSearchSink sink = new OpenSearchSink(pluginSetting); sink.output(testRecords); @@ -463,15 +454,14 @@ public void testInstantiateSinkDoesNotOverwriteNewerIndexTemplates() throws IOEx } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) - public void testOutputCustomIndex(final Function stringToRecord) throws IOException, InterruptedException { + @Test + public void testOutputCustomIndex() throws IOException, InterruptedException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; final String testId = "foo"; - final List> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId))); + final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); final OpenSearchSink sink = new OpenSearchSink(pluginSetting); @@ -490,15 +480,14 @@ public void testOutputCustomIndex(final Function stringToRecord) Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0); } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) - public void testBulkActionCreate(final Function stringToRecord) throws IOException, InterruptedException { + @Test + public void testBulkActionCreate() throws IOException, InterruptedException { final String testIndexAlias = "test-alias"; final String testTemplateFile = Objects.requireNonNull( getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile(); final String testIdField = "someId"; final String testId = "foo"; - final List> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId))); + final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTION, BulkAction.CREATE.toString()); @@ -526,7 +515,7 @@ public void testEventOutput() throws IOException, InterruptedException { .withEventType("event") .build(); - final List> testRecords = Collections.singletonList(new Record<>(testEvent)); + final List> testRecords = Collections.singletonList(new Record<>(testEvent)); final PluginSetting pluginSetting = generatePluginSetting(IndexType.TRACE_ANALYTICS_RAW.getValue(), null, null); final OpenSearchSink sink = new OpenSearchSink(pluginSetting); @@ -543,10 +532,9 @@ public void testEventOutput() throws IOException, InterruptedException { sink.shutdown(); } - @ParameterizedTest - @ArgumentsSource(MultipleRecordTypeArgumentProvider.class) + @Test @Timeout(value = 1, unit = TimeUnit.MINUTES) - public void testOutputManagementDisabled(final Function stringToRecord) throws IOException, InterruptedException { + public void testOutputManagementDisabled() throws IOException, InterruptedException { final String testIndexAlias = "test-" + UUID.randomUUID(); final String roleName = UUID.randomUUID().toString(); final String username = UUID.randomUUID().toString(); @@ -558,7 +546,7 @@ public void testOutputManagementDisabled(final Function stringTo final String testIdField = "someId"; final String testId = "foo"; - final List> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId))); + final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); @@ -722,33 +710,12 @@ private void wipeAllOpenSearchIndices() throws IOException { }); } - /** - * Provides a function for mapping a String to a Record to allow the tests to run - * against both String and Event models. - */ - static class MultipleRecordTypeArgumentProvider implements ArgumentsProvider { - @Override - public Stream provideArguments(final ExtensionContext context) { - final ObjectMapper objectMapper = new ObjectMapper(); - final Function stringModel = jsonString -> { - try { - // Normalize the JSON string. - return new Record(objectMapper.writeValueAsString(objectMapper.readValue(jsonString, Map.class))); - } catch (final JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - final Function eventModel = jsonString -> { - try { - return new Record(JacksonEvent.builder().withEventType(EventType.TRACE.toString()).withData(objectMapper.readValue(jsonString, Map.class)).build()); - } catch (final JsonProcessingException e) { - throw new RuntimeException(e); - } - }; - return Stream.of( - Arguments.of(stringModel), - Arguments.of(eventModel) - ); + private Record jsonStringToRecord(final String jsonString) { + final ObjectMapper objectMapper = new ObjectMapper(); + try { + return new Record(JacksonEvent.builder().withEventType(EventType.TRACE.toString()).withData(objectMapper.readValue(jsonString, Map.class)).build()); + } catch (final JsonProcessingException e) { + throw new RuntimeException(e); } } diff --git a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index eba0241567..7400c62798 100644 --- a/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -34,11 +34,6 @@ import org.opensearch.client.transport.OpenSearchTransport; import org.opensearch.client.transport.rest_client.RestClientTransport; import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; -import org.opensearch.common.xcontent.XContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +48,7 @@ import java.util.function.Supplier; @DataPrepperPlugin(name = "opensearch", pluginType = Sink.class) -public class OpenSearchSink extends AbstractSink> { +public class OpenSearchSink extends AbstractSink> { public static final String BULKREQUEST_LATENCY = "bulkRequestLatency"; public static final String BULKREQUEST_ERRORS = "bulkRequestErrors"; public static final String BULKREQUEST_SIZE_BYTES = "bulkRequestSizeBytes"; @@ -124,14 +119,14 @@ public void initialize() throws IOException { } @Override - public void doOutput(final Collection> records) { + public void doOutput(final Collection> records) { if (records.isEmpty()) { return; } AccumulatingBulkRequest bulkRequest = bulkRequestSupplier.get(); - for (final Record record : records) { + for (final Record record : records) { final SerializedJson document = getDocument(record.getData()); final Optional docId = getDocumentIdFromDocument(document); @@ -200,20 +195,8 @@ private Optional getDocumentIdFromDocument(final SerializedJson document return Optional.empty(); } - // Temporary function to support both trace and log ingestion pipelines. - // TODO: This function should be removed with the completion of: https://github.com/opensearch-project/data-prepper/issues/546 - private SerializedJson getDocument(final Object object) { - final String jsonString; - if (object instanceof String) { - jsonString = (String) object; - } else if (object instanceof Event) { - jsonString = ((Event) object).toJsonString(); - - } else { - throw new RuntimeException("Invalid record type. OpenSearch sink only supports String and Events"); - } - - return SerializedJson.fromString(jsonString); + private SerializedJson getDocument(final Event event) { + return SerializedJson.fromString(event.toJsonString()); } private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { @@ -230,12 +213,6 @@ private void flushBatch(AccumulatingBulkRequest accumulatingBulkRequest) { }); } - private Map getMapFromJson(final String documentJson) throws IOException { - final XContentParser parser = XContentFactory.xContent(XContentType.JSON) - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, documentJson); - return parser.map(); - } - private void logFailure(final BulkOperation bulkOperation, final Throwable failure) { if (dlqWriter != null) { try { diff --git a/data-prepper-plugins/otel-trace-group-prepper/README.md b/data-prepper-plugins/otel-trace-group-prepper/README.md deleted file mode 100644 index 61d77ee13e..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# OTel Trace Group Prepper - -This is a prepper that fills in the missing trace group related fields in the collection of raw span string records output by [otel-trace-raw-prepper](../dataPrepper-plugins/otel-trace-raw-prepper) and then convert them back into a new collection of string records. -It finds the missing trace group info for a spanId by looking up the relevant fields in its root span stored in OpenSearch or Amazon OpenSearch Service backend that the local data-prepper host ingest into. - -## Usages - -### OpenSearch - -``` -pipeline: - ... - prepper: - - otel-trace-group-prepper: - hosts: ["https://localhost:9200"] - cert: path/to/cert - username: YOUR_USERNAME_HERE - password: YOUR_PASSWORD_HERE -``` - -See [opensearch_security.md](../opensearch/opensearch_security.md) for detailed explanation. - -### Amazon OpenSearch Service - -``` -pipeline: - ... - prepper: - - otel-trace-group-prepper: - hosts: ["https://your-amazon-opensearch-service-endpoint"] - aws_sigv4: true - cert: path/to/cert - insecure: false -``` - -See [security.md](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/opensearch/security.md) for detailed explanation. - -## Configuration - -- `hosts`: A list of IP addresses of OpenSearch nodes. - -- `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that OpenSearch is using. -Default is null. - -- `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon OpenSearch Service. See [security](security.md) for details. Default to `false`. - -- `aws_region`: A String represents the region of Amazon OpenSearch Service domain, e.g. us-west-2. Only applies to Amazon OpenSearch Service. Defaults to `us-east-1`. - -- `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to `false`. - -- `username`(optional): A String of username used in the [internal users](https://opensearch.org/docs/latest/security-plugin/access-control/users-roles) of OpenSearch cluster. Default is null. - -- `password`(optional): A String of password used in the [internal users](https://opensearch.org/docs/latest/security-plugin/access-control/users-roles) of OpenSearch cluster. Default is null. - -## Metrics - -### Counter -- `recordsInMissingTraceGroup`: number of ingress records missing trace group fields. -- `recordsOutFixedTraceGroup`: number of egress records with trace group fields filled successfully. -- `recordsOutMissingTraceGroup`: number of egress records missing trace group fields. - -## Developer Guide - -This plugin is compatible with Java 8. See - -- [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-group-prepper/build.gradle b/data-prepper-plugins/otel-trace-group-prepper/build.gradle deleted file mode 100644 index 551d0d359e..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -plugins { - id 'java' -} - -ext { - opensearchVersion = System.getProperty("opensearch.version", "${versionMap.opensearchVersion}") -} - -dependencies { - implementation project(':data-prepper-api') - implementation project(':data-prepper-plugins:opensearch') - implementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearchVersion}" - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'io.micrometer:micrometer-core' - testImplementation project(':data-prepper-api').sourceSets.test.output -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepper.java b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepper.java deleted file mode 100644 index 21df3b6760..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepper.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltracegroup; - -import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.processor.AbstractProcessor; -import com.amazon.dataprepper.model.processor.Processor; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.plugins.prepper.oteltracegroup.model.TraceGroup; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; -import io.micrometer.core.instrument.Counter; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.document.DocumentField; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Instant; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Stream; - -@DataPrepperPlugin(name = "otel_trace_group_prepper", pluginType = Processor.class) -public class OTelTraceGroupPrepper extends AbstractProcessor, Record> { - - public static final String RECORDS_IN_MISSING_TRACE_GROUP = "recordsInMissingTraceGroup"; - public static final String RECORDS_OUT_FIXED_TRACE_GROUP = "recordsOutFixedTraceGroup"; - public static final String RECORDS_OUT_MISSING_TRACE_GROUP = "recordsOutMissingTraceGroup"; - - private static final Logger LOG = LoggerFactory.getLogger(OTelTraceGroupPrepper.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference>() {}; - - private final OTelTraceGroupPrepperConfig otelTraceGroupPrepperConfig; - private final RestHighLevelClient restHighLevelClient; - - private final Counter recordsInMissingTraceGroupCounter; - private final Counter recordsOutFixedTraceGroupCounter; - private final Counter recordsOutMissingTraceGroupCounter; - - public OTelTraceGroupPrepper(final PluginSetting pluginSetting) { - super(pluginSetting); - otelTraceGroupPrepperConfig = OTelTraceGroupPrepperConfig.buildConfig(pluginSetting); - restHighLevelClient = otelTraceGroupPrepperConfig.getEsConnectionConfig().createClient(); - - recordsInMissingTraceGroupCounter = pluginMetrics.counter(RECORDS_IN_MISSING_TRACE_GROUP); - recordsOutFixedTraceGroupCounter = pluginMetrics.counter(RECORDS_OUT_FIXED_TRACE_GROUP); - recordsOutMissingTraceGroupCounter = pluginMetrics.counter(RECORDS_OUT_MISSING_TRACE_GROUP); - } - - @Override - public Collection> doExecute(final Collection> rawSpanStringRecords) { - final List> recordsOut = new LinkedList<>(); - final Map, Map> recordMissingTraceGroupToRawSpanMap = new HashMap<>(); - final Set traceIdsToLookUp = new HashSet<>(); - for (Record record: rawSpanStringRecords) { - try { - final Map rawSpanMap = OBJECT_MAPPER.readValue(record.getData(), MAP_TYPE_REFERENCE); - final String traceGroupName = (String) rawSpanMap.get(TraceGroup.TRACE_GROUP_NAME_FIELD); - final String traceId = (String) rawSpanMap.get(OTelTraceGroupPrepperConfig.TRACE_ID_FIELD); - if (Strings.isNullOrEmpty(traceGroupName)) { - traceIdsToLookUp.add(traceId); - recordMissingTraceGroupToRawSpanMap.put(record, rawSpanMap); - recordsInMissingTraceGroupCounter.increment(); - } else { - recordsOut.add(record); - } - } catch (JsonProcessingException e) { - LOG.error("Failed to parse the record: [{}]", record.getData()); - } - } - - final Map traceIdToTraceGroup = searchTraceGroupByTraceIds(traceIdsToLookUp); - for (final Map.Entry, Map> entry: recordMissingTraceGroupToRawSpanMap.entrySet()) { - final Record record = entry.getKey(); - final Map rawSpanMap = entry.getValue(); - final String traceId = (String) rawSpanMap.get(OTelTraceGroupPrepperConfig.TRACE_ID_FIELD); - final TraceGroup traceGroup = traceIdToTraceGroup.get(traceId); - if (traceGroup != null) { - try { - Map traceGroupMap = OBJECT_MAPPER.convertValue(traceGroup, MAP_TYPE_REFERENCE); - rawSpanMap.putAll(traceGroupMap); - final String newData = OBJECT_MAPPER.writeValueAsString(rawSpanMap); - recordsOut.add(new Record<>(newData, record.getMetadata())); - recordsOutFixedTraceGroupCounter.increment(); - } catch (Exception e) { - recordsOut.add(record); - recordsOutMissingTraceGroupCounter.increment(); - LOG.error("Failed to process the raw span: [{}]", record.getData(), e); - } - } else { - recordsOut.add(record); - recordsOutMissingTraceGroupCounter.increment(); - final String spanId = (String) rawSpanMap.get(OTelTraceGroupPrepperConfig.SPAN_ID_FIELD); - LOG.warn("Failed to find traceGroup for spanId: {} due to traceGroup missing for traceId: {}", spanId, traceId); - } - } - - return recordsOut; - } - - private Map searchTraceGroupByTraceIds(final Collection traceIds) { - final Map traceIdToTraceGroup = new HashMap<>(); - final SearchRequest searchRequest = createSearchRequest(traceIds); - - try { - final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - final SearchHit[] searchHits = searchResponse.getHits().getHits(); - Arrays.asList(searchHits).forEach(searchHit -> { - final Optional> optionalStringTraceGroupEntry = fromSearchHitToMapEntry(searchHit); - optionalStringTraceGroupEntry.ifPresent(entry -> traceIdToTraceGroup.put(entry.getKey(), entry.getValue())); - }); - } catch (Exception e) { - // TODO: retry for status code 429 of OpenSearchException? - LOG.error("Search request for traceGroup failed for traceIds: {} due to {}", traceIds, e.getMessage()); - } - - return traceIdToTraceGroup; - } - - private SearchRequest createSearchRequest(final Collection traceIds) { - final SearchRequest searchRequest = new SearchRequest(OTelTraceGroupPrepperConfig.RAW_INDEX_ALIAS); - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query( - QueryBuilders.boolQuery() - .must(QueryBuilders.termsQuery(OTelTraceGroupPrepperConfig.TRACE_ID_FIELD, traceIds)) - .must(QueryBuilders.termQuery(OTelTraceGroupPrepperConfig.PARENT_SPAN_ID_FIELD, "")) - ); - searchSourceBuilder.docValueField(OTelTraceGroupPrepperConfig.TRACE_ID_FIELD); - searchSourceBuilder.docValueField(TraceGroup.TRACE_GROUP_NAME_FIELD); - searchSourceBuilder.docValueField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, OTelTraceGroupPrepperConfig.STRICT_DATE_TIME); - searchSourceBuilder.docValueField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD); - searchSourceBuilder.docValueField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD); - searchSourceBuilder.fetchSource(false); - searchRequest.source(searchSourceBuilder); - - return searchRequest; - } - - private Optional> fromSearchHitToMapEntry(final SearchHit searchHit) { - final DocumentField traceIdDocField = searchHit.field(OTelTraceGroupPrepperConfig.TRACE_ID_FIELD); - final DocumentField traceGroupNameDocField = searchHit.field(TraceGroup.TRACE_GROUP_NAME_FIELD); - final DocumentField traceGroupEndTimeDocField = searchHit.field(TraceGroup.TRACE_GROUP_END_TIME_FIELD); - final DocumentField traceGroupDurationInNanosDocField = searchHit.field(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD); - final DocumentField traceGroupStatusCodeDocField = searchHit.field(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD); - if (Stream.of(traceIdDocField, traceGroupNameDocField, traceGroupEndTimeDocField, traceGroupDurationInNanosDocField, - traceGroupStatusCodeDocField).allMatch(Objects::nonNull)) { - final String traceId = traceIdDocField.getValue(); - final String traceGroupName = traceGroupNameDocField.getValue(); - // Restore trailing zeros for thousand, e.g. 2020-08-20T05:40:46.0895568Z -> 2020-08-20T05:40:46.089556800Z - final String traceGroupEndTime = Instant.parse(traceGroupEndTimeDocField.getValue()).toString(); - final Number traceGroupDurationInNanos = traceGroupDurationInNanosDocField.getValue(); - final Number traceGroupStatusCode = traceGroupStatusCodeDocField.getValue(); - return Optional.of(new AbstractMap.SimpleEntry<>(traceId, new TraceGroup(traceGroupName, traceGroupEndTime, - traceGroupDurationInNanos.longValue(), traceGroupStatusCode.intValue()))); - } - return Optional.empty(); - } - - @Override - public void prepareForShutdown() { - - } - - @Override - public boolean isReadyForShutdown() { - return true; - } - - @Override - public void shutdown() { - try { - restHighLevelClient.close(); - } catch (IOException e) { - throw new RuntimeException(e.getMessage(), e); - } - } -} diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfig.java b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfig.java deleted file mode 100644 index a3bfecff06..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltracegroup; - -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexConstants; -import com.amazon.dataprepper.plugins.sink.opensearch.index.IndexType; - -public class OTelTraceGroupPrepperConfig { - protected static final String TRACE_ID_FIELD = "traceId"; - protected static final String SPAN_ID_FIELD = "spanId"; - protected static final String PARENT_SPAN_ID_FIELD = "parentSpanId"; - protected static final String RAW_INDEX_ALIAS = IndexConstants.TYPE_TO_DEFAULT_ALIAS.get(IndexType.TRACE_ANALYTICS_RAW); - protected static final String STRICT_DATE_TIME = "strict_date_time"; - - private final ConnectionConfiguration esConnectionConfig; - - public ConnectionConfiguration getEsConnectionConfig() { - return esConnectionConfig; - } - - private OTelTraceGroupPrepperConfig(final ConnectionConfiguration esConnectionConfig) { - this.esConnectionConfig = esConnectionConfig; - } - - public static OTelTraceGroupPrepperConfig buildConfig(final PluginSetting pluginSetting) { - final ConnectionConfiguration esConnectionConfig = ConnectionConfiguration.readConnectionConfiguration(pluginSetting); - return new OTelTraceGroupPrepperConfig(esConnectionConfig); - } -} diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java b/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java deleted file mode 100644 index 956a7bae1d..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/model/TraceGroup.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltracegroup.model; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Objects; - -public class TraceGroup { - public static final String TRACE_GROUP_NAME_FIELD = "traceGroup"; - public static final String TRACE_GROUP_END_TIME_FIELD = "traceGroupFields.endTime"; - public static final String TRACE_GROUP_STATUS_CODE_FIELD = "traceGroupFields.statusCode"; - public static final String TRACE_GROUP_DURATION_IN_NANOS_FIELD = "traceGroupFields.durationInNanos"; - - @JsonProperty(TRACE_GROUP_NAME_FIELD) - private final String name; - - @JsonProperty(TRACE_GROUP_END_TIME_FIELD) - private final String endTime; - - @JsonProperty(TRACE_GROUP_DURATION_IN_NANOS_FIELD) - private final Long durationInNanos; - - @JsonProperty(TRACE_GROUP_STATUS_CODE_FIELD) - private final Integer statusCode; - - public String getName() { - return name; - } - - public String getEndTime() { - return endTime; - } - - public Long getDurationInNanos() { - return durationInNanos; - } - - public Integer getStatusCode() { - return statusCode; - } - - public TraceGroup(final String name, final String endTime, final Long durationInNanos, final Integer statusCode) { - this.name = name; - this.endTime = endTime; - this.durationInNanos = durationInNanos; - this.statusCode = statusCode; - } - - @Override - public boolean equals(final Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - final TraceGroup that = (TraceGroup) o; - return Objects.equals(name, that.name) && - Objects.equals(endTime, that.endTime) && - Objects.equals(durationInNanos, that.durationInNanos) && - Objects.equals(statusCode, that.statusCode); - } - - @Override - public int hashCode() { - return Objects.hash(name, endTime, durationInNanos, statusCode); - } -} diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfigTests.java b/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfigTests.java deleted file mode 100644 index c24a93ab07..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperConfigTests.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltracegroup; - -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import java.util.HashMap; - -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; - -@RunWith(MockitoJUnitRunner.class) -public class OTelTraceGroupPrepperConfigTests { - - @Mock - private ConnectionConfiguration connectionConfigurationMock; - - @Test - public void testInitialize() { - try (MockedStatic connectionConfigurationMockedStatic = Mockito.mockStatic(ConnectionConfiguration.class)) { - connectionConfigurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(any(PluginSetting.class))) - .thenReturn(connectionConfigurationMock); - PluginSetting testPluginSetting = new PluginSetting("otel_trace_group_prepper", new HashMap<>()); - OTelTraceGroupPrepperConfig otelTraceGroupPrepperConfig = OTelTraceGroupPrepperConfig.buildConfig(testPluginSetting); - assertEquals(connectionConfigurationMock, otelTraceGroupPrepperConfig.getEsConnectionConfig()); - } - } -} diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperTests.java b/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperTests.java deleted file mode 100644 index 1f630cb8ef..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltracegroup/OTelTraceGroupPrepperTests.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltracegroup; - -import com.amazon.dataprepper.metrics.MetricNames; -import com.amazon.dataprepper.metrics.MetricsTestUtil; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.plugins.prepper.oteltracegroup.model.TraceGroup; -import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.micrometer.core.instrument.Measurement; -import org.opensearch.OpenSearchException; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.document.DocumentField; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.StringJoiner; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class OTelTraceGroupPrepperTests { - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String TEST_PIPELINE_NAME = "testPipelineName"; - private static final String PLUGIN_NAME = "otel_trace_group_prepper"; - private static final String TEST_TRACE_ID_1 = "6d0ff634d126b6ec2c180391e67b4237"; - private static final TraceGroup TEST_TRACE_GROUP_1 = new TraceGroup("/test_trace_group_1", - "2020-08-19T05:30:46.089556800Z", 48545100L, 1); - private static final String TEST_TRACE_ID_2 = "ffa576d321173ac6cef3601c8f4bde75"; - private static final TraceGroup TEST_TRACE_GROUP_2 = new TraceGroup("/test_trace_group_2", - "2020-08-20T05:30:46.089556800Z", 48545300L, 0); - private static final String TEST_RAW_SPAN_COMPLETE_JSON_FILE_1 = "raw-span-complete-1.json"; - private static final String TEST_RAW_SPAN_COMPLETE_JSON_FILE_2 = "raw-span-complete-2.json"; - private static final String TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1 = "raw-span-missing-trace-group-1.json"; - private static final String TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_2 = "raw-span-missing-trace-group-2.json"; - private static final int TEST_NUM_WORKERS = 2; - - private MockedStatic connectionConfigurationMockedStatic; - - private OTelTraceGroupPrepper otelTraceGroupPrepper; - private ExecutorService executorService; - - @Mock - private ConnectionConfiguration connectionConfigurationMock; - - @Mock - private RestHighLevelClient restHighLevelClient; - - @Mock - private SearchResponse testSearchResponse; - - @Mock - private SearchHits testSearchHits; - - @Mock - private SearchHit testSearchHit1; - - @Mock - private SearchHit testSearchHit2; - - @Before - public void setUp() throws Exception{ - MetricsTestUtil.initMetrics(); - connectionConfigurationMockedStatic = Mockito.mockStatic(ConnectionConfiguration.class); - connectionConfigurationMockedStatic.when(() -> ConnectionConfiguration.readConnectionConfiguration(any(PluginSetting.class))) - .thenReturn(connectionConfigurationMock); - when(connectionConfigurationMock.createClient()).thenReturn(restHighLevelClient); - when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))).thenReturn(testSearchResponse); - doNothing().when(restHighLevelClient).close(); - when(testSearchResponse.getHits()).thenReturn(testSearchHits); - when(testSearchHits.getHits()).thenReturn(new SearchHit[] {testSearchHit1}); - when(testSearchHit1.field("traceId")).thenReturn(new DocumentField("traceId", Collections.singletonList(TEST_TRACE_ID_1))); - when(testSearchHit1.field(TraceGroup.TRACE_GROUP_NAME_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_NAME_FIELD, Collections.singletonList(TEST_TRACE_GROUP_1.getName()))); - when(testSearchHit1.field(TraceGroup.TRACE_GROUP_END_TIME_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, Collections.singletonList(TEST_TRACE_GROUP_1.getEndTime()))); - when(testSearchHit1.field(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD, Collections.singletonList(TEST_TRACE_GROUP_1.getDurationInNanos()))); - when(testSearchHit1.field(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD, Collections.singletonList(TEST_TRACE_GROUP_1.getStatusCode()))); - when(testSearchHit2.field("traceId")).thenReturn(new DocumentField("traceId", Collections.singletonList(TEST_TRACE_ID_2))); - when(testSearchHit2.field(TraceGroup.TRACE_GROUP_NAME_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_NAME_FIELD, Collections.singletonList(TEST_TRACE_GROUP_2.getName()))); - when(testSearchHit2.field(TraceGroup.TRACE_GROUP_END_TIME_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, Collections.singletonList(TEST_TRACE_GROUP_2.getEndTime()))); - when(testSearchHit2.field(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD, Collections.singletonList(TEST_TRACE_GROUP_2.getDurationInNanos()))); - when(testSearchHit2.field(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD)) - .thenReturn(new DocumentField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD, Collections.singletonList(TEST_TRACE_GROUP_2.getStatusCode()))); - final PluginSetting testPluginSetting = new PluginSetting("otel_trace_group_prepper", new HashMap<>()) {{ - setPipelineName(TEST_PIPELINE_NAME); - }}; - otelTraceGroupPrepper = new OTelTraceGroupPrepper(testPluginSetting); - executorService = Executors.newFixedThreadPool(TEST_NUM_WORKERS); - } - - @After - public void tearDown() { - otelTraceGroupPrepper.shutdown(); - connectionConfigurationMockedStatic.close(); - executorService.shutdown(); - } - - @Test - public void testShutDown() throws IOException { - // Act - otelTraceGroupPrepper.shutdown(); - - // Assert - verify(restHighLevelClient, times(1)).close(); - } - - @Test - public void testTraceGroupFillSuccess() throws IOException { - // Arrange - Record testRecord = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1); - List> testRecords = Collections.singletonList(testRecord); - - // Act - List> recordsOut = (List>) otelTraceGroupPrepper.doExecute(testRecords); - - // Assert - assertEquals(1, recordsOut.size()); - Record recordOut = recordsOut.get(0); - assertEquals(TEST_TRACE_GROUP_1, extractTraceGroupFromRecord(recordOut)); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_IN_MISSING_TRACE_GROUP, 1.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_FIXED_TRACE_GROUP, 1.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_MISSING_TRACE_GROUP, 0.0); - } - - @Test - public void testTraceGroupFillFailDueToFailedRequest() throws IOException { - // Arrange - Record testRecord = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1); - List> testRecords = Collections.singletonList(testRecord); - when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))) - .thenThrow(new OpenSearchException("Failure due to search request")); - - // Act - List> recordsOut = (List>) otelTraceGroupPrepper.doExecute(testRecords); - - // Assert - assertEquals(1, recordsOut.size()); - Record recordOut = recordsOut.get(0); - assertEquals(testRecord, recordOut); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_IN_MISSING_TRACE_GROUP, 1.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_FIXED_TRACE_GROUP, 0.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_MISSING_TRACE_GROUP, 1.0); - } - - @Test - public void testTraceGroupFillFailDueToNoHits() throws IOException { - // Arrange - Record testRecord = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1); - List> testRecords = Collections.singletonList(testRecord); - when(restHighLevelClient.search(any(SearchRequest.class), any(RequestOptions.class))).thenReturn(testSearchResponse); - when(testSearchResponse.getHits()).thenReturn(testSearchHits); - when(testSearchHits.getHits()).thenReturn(new SearchHit[] {}); - - // Act - List> recordsOut = (List>) otelTraceGroupPrepper.doExecute(testRecords); - - // Assert - assertEquals(1, recordsOut.size()); - Record recordOut = recordsOut.get(0); - assertEquals(testRecord, recordOut); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_IN_MISSING_TRACE_GROUP, 1.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_FIXED_TRACE_GROUP, 0.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_MISSING_TRACE_GROUP, 1.0); - } - - @Test - public void testTraceGroupFieldAlreadyPopulated() throws IOException { - // Arrange - Record testRecord = buildRawSpanRecord(TEST_RAW_SPAN_COMPLETE_JSON_FILE_1); - List> testRecords = Collections.singletonList(testRecord); - - // Act - List> recordsOut = (List>) otelTraceGroupPrepper.doExecute(testRecords); - - // Assert - assertEquals(1, recordsOut.size()); - Record recordOut = recordsOut.get(0); - assertEquals(testRecord, recordOut); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_IN_MISSING_TRACE_GROUP, 0.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_FIXED_TRACE_GROUP, 0.0); - checkMeasurementValue(OTelTraceGroupPrepper.RECORDS_OUT_MISSING_TRACE_GROUP, 0.0); - } - - @Test - public void testTraceGroupProcessMultiWorker() throws IOException, ExecutionException, InterruptedException { - /* - * Note: we only test the threadsafety of the business logic in OtelTraceGroupPrepper. The OpenSearch REST client - * itself is thread-safe {https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/_changing_the_client_8217_s_initialization_code.html}. - */ - // Arrange - when(testSearchHits.getHits()).thenReturn(new SearchHit[] {testSearchHit1, testSearchHit2}); - Record testCompleteRecord1 = buildRawSpanRecord(TEST_RAW_SPAN_COMPLETE_JSON_FILE_1); - Record testMissingRecord1 = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_1); - Record testCompleteRecord2 = buildRawSpanRecord(TEST_RAW_SPAN_COMPLETE_JSON_FILE_2); - Record testMissingRecord2 = buildRawSpanRecord(TEST_RAW_SPAN_MISSING_TRACE_GROUP_JSON_FILE_2); - final List> processedRecords = new ArrayList<>(); - List>>> futures = new ArrayList<>(); - - // Act - futures.addAll(submitBatchRecords(Arrays.asList(testCompleteRecord1, testMissingRecord1))); - futures.addAll(submitBatchRecords(Arrays.asList(testCompleteRecord2, testMissingRecord2))); - for (Future>> future : futures) { - processedRecords.addAll(future.get()); - } - - // Assert - assertEquals(4, processedRecords.size()); - for (Record record: processedRecords) { - assertNotNull(extractTraceGroupFromRecord(record)); - } - } - - @Test - public void testPrepareForShutdown() { - otelTraceGroupPrepper.prepareForShutdown(); - - assertTrue(otelTraceGroupPrepper.isReadyForShutdown()); - } - - private Record buildRawSpanRecord(String rawSpanJsonFileName) throws IOException { - final StringBuilder jsonBuilder = new StringBuilder(); - try (final InputStream inputStream = Objects.requireNonNull( - getClass().getClassLoader().getResourceAsStream(rawSpanJsonFileName))){ - final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); - bufferedReader.lines().forEach(jsonBuilder::append); - } - return new Record<>(jsonBuilder.toString()); - } - - private TraceGroup extractTraceGroupFromRecord(final Record record) throws JsonProcessingException { - Map rawSpanMap = OBJECT_MAPPER.readValue(record.getData(), new TypeReference>() {}); - final String traceGroupName = (String) rawSpanMap.get(TraceGroup.TRACE_GROUP_NAME_FIELD); - final String traceGroupEndTime = (String) rawSpanMap.get(TraceGroup.TRACE_GROUP_END_TIME_FIELD); - final Long traceGroupDurationInNanos = ((Number) rawSpanMap.get(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD)).longValue(); - final Integer traceGroupStatusCode = ((Number) rawSpanMap.get(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD)).intValue(); - return new TraceGroup(traceGroupName, traceGroupEndTime, traceGroupDurationInNanos, traceGroupStatusCode); - } - - private List>>> submitBatchRecords(List> records) { - final List>>> futures = new ArrayList<>(); - futures.add(executorService.submit(() -> otelTraceGroupPrepper.doExecute(records))); - return futures; - } - - private void checkMeasurementValue(final String name, final double expectedValue) { - final List spansMissingTraceGroupMeasures = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add(PLUGIN_NAME).add(name).toString()); - assertEquals(1, spansMissingTraceGroupMeasures.size()); - final Measurement spansMissingTraceGroupMeasure = spansMissingTraceGroupMeasures.get(0); - assertEquals(expectedValue, spansMissingTraceGroupMeasure.getValue(), 0); - } -} diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index 78ccc25012..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1,3 +0,0 @@ -# To enable mocking of final classes with vanilla Mockito -# https://github.com/mockito/mockito/wiki/What%27s-new-in-Mockito-2#mock-the-unmockable-opt-in-mocking-of-final-classesmethods -mock-maker-inline \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json deleted file mode 100644 index 1226514180..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-1.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "traceId": "6d0ff634d126b6ec2c180391e67b4237", - "spanId": "9daf5fc00a50b18b", - "traceState": "", - "parentSpanId": "5ac57618587da71c", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTime": "2020-08-20T05:40:46.052809200Z", - "endTime": "2020-08-20T05:40:46.084822500Z", - "durationInNanos": 32013300, - "serviceName": "analytics-service", - "events": [], - "links": [], - "droppedAttributesCount": 0, - "droppedEventsCount": 0, - "droppedLinksCount": 0, - "traceGroup": "/logs", - "traceGroupFields.endTime": "2020-08-20T05:40:46.089556800Z", - "traceGroupFields.statusCode": 1, - "traceGroupFields.durationInNanos": 48545200, - "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", - "span.attributes.http@method": "PUT", - "resource.attributes.telemetry@sdk@name": "opentelemetry", - "resource.attributes.telemetry@sdk@language": "java", - "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", - "resource.attributes.service@name": "analytics-service", - "span.attributes.http@status_code": 200, - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json deleted file mode 100644 index 7c119322c4..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-complete-2.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "traceId": "ffa576d321173ac6cef3601c8f4bde75", - "spanId": "cb1c0734d14940fd", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTime": "2020-08-20T05:40:43.168010200Z", - "endTime": "2020-08-20T05:40:43.217170200Z", - "durationInNanos": 49160000, - "serviceName": "analytics-service", - "events": [], - "links": [], - "droppedAttributesCount": 0, - "droppedEventsCount": 0, - "droppedLinksCount": 0, - "traceGroup": "/logs", - "traceGroupFields.endTime": "2020-08-20T05:40:43.217170200Z", - "traceGroupFields.statusCode": 1, - "traceGroupFields.durationInNanos": 49160000, - "span.attributes.http@url": "http://0.0.0.0:8087/logs", - "span.attributes.net@peer@ip": "172.29.0.1", - "resource.attributes.telemetry@sdk@language": "java", - "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", - "resource.attributes.service@name": "analytics-service", - "span.attributes.servlet@path": "/logs", - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.servlet-3.0", - "span.attributes.http@response_content_length": 7, - "span.attributes.http@user_agent": "curl/7.54.0", - "span.attributes.http@method": "POST", - "span.attributes.net@peer@port": 41164, - "resource.attributes.telemetry@sdk@name": "opentelemetry", - "span.attributes.servlet@context": "", - "span.attributes.http@flavor": "HTTP/1.1", - "span.attributes.http@status_code": 200, - "span.attributes.http@client_ip": "172.29.0.1" -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json deleted file mode 100644 index 239508132a..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-1.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "traceId": "6d0ff634d126b6ec2c180391e67b4237", - "spanId": "9daf5fc00a50b18b", - "traceState": "", - "parentSpanId": "5ac57618587da71c", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTime": "2020-08-20T05:40:46.052809200Z", - "endTime": "2020-08-20T05:40:46.084822500Z", - "durationInNanos": 32013300, - "serviceName": "analytics-service", - "events": [], - "links": [], - "droppedAttributesCount": 0, - "droppedEventsCount": 0, - "droppedLinksCount": 0, - "traceGroup": null, - "traceGroupFields.endTime": null, - "traceGroupFields.statusCode": null, - "traceGroupFields.durationInNanos": null, - "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", - "span.attributes.http@method": "PUT", - "resource.attributes.telemetry@sdk@name": "opentelemetry", - "resource.attributes.telemetry@sdk@language": "java", - "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", - "resource.attributes.service@name": "analytics-service", - "span.attributes.http@status_code": 200, - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json b/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json deleted file mode 100644 index cb4ae8c110..0000000000 --- a/data-prepper-plugins/otel-trace-group-prepper/src/test/resources/raw-span-missing-trace-group-2.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "traceId": "ffa576d321173ac6cef3601c8f4bde75", - "spanId": "1b83d176c0b5fb46", - "traceState": "", - "parentSpanId": "085ac082ffcfbf8d", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTime": "2020-08-20T05:40:43.175204700Z", - "endTime": "2020-08-20T05:40:43.205117100Z", - "durationInNanos": 29912400, - "serviceName": "analytics-service", - "events": [], - "links": [], - "droppedAttributesCount": 0, - "droppedEventsCount": 0, - "droppedLinksCount": 0, - "traceGroup": null, - "traceGroupFields.endTime": null, - "traceGroupFields.statusCode": null, - "traceGroupFields.durationInNanos": null, - "span.attributes.http@url": "/logs/_doc/service_1?timeout\\u003d1m", - "span.attributes.http@method": "PUT", - "resource.attributes.telemetry@sdk@name": "opentelemetry", - "resource.attributes.telemetry@sdk@language": "java", - "resource.attributes.telemetry@sdk@version": "0.8.0-SNAPSHOT", - "resource.attributes.service@name": "analytics-service", - "span.attributes.http@status_code": 200, - "status.code": 1, - "instrumentationLibrary.name": "io.opentelemetry.auto.apache-httpasyncclient-4.0" -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-group-processor/README.md b/data-prepper-plugins/otel-trace-group-processor/README.md index 9b866faa67..790a5dab06 100644 --- a/data-prepper-plugins/otel-trace-group-processor/README.md +++ b/data-prepper-plugins/otel-trace-group-processor/README.md @@ -1,6 +1,6 @@ # OTel Trace Group Processor -This is a processor that fills in the missing trace group related fields in the collection of [Span](../../data-prepper-api/src/main/java/com/amazon/dataprepper/model/trace/Span.java) records output by [otel_trace_raw](../otel-trace-raw-processor) processor. +This is a processor that fills in the missing trace group related fields in the collection of [Span](../../data-prepper-api/src/main/java/com/amazon/dataprepper/model/trace/Span.java) records by looking up the opensearch backend. It finds the missing trace group info for a spanId by looking up the relevant fields in its root span stored in OpenSearch or Amazon OpenSearch Service backend that the local data-prepper host ingest into. ## Usages diff --git a/data-prepper-plugins/otel-trace-raw-prepper/README.md b/data-prepper-plugins/otel-trace-raw-prepper/README.md deleted file mode 100644 index b962f0c5b3..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# OTel Trace Raw Prepper - -This is a prepper that serializes collection of `ExportTraceServiceRequest` sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source) into collection of string records. - -## Usages -Example `.yaml` configuration -``` -prepper: - - otel_trace_raw_prepper: -``` - -## Configuration - -* `trace_flush_interval`: An `int` represents the time interval in seconds to flush all the descendant spans without any root span. Default to 180. - -## Metrics -Apart from common metrics in [AbstractProcessor](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-api/src/main/java/com/amazon/dataprepper/model/processor/AbstractProcessor.java), otel-trace-raw-prepper introduces the following custom metrics. - -### Counter -- `spanProcessingErrors`: records the number of processing exceptions for invalid spans. -- `resourceSpansProcessingErrors`: records the number of processing exceptions for invalid resource spans. -- `totalProcessingErrors`: records the total number of processing errors for spans and resource spans. - -## Developer Guide -This plugin is compatible with Java 8. See -- [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) -- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/monitoring.md) diff --git a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle b/data-prepper-plugins/otel-trace-raw-prepper/build.gradle deleted file mode 100644 index 681b67ae49..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/build.gradle +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -plugins { - id 'java' -} - -dependencies { - implementation project(':data-prepper-api') - implementation project(':data-prepper-plugins:common') - implementation 'commons-codec:commons-codec:1.15' - testImplementation project(':data-prepper-api').sourceSets.test.output - implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}" - implementation "com.google.protobuf:protobuf-java-util:${versionMap.protobufJavaUtil}" - implementation "com.linecorp.armeria:armeria:${versionMap.armeria}" - implementation "com.linecorp.armeria:armeria-grpc:${versionMap.armeriaGrpc}" - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'com.google.guava:guava:31.1-jre' - testImplementation 'org.assertj:assertj-core:3.23.1' - testImplementation "org.mockito:mockito-inline:${versionMap.mockito}" -} - -jacocoTestCoverageVerification { - dependsOn jacocoTestReport - violationRules { - rule { //in addition to core projects rule - limit { - minimum = 0.95 //Increase this to 0.90 - } - } - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java deleted file mode 100644 index f788af5b58..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepper.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace; - -import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.processor.AbstractProcessor; -import com.amazon.dataprepper.model.processor.Processor; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.plugins.prepper.oteltrace.model.OTelProtoHelper; -import com.amazon.dataprepper.plugins.prepper.oteltrace.model.RawSpan; -import com.amazon.dataprepper.plugins.prepper.oteltrace.model.RawSpanBuilder; -import com.amazon.dataprepper.plugins.prepper.oteltrace.model.RawSpanSet; -import com.amazon.dataprepper.plugins.prepper.oteltrace.model.TraceGroup; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.util.StringUtils; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; -import io.opentelemetry.proto.trace.v1.ResourceSpans; -import io.opentelemetry.proto.trace.v1.Span; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - - -@DataPrepperPlugin(name = "otel_trace_raw_prepper", pluginType = Processor.class) -public class OTelTraceRawPrepper extends AbstractProcessor, Record> { - private static final long SEC_TO_MILLIS = 1_000L; - private static final Logger LOG = LoggerFactory.getLogger(OTelTraceRawPrepper.class); - - public static final String SPAN_PROCESSING_ERRORS = "spanProcessingErrors"; - public static final String RESOURCE_SPANS_PROCESSING_ERRORS = "resourceSpansProcessingErrors"; - public static final String TOTAL_PROCESSING_ERRORS = "totalProcessingErrors"; - - private final long traceFlushInterval; - - private final Counter spanErrorsCounter; - private final Counter resourceSpanErrorsCounter; - private final Counter totalProcessingErrorsCounter; - - private final Map traceIdRawSpanSetMap = new ConcurrentHashMap<>(); - - private final Cache traceIdTraceGroupCache; - - private long lastTraceFlushTime = 0L; - - private final ReentrantLock traceFlushLock = new ReentrantLock(); - private final ReentrantLock prepareForShutdownLock = new ReentrantLock(); - - private volatile boolean isShuttingDown = false; - - public OTelTraceRawPrepper(final PluginSetting pluginSetting) { - super(pluginSetting); - traceFlushInterval = SEC_TO_MILLIS * pluginSetting.getLongOrDefault( - OtelTraceRawPrepperConfig.TRACE_FLUSH_INTERVAL, OtelTraceRawPrepperConfig.DEFAULT_TG_FLUSH_INTERVAL_SEC); - final int numProcessWorkers = pluginSetting.getNumberOfProcessWorkers(); - traceIdTraceGroupCache = CacheBuilder.newBuilder() - .concurrencyLevel(numProcessWorkers) - .maximumSize(OtelTraceRawPrepperConfig.MAX_TRACE_ID_CACHE_SIZE) - .expireAfterWrite(OtelTraceRawPrepperConfig.DEFAULT_TRACE_ID_TTL_SEC, TimeUnit.SECONDS) - .build(); - spanErrorsCounter = pluginMetrics.counter(SPAN_PROCESSING_ERRORS); - resourceSpanErrorsCounter = pluginMetrics.counter(RESOURCE_SPANS_PROCESSING_ERRORS); - totalProcessingErrorsCounter = pluginMetrics.counter(TOTAL_PROCESSING_ERRORS); - } - - /** - * execute the prepper logic which could potentially modify the incoming record. The level to which the record has - * been modified depends on the implementation - * - * @param records Input records that will be modified/processed - * @return Record modified output records - */ - @Override - public Collection> doExecute(Collection> records) { - final List rawSpans = new LinkedList<>(); - - for (Record ets : records) { - for (ResourceSpans rs : ets.getData().getResourceSpansList()) { - try { - final String serviceName = OTelProtoHelper.getServiceName(rs.getResource()).orElse(null); - final Map resourceAttributes = OTelProtoHelper.getResourceAttributes(rs.getResource()); - for (InstrumentationLibrarySpans is : rs.getInstrumentationLibrarySpansList()) { - for (Span sp : is.getSpansList()) { - final RawSpan rawSpan = new RawSpanBuilder() - .setFromSpan(sp, is.getInstrumentationLibrary(), serviceName, resourceAttributes) - .build(); - - processRawSpan(rawSpan, rawSpans); - } - } - } catch (Exception ex) { - LOG.error("Unable to process invalid ResourceSpan {} :", rs, ex); - resourceSpanErrorsCounter.increment(); - totalProcessingErrorsCounter.increment(); - } - } - } - - rawSpans.addAll(getTracesToFlushByGarbageCollection()); - - return convertRawSpansToJsonRecords(rawSpans); - } - - /** - * Branching logic to handle root and child spans. - * A root span is the first span of a trace, it has no parentSpanId. - * - * @param rawSpan Span to be evaluated - * @param spanSet Collection to insert spans to - */ - private void processRawSpan(final RawSpan rawSpan, final Collection spanSet) { - if (StringUtils.isBlank(rawSpan.getParentSpanId())) { - final List rootSpanAndChildren = processRootSpan(rawSpan); - spanSet.addAll(rootSpanAndChildren); - } else { - final Optional populatedChildSpanOptional = processChildSpan(rawSpan); - if (populatedChildSpanOptional.isPresent()) { - spanSet.add(populatedChildSpanOptional.get()); - } - } - } - - /** - * Retrieves all child spans from memory and returns them as a set with the root span. - * Also adds an entry to the traceID cache so that later child spans can be tagged, - * in the case where a child span is processed AFTER the root span. - * - * @param parentSpan - * @return List containing root span, along with any child spans that have already been processed. - */ - private List processRootSpan(final RawSpan parentSpan) { - traceIdTraceGroupCache.put(parentSpan.getTraceId(), parentSpan.getTraceGroup()); - - final List recordsToFlush = new LinkedList<>(); - recordsToFlush.add(parentSpan); - - final TraceGroup traceGroup = parentSpan.getTraceGroup(); - final String parentSpanTraceId = parentSpan.getTraceId(); - - final RawSpanSet rawSpanSet = traceIdRawSpanSetMap.get(parentSpanTraceId); - if (rawSpanSet != null) { - for (final RawSpan rawSpan : rawSpanSet.getRawSpans()) { - rawSpan.setTraceGroup(traceGroup); - recordsToFlush.add(rawSpan); - } - - traceIdRawSpanSetMap.remove(parentSpanTraceId); - } - - return recordsToFlush; - } - - /** - * Attempts to populate the traceGroup of the child span by fetching from a cache. If the traceGroup is not in the cache, - * the child span is kept in memory to be populated when its corresponding root span arrives. - * - * @param childSpan - * @return Optional containing childSpan if its traceGroup is in memory, otherwise an empty Optional - */ - private Optional processChildSpan(final RawSpan childSpan) { - final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(childSpan.getTraceId()); - - if (traceGroup != null) { - childSpan.setTraceGroup(traceGroup); - return Optional.of(childSpan); - } else { - traceIdRawSpanSetMap.compute(childSpan.getTraceId(), (traceId, rawSpanSet) -> { - if (rawSpanSet == null) { - rawSpanSet = new RawSpanSet(); - } - rawSpanSet.addRawSpan(childSpan); - return rawSpanSet; - }); - - return Optional.empty(); - } - } - - private List> convertRawSpansToJsonRecords(final List rawSpans) { - final List> records = new LinkedList<>(); - - for (RawSpan rawSpan : rawSpans) { - String rawSpanJson; - try { - rawSpanJson = rawSpan.toJson(); - } catch (JsonProcessingException e) { - LOG.error("Unable to process invalid Span {}:", rawSpan, e); - spanErrorsCounter.increment(); - totalProcessingErrorsCounter.increment(); - continue; - } - - records.add(new Record<>(rawSpanJson)); - } - - return records; - } - - /** - * Periodically flush spans from memory. Typically all spans of a trace are written - * once the trace's root span arrives, however some child spans my arrive after the root span. - * This method ensures "orphaned" child spans are eventually flushed from memory. - * @return List of RawSpans to be sent down the pipeline - */ - private List getTracesToFlushByGarbageCollection() { - final List recordsToFlush = new LinkedList<>(); - - if (shouldGarbageCollect()) { - final boolean isLockAcquired = traceFlushLock.tryLock(); - - if (isLockAcquired) { - try { - final long now = System.currentTimeMillis(); - lastTraceFlushTime = now; - - final Iterator> entryIterator = traceIdRawSpanSetMap.entrySet().iterator(); - while (entryIterator.hasNext()) { - final Map.Entry entry = entryIterator.next(); - final String traceId = entry.getKey(); - final TraceGroup traceGroup = traceIdTraceGroupCache.getIfPresent(traceId); - final RawSpanSet rawSpanSet = entry.getValue(); - final long traceTime = rawSpanSet.getTimeSeen(); - if (now - traceTime >= traceFlushInterval || isShuttingDown) { - final Set rawSpans = rawSpanSet.getRawSpans(); - if (traceGroup != null) { - rawSpans.forEach(rawSpan -> { - rawSpan.setTraceGroup(traceGroup); - recordsToFlush.add(rawSpan); - }); - } else { - rawSpans.forEach(rawSpan -> { - recordsToFlush.add(rawSpan); - LOG.warn("Missing trace group for SpanId: {}", rawSpan.getSpanId()); - }); - } - - entryIterator.remove(); - } - } - if (recordsToFlush.size() > 0) { - LOG.info("Flushing {} records due to GC", recordsToFlush.size()); - } - } finally { - traceFlushLock.unlock(); - } - } - } - - return recordsToFlush; - } - - private boolean shouldGarbageCollect() { - return System.currentTimeMillis() - lastTraceFlushTime >= traceFlushInterval || isShuttingDown; - } - - /** - * Forces a flush of all spans in memory - */ - @Override - public void prepareForShutdown() { - boolean isLockAcquired = prepareForShutdownLock.tryLock(); - - if (isLockAcquired) { - try { - LOG.info("Preparing for shutdown, will attempt to flush {} spans", traceIdRawSpanSetMap.size()); - isShuttingDown = true; - } finally { - prepareForShutdownLock.unlock(); - } - } - } - - @Override - public boolean isReadyForShutdown() { - return traceIdRawSpanSetMap.isEmpty(); - } - - @Override - public void shutdown() { - traceIdTraceGroupCache.cleanUp(); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java deleted file mode 100644 index fc71cfda79..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OtelTraceRawPrepperConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace; - -public class OtelTraceRawPrepperConfig { - static final String TRACE_FLUSH_INTERVAL = "trace_flush_interval"; - static final long DEFAULT_TG_FLUSH_INTERVAL_SEC = 180L; - static final long DEFAULT_TRACE_ID_TTL_SEC = 15L; - static final long MAX_TRACE_ID_CACHE_SIZE = 1000_000L; -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java deleted file mode 100644 index 4f43aa7ac8..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelper.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.Span; -import io.opentelemetry.proto.trace.v1.Status; - -import java.time.Instant; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -public final class OTelProtoHelper { - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final String SERVICE_NAME = "service.name"; - private static final String SPAN_ATTRIBUTES = "span.attributes"; - static final String RESOURCE_ATTRIBUTES = "resource.attributes"; - static final String INSTRUMENTATION_LIBRARY_NAME = "instrumentationLibrary.name"; - static final String INSTRUMENTATION_LIBRARY_VERSION = "instrumentationLibrary.version"; - static final String STATUS_CODE = "status.code"; - static final String STATUS_MESSAGE = "status.message"; - /** - * To make it ES friendly we will replace '.' in keys with '@' in all the Keys in {@link io.opentelemetry.proto.common.v1.KeyValue} - */ - private static final String DOT = "."; - private static final String AT = "@"; - public static final Function REPLACE_DOT_WITH_AT = i -> i.replace(DOT, AT); - /** - * Span and Resource attributes are essential for OpenSearch so they should not be nested. SO we will prefix them with "span.attributes" - * and "resource.attributes". - * - */ - public static final Function SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> SPAN_ATTRIBUTES + DOT + i.replace(DOT, AT); - public static final Function RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT = i -> RESOURCE_ATTRIBUTES + DOT + i.replace(DOT, AT); - - - public static Object convertAnyValue(final AnyValue value) { - switch (value.getValueCase()) { - case VALUE_NOT_SET: - case STRING_VALUE: - return value.getStringValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case INT_VALUE: - return value.getIntValue(); - case DOUBLE_VALUE: - return value.getDoubleValue(); - /** - * Both {@link AnyValue.ARRAY_VALUE_FIELD_NUMBER} and {@link AnyValue.KVLIST_VALUE_FIELD_NUMBER} are - * nested objects. Storing them in flatten structure is not OpenSearch friendly. So they are stored - * as Json string. - */ - case ARRAY_VALUE: - try { - return OBJECT_MAPPER.writeValueAsString(value.getArrayValue().getValuesList().stream() - .map(OTelProtoHelper::convertAnyValue).collect(Collectors.toList())); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - case KVLIST_VALUE: - try { - return OBJECT_MAPPER.writeValueAsString(value.getKvlistValue().getValuesList().stream() - .collect(Collectors.toMap(i -> REPLACE_DOT_WITH_AT.apply(i.getKey()), - i ->convertAnyValue(i.getValue())))); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - default: - throw new RuntimeException("Unknown case"); - } - } - - public static Map getSpanAttributes(final Span span) { - return span.getAttributesList().stream().collect(Collectors.toMap(i -> SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - public static Map getResourceAttributes(final Resource resource) { - return resource.getAttributesList().stream().collect(Collectors.toMap(i -> RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - public static Map getLinkAttributes(final Span.Link link) { - return link.getAttributesList().stream().collect(Collectors.toMap(i -> REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - public static Map getEventAttributes(final Span.Event event) { - return event.getAttributesList().stream().collect(Collectors.toMap(i -> REPLACE_DOT_WITH_AT.apply(i.getKey()), i -> convertAnyValue(i.getValue()))); - } - - /** - * Trace group represent root span i.e the span which triggers the trace event with the microservice architecture. This field is not part of the OpenTelemetry Spec. - * This field is something specific to Trace Analytics feature that OpenSearch Dashboards will be supporting. This field is derived from the opentelemetry spec and set as below, - *

- * if (this.parentSpanId.isEmpty()) { - * traceGroup = this.name; - * } - *

- * Note: The reason this method is part of the helper class is because the trace group definition will be expanded in the future when we support Links in OpenSearch Dashboards Trace Analytics. - */ - public static TraceGroup getTraceGroup(final Span span) { - final TraceGroup.TraceGroupBuilder traceGroupBuilder = new TraceGroup.TraceGroupBuilder(); - if (span.getParentSpanId().isEmpty()) { - traceGroupBuilder.setFromSpan(span); - } - return traceGroupBuilder.build(); - } - - - public static Map getInstrumentationLibraryAttributes(final InstrumentationLibrary instrumentationLibrary) { - final Map instrumentationAttr = new HashMap<>(); - if (!instrumentationLibrary.getName().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_NAME, instrumentationLibrary.getName()); - } - if (!instrumentationLibrary.getVersion().isEmpty()) { - instrumentationAttr.put(INSTRUMENTATION_LIBRARY_VERSION, instrumentationLibrary.getVersion()); - } - return instrumentationAttr; - } - - public static Map getSpanStatusAttributes(final Status status) { - final Map statusAttr = new HashMap<>(); - statusAttr.put(STATUS_CODE, status.getCodeValue()); - if(!status.getMessage().isEmpty()){ - statusAttr.put(STATUS_MESSAGE, status.getMessage()); - } - return statusAttr; - } - - private static String convertUnixNanosToISO8601(final long unixNano) { - return Instant.ofEpochSecond(0L, unixNano).toString(); - } - - public static String getStartTimeISO8601(final Span span) { - return convertUnixNanosToISO8601(span.getStartTimeUnixNano()); - } - - public static String getEndTimeISO8601(final Span span) { - return convertUnixNanosToISO8601(span.getEndTimeUnixNano()); - } - - public static String getTimeISO8601(final Span.Event event) { - return convertUnixNanosToISO8601(event.getTimeUnixNano()); - } - - public static Optional getServiceName(final Resource resource) { - return resource.getAttributesList().stream().filter( - keyValue -> keyValue.getKey().equals(SERVICE_NAME) - && !keyValue.getValue().getStringValue().isEmpty() - ).findFirst().map(i -> i.getValue().getStringValue()); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEvent.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEvent.java deleted file mode 100644 index 13d6d6d5c9..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEvent.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import io.opentelemetry.proto.trace.v1.Span; - -import java.util.Map; - -/** - * Java POJO of https://github.com/open-telemetry/opentelemetry-proto/blob/master/opentelemetry/proto/trace/v1/trace.proto#L169 - * which is compatible with OpenSearch - */ -public final class RawEvent { - private final String time; - private final String name; - private final Map attributes; - private final int droppedAttributesCount; - - public String getTime() { - return time; - } - - public String getName() { - return name; - } - - public Map getAttributes() { - return attributes; - } - - public int getDroppedAttributesCount() { - return droppedAttributesCount; - } - - private RawEvent(final String time, final String name, final Map attributes, int droppedAttributesCount) { - this.time = time; - this.name = name; - this.attributes = attributes; - this.droppedAttributesCount = droppedAttributesCount; - } - - public static RawEvent buildRawEvent(final Span.Event event) { - return new RawEvent(OTelProtoHelper.getTimeISO8601(event), event.getName(), OTelProtoHelper.getEventAttributes(event), event.getDroppedAttributesCount()); - } - -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLink.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLink.java deleted file mode 100644 index 7b1fd73001..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLink.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import io.opentelemetry.proto.trace.v1.Span; -import org.apache.commons.codec.binary.Hex; - -import java.util.Map; - -/** - * Java POJO of https://github.com/open-telemetry/opentelemetry-proto/blob/master/opentelemetry/proto/trace/v1/trace.proto#L196 - * which is compatible with OpenSearch - */ -public final class RawLink { - - /** - * HexString representation of the trace_id in the @see OpenTelemetry spec - */ - private final String traceId; - /** - * HexString representation of the span_id in the @see OpenTelemetry spec - */ - private final String spanId; - - private final String traceState; - private final Map attributes; - - private final int droppedAttributesCount; - - public String getTraceId() { - return traceId; - } - - public String getSpanId() { - return spanId; - } - - public String getTraceState() { - return traceState; - } - - public Map getAttributes() { - return attributes; - } - - public int getDroppedAttributesCount() { - return droppedAttributesCount; - } - - private RawLink(String traceId, String spanId, String traceState, Map attributes, int droppedAttributesCount) { - this.traceId = traceId; - this.spanId = spanId; - this.traceState = traceState; - this.attributes = attributes; - this.droppedAttributesCount = droppedAttributesCount; - } - - public static RawLink buildRawLink(final Span.Link link) { - return new RawLink(Hex.encodeHexString(link.getTraceId().toByteArray()), - Hex.encodeHexString(link.getSpanId().toByteArray()), - link.getTraceState(), - OTelProtoHelper.getLinkAttributes(link), - link.getDroppedAttributesCount()); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java deleted file mode 100644 index 79fe42aab7..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpan.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.List; -import java.util.Map; - - -public final class RawSpan { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - /** - * HexString representation of the trace_id in the @see OpenTelemetry spec - */ - private final String traceId; - /** - * HexString representation of the span_id in the @see OpenTelemetry spec - */ - private final String spanId; - /** - * trace_state in the @see OpenTelemetry spec - * This is w3c information field set when different vendors are used. - */ - private final String traceState; - /** - * HexString representation of the parent_span_id in the @see OpenTelemetry spec - */ - private final String parentSpanId; - /** - * String description of the span's operation. - * name in the the see OpenTelemetry spec - */ - private final String name; - /** - * String representation of the span context in the OpenTelemetry spec - */ - private final String kind; - /** - * ISO8601 representation of the start_time_in_unix_nano in the OpenTelemetry spec - */ - private final String startTime; - /** - * ISO8601 representation of the end_time_in_unix_nano in the OpenTelemetry spec - */ - private final String endTime; - /** - * Duration is calculated from startTime and endTime in nanos. This is a friendly field to make trace analytics OpenSearch Dashboards UI. - */ - private final long durationInNanos; - /** - * Extract the serviceName from the span. - */ - private final String serviceName; - /** - * Collection of key-value pairs related to the span.attributes, resource.attributes, span.status and instrumentationLibrary - * This is done to avoid nested objects and support OpenSearch Dashboards query and aggregations. - */ - private final Map attributes; - /** - * Collection of events related to the span. - * events field in the OpenTelemetry spec - */ - private final List events; - /** - * Collection of links related to the span. - * links field in the OpenTelemetry spec - */ - private final List links; - - /** - * dropped_attributes_count in the OpenTelemetry Spec - */ - private final int droppedAttributesCount; - - /** - * dropped_events_count in the OpenTelemetry Spec - */ - private final int droppedEventsCount; - - /** - * dropped_links_count in the OpenTelemetry Spec - */ - private final int droppedLinksCount; - - /** - * Trace group represent root span i.e the span which triggers the trace event with the microservice architecture. This field is not part of the OpenTelemetry Spec. - * This field is something specific to Trace Analytics feature that OpenSearch Dashboards will be supporting. This field is derived from the opentelemetry spec and set as below, - *

- * if (this.parentSpanId.isEmpty()) { - * traceGroup = this.name; - * } - */ - private TraceGroup traceGroup; - - - public String getTraceId() { - return traceId; - } - - public String getSpanId() { - return spanId; - } - - public String getParentSpanId() { - return parentSpanId; - } - - public String getServiceName() { - return serviceName; - } - - public String getStartTime() { - return startTime; - } - - public String getEndTime() { - return endTime; - } - - public String getName() { - return name; - } - - public String getTraceState() { - return traceState; - } - - public String getKind() { - return kind; - } - - public long getDurationInNanos() { - return durationInNanos; - } - - public List getEvents() { - return events; - } - - public List getLinks() { - return links; - } - - public int getDroppedAttributesCount() { - return droppedAttributesCount; - } - - public int getDroppedEventsCount() { - return droppedEventsCount; - } - - public int getDroppedLinksCount() { - return droppedLinksCount; - } - - public void setTraceGroup(TraceGroup traceGroup) { - this.traceGroup = traceGroup; - } - - RawSpan(RawSpanBuilder builder) { - this.traceId = builder.traceId; - this.spanId = builder.spanId; - this.traceState = builder.traceState; - this.parentSpanId = builder.parentSpanId; - this.kind = builder.kind; - this.name = builder.name; - this.startTime = builder.startTime; - this.endTime = builder.endTime; - this.durationInNanos = builder.durationInNanos; - this.serviceName = builder.serviceName; - this.attributes = builder.attributes; - this.events = builder.events; - this.links = builder.links; - this.droppedAttributesCount = builder.droppedAttributesCount; - this.droppedEventsCount = builder.droppedEventsCount; - this.droppedLinksCount = builder.droppedLinksCount; - this.traceGroup = builder.traceGroup; - } - - @JsonAnyGetter - public Map getAttributes() { - return attributes; - } - - @JsonUnwrapped - public TraceGroup getTraceGroup() { - return traceGroup; - } - - public String toJson() throws JsonProcessingException { - return OBJECT_MAPPER.writeValueAsString(this); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanBuilder.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanBuilder.java deleted file mode 100644 index 4e3c6b9625..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanBuilder.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.trace.v1.Span; -import org.apache.commons.codec.binary.Hex; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - - -public final class RawSpanBuilder { - private static final String SERVICE_NAME = "service.name"; - - String traceId; - String spanId; - String traceState; - String parentSpanId; - String name; - String kind; - String startTime; - String endTime; - long durationInNanos; - String serviceName; - Map attributes; - List events; - List links; - int droppedAttributesCount; - int droppedEventsCount; - int droppedLinksCount; - TraceGroup traceGroup; - - - public RawSpanBuilder() { - } - - private RawSpanBuilder setTraceId(final String traceId) { - this.traceId = traceId; - return this; - } - - private RawSpanBuilder setSpanId(final String spanId) { - this.spanId = spanId; - return this; - } - - private RawSpanBuilder setTraceState(final String traceState) { - this.traceState = traceState; - return this; - } - - private RawSpanBuilder setParentSpanId(final String parentSpanId) { - this.parentSpanId = parentSpanId; - return this; - } - - private RawSpanBuilder setName(final String name) { - this.name = name; - return this; - } - - private RawSpanBuilder setKind(final String kind) { - this.kind = kind; - return this; - } - - private RawSpanBuilder setStartTime(final String startTime) { - this.startTime = startTime; - return this; - } - - private RawSpanBuilder setEndTime(final String endTime) { - this.endTime = endTime; - return this; - } - - private RawSpanBuilder setDurationInNanos(final long durationInNanos) { - this.durationInNanos = durationInNanos; - return this; - } - - private RawSpanBuilder setServiceName(final String serviceName) { - this.serviceName = serviceName; - return this; - } - - private RawSpanBuilder setDroppedAttributesCount(final int droppedAttributesCount) { - this.droppedAttributesCount = droppedAttributesCount; - return this; - } - - private RawSpanBuilder setDroppedEventsCount(final int droppedEventsCount) { - this.droppedEventsCount = droppedEventsCount; - return this; - } - - private RawSpanBuilder setDroppedLinksCount(final int droppedLinksCount) { - this.droppedLinksCount = droppedLinksCount; - return this; - } - - private RawSpanBuilder setTraceGroup(final TraceGroup traceGroup) { - this.traceGroup = traceGroup; - return this; - } - - - private RawSpanBuilder setSpanAttributes(final Map spanAttributes, - final Map resourceAttributes, - final Map instrumentationAttributes, - final Map statusAttributes) { - this.attributes = new HashMap<>(); - this.attributes.putAll(spanAttributes); - this.attributes.putAll(resourceAttributes); - this.attributes.putAll(instrumentationAttributes); - this.attributes.putAll(statusAttributes); - return this; - } - - private RawSpanBuilder setEvents(final List events) { - this.events = events; - return this; - } - - private RawSpanBuilder setLinks(final List links) { - this.links = links; - return this; - } - - public RawSpan build() { - return new RawSpan(this); - } - - - public RawSpanBuilder setFromSpan(final Span span, final InstrumentationLibrary instrumentationLibrary, final String serviceName, final Map resourceAttributes) { - return this - .setTraceId(Hex.encodeHexString(span.getTraceId().toByteArray())) - .setSpanId(Hex.encodeHexString(span.getSpanId().toByteArray())) - .setTraceState(span.getTraceState()) - .setParentSpanId(Hex.encodeHexString(span.getParentSpanId().toByteArray())) - .setName(span.getName()) - .setKind(span.getKind().name()) - .setStartTime(OTelProtoHelper.getStartTimeISO8601(span)) - .setEndTime(OTelProtoHelper.getEndTimeISO8601(span)) - .setDurationInNanos(span.getEndTimeUnixNano() - span.getStartTimeUnixNano()) - .setServiceName(serviceName) - .setSpanAttributes(OTelProtoHelper.getSpanAttributes(span), - resourceAttributes, - OTelProtoHelper.getInstrumentationLibraryAttributes(instrumentationLibrary), - OTelProtoHelper.getSpanStatusAttributes(span.getStatus())) - .setEvents(span.getEventsList().stream().map(RawEvent::buildRawEvent).collect(Collectors.toList())) - .setLinks(span.getLinksList().stream().map(RawLink::buildRawLink).collect(Collectors.toList())) - .setDroppedAttributesCount(span.getDroppedAttributesCount()) - .setDroppedEventsCount(span.getDroppedEventsCount()) - .setDroppedLinksCount(span.getDroppedLinksCount()) - .setTraceGroup(OTelProtoHelper.getTraceGroup(span)); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanSet.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanSet.java deleted file mode 100644 index d4a25c6677..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawSpanSet.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.google.common.collect.Sets; - -import java.util.Set; - -public class RawSpanSet { - - private final Set rawSpans; - private final long timeSeen; - - public RawSpanSet() { - this.rawSpans = Sets.newConcurrentHashSet(); - this.timeSeen = System.currentTimeMillis(); - } - - public Set getRawSpans() { - return rawSpans; - } - - public long getTimeSeen() { - return timeSeen; - } - - public void addRawSpan(final RawSpan rawSpan) { - rawSpans.add(rawSpan); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java b/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java deleted file mode 100644 index 1bb82b4050..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/main/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TraceGroup.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import io.opentelemetry.proto.trace.v1.Span; - -import java.util.Objects; - -public class TraceGroup { - @JsonProperty("traceGroup") - private final String name; - @JsonProperty("traceGroupFields.endTime") - private final String endTime; - @JsonProperty("traceGroupFields.statusCode") - private final Integer statusCode; - @JsonProperty("traceGroupFields.durationInNanos") - private final Long durationInNanos; - - public String getName() { - return name; - } - - public String getEndTime() { - return endTime; - } - - public Integer getStatusCode() { - return statusCode; - } - - public Long getDurationInNanos() { - return durationInNanos; - } - - TraceGroup(TraceGroupBuilder traceGroupBuilder) { - name = traceGroupBuilder.name; - endTime = traceGroupBuilder.endTime; - statusCode = traceGroupBuilder.statusCode; - durationInNanos = traceGroupBuilder.durationInNanos; - } - - @Override - public boolean equals(final Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - final TraceGroup that = (TraceGroup) o; - return Objects.equals(name, that.name) && - Objects.equals(endTime, that.endTime) && - Objects.equals(statusCode, that.statusCode) && - Objects.equals(durationInNanos, that.durationInNanos); - } - - @Override - public int hashCode() { - return Objects.hash(name, endTime, statusCode, durationInNanos); - } - - public static class TraceGroupBuilder { - private String name; - private String endTime; - private Integer statusCode; - private Long durationInNanos; - - private TraceGroupBuilder setName(final String name) { - this.name = name; - return this; - } - - private TraceGroupBuilder setEndTime(final String endTime) { - this.endTime = endTime; - return this; - } - - private TraceGroupBuilder setStatusCode(final Integer statusCode) { - this.statusCode = statusCode; - return this; - } - - private TraceGroupBuilder setDurationInNanos(final Long durationInNanos) { - this.durationInNanos = durationInNanos; - return this; - } - - public TraceGroup build() { - return new TraceGroup(this); - } - - public TraceGroupBuilder setFromSpan(final Span span) { - return this - .setName(span.getName()) - .setDurationInNanos(span.getEndTimeUnixNano() - span.getStartTimeUnixNano()) - .setEndTime(OTelProtoHelper.getEndTimeISO8601(span)) - .setStatusCode(span.getStatus().getCodeValue()); - } - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java deleted file mode 100644 index 92ab4c84f7..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/OTelTraceRawPrepperTest.java +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace; - -import com.amazon.dataprepper.metrics.MetricNames; -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.metrics.MetricsTestUtil; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.util.JsonFormat; -import io.micrometer.core.instrument.Measurement; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.trace.v1.ResourceSpans; -import org.assertj.core.api.Assertions; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.junit.Before; - -import static org.awaitility.Awaitility.await; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.StringJoiner; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -public class OTelTraceRawPrepperTest { - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final long TEST_TRACE_FLUSH_INTERVAL = 3L; - private static final int TEST_CONCURRENCY_SCALE = 2; - - private static final String TEST_REQUEST_ONE_FULL_TRACE_GROUP_JSON_FILE = "sample-request-one-full-trace-group.json"; - private static final String TEST_REQUEST_ONE_TRACE_GROUP_MISSING_ROOT_JSON_FILE = "sample-request-one-trace-group-missing-root.json"; - private static final String TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE = "sample-request-two-full-trace-group.json"; - private static final String TEST_REQUEST_TWO_TRACE_GROUP_INTERLEAVED_JSON_FILE_1 = "sample-request-two-trace-group-interleaved-1.json"; - private static final String TEST_REQUEST_TWO_TRACE_GROUP_INTERLEAVED_JSON_FILE_2 = "sample-request-two-trace-group-interleaved-2.json"; - private static final String TEST_REQUEST_TWO_TRACE_GROUP_MISSING_ROOTS_JSON_FILE = "sample-request-two-trace-group-missing-roots.json"; - - PluginSetting pluginSetting; - public OTelTraceRawPrepper oTelTraceRawPrepper; - public ExecutorService executorService; - - @Before - public void setup() { - MetricsTestUtil.initMetrics(); - pluginSetting = new PluginSetting( - "OTelTrace", - new HashMap() {{ - put(OtelTraceRawPrepperConfig.TRACE_FLUSH_INTERVAL, TEST_TRACE_FLUSH_INTERVAL); - }}); - pluginSetting.setPipelineName("pipelineOTelTrace"); - pluginSetting.setProcessWorkers(TEST_CONCURRENCY_SCALE); - oTelTraceRawPrepper = new OTelTraceRawPrepper(pluginSetting); - executorService = Executors.newFixedThreadPool(TEST_CONCURRENCY_SCALE); - } - - @After - public void tearDown() { - oTelTraceRawPrepper.shutdown(); - executorService.shutdown(); - } - - @Test - public void testResourceSpansProcessingErrorMetrics() { - ExportTraceServiceRequest mockData = mock(ExportTraceServiceRequest.class); - Record record = new Record(mockData); - ResourceSpans mockResourceSpans = mock(ResourceSpans.class); - List mockResourceSpansList = Collections.singletonList(mockResourceSpans); - - when(mockData.getResourceSpansList()).thenReturn(mockResourceSpansList); - when(mockResourceSpans.getResource()).thenThrow(new RuntimeException()); - - oTelTraceRawPrepper.doExecute(Collections.singletonList(record)); - - final List resourceSpansErrorsMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("pipelineOTelTrace").add("OTelTrace") - .add(OTelTraceRawPrepper.RESOURCE_SPANS_PROCESSING_ERRORS).toString()); - final List totalErrorsMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("pipelineOTelTrace").add("OTelTrace") - .add(OTelTraceRawPrepper.TOTAL_PROCESSING_ERRORS).toString()); - - Assert.assertEquals(1, resourceSpansErrorsMeasurement.size()); - Assert.assertEquals(1.0, resourceSpansErrorsMeasurement.get(0).getValue(), 0); - Assert.assertEquals(1, totalErrorsMeasurement.size()); - Assert.assertEquals(1.0, totalErrorsMeasurement.get(0).getValue(), 0); - } - - @Test - public void testEmptyCollection() { - assertThat(oTelTraceRawPrepper.doExecute(Collections.EMPTY_LIST)).isEmpty(); - } - - @Test - public void testEmptyTraceRequests() { - assertThat(oTelTraceRawPrepper.doExecute(Arrays.asList(new Record<>(ExportTraceServiceRequest.newBuilder().build()), - new Record<>(ExportTraceServiceRequest.newBuilder().build())))).isEmpty(); - } - - @Test - public void testEmptySpans() { - assertThat(oTelTraceRawPrepper.doExecute(Arrays.asList(new Record<>(ExportTraceServiceRequest.newBuilder().build()), - new Record<>(ExportTraceServiceRequest.newBuilder().build())))).isEmpty(); - } - - @Test - public void testExportRequestFlushByParentSpan() throws IOException { - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_FULL_TRACE_GROUP_JSON_FILE); - final List> processedRecords = (List>)oTelTraceRawPrepper.doExecute( - Collections.singletonList(new Record<>(exportTraceServiceRequest)) - ); - - Assertions.assertThat(processedRecords.size()).isEqualTo(6); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); - } - - @Test - public void testExportRequestFlushByParentSpanMultiThread() throws IOException, InterruptedException, ExecutionException { - final ExportTraceServiceRequest exportTraceServiceRequest1 = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_INTERLEAVED_JSON_FILE_1); - final ExportTraceServiceRequest exportTraceServiceRequest2 = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_INTERLEAVED_JSON_FILE_2); - final List> processedRecords = new ArrayList<>(); - List>>> futures = new ArrayList<>(); - futures.addAll(submitExportTraceServiceRequests(Collections.singletonList(exportTraceServiceRequest1))); - futures.addAll(submitExportTraceServiceRequests(Collections.singletonList(exportTraceServiceRequest2))); - for (Future>> future : futures) { - processedRecords.addAll(future.get()); - } - await().atMost(2 * TEST_TRACE_FLUSH_INTERVAL, TimeUnit.SECONDS).untilAsserted(() -> { - List>>> futureList = submitExportTraceServiceRequests(Collections.emptyList()); - for (Future>> future : futureList) { - processedRecords.addAll(future.get()); - } - Assertions.assertThat(processedRecords.size()).isEqualTo(6); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(0); - }); - } - - @Test - public void testExportRequestFlushByGC() throws IOException { - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_MISSING_ROOTS_JSON_FILE); - oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); - await().atMost(2 * TEST_TRACE_FLUSH_INTERVAL, TimeUnit.SECONDS).untilAsserted(() -> { - final List> processedRecords = (List>) oTelTraceRawPrepper.doExecute(Collections.emptyList()); - Assertions.assertThat(processedRecords.size()).isEqualTo(4); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(4); - }); - } - - @Test - public void testExportRequestFlushByMixedMultiThread() throws IOException, InterruptedException, ExecutionException { - final ExportTraceServiceRequest exportTraceServiceRequest1 = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_ONE_FULL_TRACE_GROUP_JSON_FILE); - final ExportTraceServiceRequest exportTraceServiceRequest2 = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_ONE_TRACE_GROUP_MISSING_ROOT_JSON_FILE); - List> processedRecords = new ArrayList<>(); - List>>> futures = new ArrayList<>(); - futures.addAll(submitExportTraceServiceRequests(Collections.singletonList(exportTraceServiceRequest1))); - futures.addAll(submitExportTraceServiceRequests(Collections.singletonList(exportTraceServiceRequest2))); - for (Future>> future : futures) { - processedRecords.addAll(future.get()); - } - await().atMost(2 * TEST_TRACE_FLUSH_INTERVAL, TimeUnit.SECONDS).untilAsserted(() -> { - List>>> futureList = submitExportTraceServiceRequests(Collections.emptyList()); - for (Future>> future : futureList) { - processedRecords.addAll(future.get()); - } - Assertions.assertThat(processedRecords.size()).isEqualTo(5); - Assertions.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords)).isEqualTo(2); - }); - } - - @Test - public void testPrepareForShutdown() throws Exception { - // Assert no records in memory - assertTrue(oTelTraceRawPrepper.isReadyForShutdown()); - - // Add records to memory/queue - final ExportTraceServiceRequest exportTraceServiceRequest = buildExportTraceServiceRequestFromJsonFile(TEST_REQUEST_TWO_TRACE_GROUP_MISSING_ROOTS_JSON_FILE); - oTelTraceRawPrepper.doExecute(Collections.singletonList(new Record<>(exportTraceServiceRequest))); - - // Assert records exist in memory - assertFalse(oTelTraceRawPrepper.isReadyForShutdown()); - - // Force records to be flushed - oTelTraceRawPrepper.prepareForShutdown(); - oTelTraceRawPrepper.doExecute(Collections.emptyList()); - - // Assert records have been flushed - assertTrue(oTelTraceRawPrepper.isReadyForShutdown()); - } - - private ExportTraceServiceRequest buildExportTraceServiceRequestFromJsonFile(String requestJsonFileName) throws IOException { - final StringBuilder jsonBuilder = new StringBuilder(); - try (final InputStream inputStream = Objects.requireNonNull( - getClass().getClassLoader().getResourceAsStream(requestJsonFileName))){ - final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); - bufferedReader.lines().forEach(jsonBuilder::append); - } - final String requestJson = jsonBuilder.toString(); - final ExportTraceServiceRequest.Builder builder = ExportTraceServiceRequest.newBuilder(); - JsonFormat.parser().merge(requestJson, builder); - return builder.build(); - } - - private List>>> submitExportTraceServiceRequests(Collection exportTraceServiceRequests) { - final List>>> futures = new ArrayList<>(); - final List> records = exportTraceServiceRequests.stream().map(Record::new).collect(Collectors.toList()); - futures.add(executorService.submit(() -> oTelTraceRawPrepper.doExecute(records))); - return futures; - } - - private int getMissingTraceGroupFieldsSpanCount(List> records) throws JsonProcessingException { - int count = 0; - for (Record record: records) { - final String spanJson = record.getData(); - Map spanMap = OBJECT_MAPPER.readValue(spanJson, new TypeReference>() {}); - final String traceGroupName = (String) spanMap.get("traceGroup"); - final String traceGroupEndTime = (String) spanMap.get("traceGroupFields.endTime"); - final Number traceGroupDurationInNanos = (Number) spanMap.get("traceGroupFields.durationInNanos"); - final Number traceGroupStatusCode = (Number) spanMap.get("traceGroupFields.statusCode"); - if (Stream.of(traceGroupName, traceGroupEndTime, traceGroupDurationInNanos, traceGroupStatusCode).allMatch(Objects::isNull)) { - count += 1; - } - } - return count; - } -} - diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java deleted file mode 100644 index a5aeb7ebff..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/OTelProtoHelperTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.ArrayValue; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.common.v1.KeyValueList; -import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.Span; -import io.opentelemetry.proto.trace.v1.Status; -import org.junit.Test; - -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; - -public class OTelProtoHelperTest { - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - - private Map returnMap(final String jsonStr) throws JsonProcessingException { - return (Map) OBJECT_MAPPER.readValue(jsonStr, Map.class); - } - - private List returnList(final String jsonStr) throws JsonProcessingException { - return (List) OBJECT_MAPPER.readValue(jsonStr, List.class); - } - - /** - * Below object has a KeyValue with a key mapped to KeyValueList and is part of the span attributes - * - * @throws JsonProcessingException - */ - @Test - public void testKeyValueListAsSpanAttributes() throws JsonProcessingException { - - final KeyValue childAttr1 = KeyValue.newBuilder().setKey("statement").setValue(AnyValue.newBuilder() - .setIntValue(1_000).build()).build(); - final KeyValue childAttr2 = KeyValue.newBuilder().setKey("statement.params").setValue(AnyValue.newBuilder() - .setStringValue("us-east-1").build()).build(); - final KeyValue spanAttribute1 = KeyValue.newBuilder().setKey("db.details").setValue(AnyValue.newBuilder() - .setKvlistValue(KeyValueList.newBuilder().addAllValues(Arrays.asList(childAttr1, childAttr2)).build()).build()).build(); - final KeyValue spanAttribute2 = KeyValue.newBuilder().setKey("http.status").setValue(AnyValue.newBuilder() - .setStringValue("4xx").build()).build(); - - final Map actual = OTelProtoHelper.getSpanAttributes(Span.newBuilder() - .addAllAttributes(Arrays.asList(spanAttribute1, spanAttribute2)).build()); - assertThat(actual.get(OTelProtoHelper.SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute2.getKey())) - .equals(spanAttribute2.getValue().getStringValue())).isTrue(); - assertThat(actual.containsKey(OTelProtoHelper.SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))).isTrue(); - final Map actualValue = returnMap((String) actual - .get(OTelProtoHelper.SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))); - assertThat((Integer) actualValue.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr1.getKey())) == childAttr1.getValue().getIntValue()).isTrue(); - assertThat(actualValue.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr2.getKey())).equals(childAttr2.getValue().getStringValue())).isTrue(); - } - - /** - * Below object has a KeyValue with a key mapped to KeyValueList and is part of the resource attributes - * - * @throws JsonProcessingException - */ - @Test - public void testKeyValueListAsResourceAttributes() throws JsonProcessingException { - final KeyValue childAttr1 = KeyValue.newBuilder().setKey("ec2.instances").setValue(AnyValue.newBuilder() - .setIntValue(20).build()).build(); - final KeyValue childAttr2 = KeyValue.newBuilder().setKey("ec2.instance.az").setValue(AnyValue.newBuilder() - .setStringValue("us-east-1").build()).build(); - final KeyValue spanAttribute1 = KeyValue.newBuilder().setKey("aws.details").setValue(AnyValue.newBuilder() - .setKvlistValue(KeyValueList.newBuilder().addAllValues(Arrays.asList(childAttr1, childAttr2)).build()).build()).build(); - final KeyValue spanAttribute2 = KeyValue.newBuilder().setKey("service.name").setValue(AnyValue.newBuilder() - .setStringValue("EaglesService").build()).build(); - - final Map actual = OTelProtoHelper.getResourceAttributes(Resource.newBuilder() - .addAllAttributes(Arrays.asList(spanAttribute1, spanAttribute2)).build()); - assertThat(actual.get(OTelProtoHelper.RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute2.getKey())) - .equals(spanAttribute2.getValue().getStringValue())).isTrue(); - assertThat(actual.containsKey(OTelProtoHelper.RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))).isTrue(); - final Map actualValue = returnMap((String) actual - .get(OTelProtoHelper.RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))); - assertThat((Integer) actualValue.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr1.getKey())) == childAttr1.getValue().getIntValue()).isTrue(); - assertThat(actualValue.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr2.getKey())).equals(childAttr2.getValue().getStringValue())).isTrue(); - - } - - - /** - * Below object has a KeyValue with a key mapped to KeyValueList and is part of the span attributes - * - * @throws JsonProcessingException - */ - @Test - public void testArrayOfValueAsResourceAttributes() throws JsonProcessingException { - final KeyValue childAttr1 = KeyValue.newBuilder().setKey("ec2.instances").setValue(AnyValue.newBuilder() - .setIntValue(20).build()).build(); - final KeyValue childAttr2 = KeyValue.newBuilder().setKey("ec2.instance.az").setValue(AnyValue.newBuilder() - .setStringValue("us-east-1").build()).build(); - final AnyValue anyValue1 = AnyValue.newBuilder().setStringValue(UUID.randomUUID().toString()).build(); - final AnyValue anyValue2 = AnyValue.newBuilder().setDoubleValue(2000.123).build(); - final AnyValue anyValue3 = AnyValue.newBuilder().setKvlistValue(KeyValueList.newBuilder().addAllValues(Arrays.asList(childAttr1, childAttr2))).build(); - final ArrayValue arrayValue = ArrayValue.newBuilder().addAllValues(Arrays.asList(anyValue1, anyValue2, anyValue3)).build(); - final KeyValue spanAttribute1 = KeyValue.newBuilder().setKey("aws.details").setValue(AnyValue.newBuilder() - .setArrayValue(arrayValue)).build(); - - final Map actual = OTelProtoHelper.getResourceAttributes(Resource.newBuilder() - .addAllAttributes(Collections.singletonList(spanAttribute1)).build()); - assertThat(actual.containsKey(OTelProtoHelper.RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))).isTrue(); - final List actualValue = returnList((String) actual - .get(OTelProtoHelper.RESOURCE_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply(spanAttribute1.getKey()))); - assertThat((actualValue.get(0)).equals(anyValue1.getStringValue())).isTrue(); - assertThat(((Double) actualValue.get(1)) == (anyValue2.getDoubleValue())).isTrue(); - final Map map = returnMap((String) actualValue.get(2)); - assertThat((Integer) map.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr1.getKey())) == childAttr1.getValue().getIntValue()).isTrue(); - assertThat(map.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr2.getKey())).equals(childAttr2.getValue().getStringValue())).isTrue(); - assertThat((Integer) map.get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr1.getKey())) == (childAttr1.getValue().getIntValue())).isTrue(); - - } - - - @Test - public void testInstrumentationLibraryAttributes() { - final InstrumentationLibrary il1 = InstrumentationLibrary.newBuilder().setName("Jaeger").setVersion("0.6.0").build(); - final InstrumentationLibrary il2 = InstrumentationLibrary.newBuilder().setName("Jaeger").build(); - final InstrumentationLibrary il3 = InstrumentationLibrary.newBuilder().setVersion("0.6.0").build(); - final InstrumentationLibrary il4 = InstrumentationLibrary.newBuilder().build(); - - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il1).size() == 2).isTrue(); - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il1).get(OTelProtoHelper.INSTRUMENTATION_LIBRARY_NAME).equals(il1.getName())).isTrue(); - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il1).get(OTelProtoHelper.INSTRUMENTATION_LIBRARY_VERSION).equals(il1.getVersion())).isTrue(); - - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il2).size() == 1).isTrue(); - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il2).get(OTelProtoHelper.INSTRUMENTATION_LIBRARY_NAME).equals(il2.getName())).isTrue(); - - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il3).size() == 1).isTrue(); - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il3).get(OTelProtoHelper.INSTRUMENTATION_LIBRARY_VERSION).equals(il3.getVersion())).isTrue(); - - assertThat(OTelProtoHelper.getInstrumentationLibraryAttributes(il4).isEmpty()).isTrue(); - } - - @Test - public void testStatusAttributes() { - final Status st1 = Status.newBuilder().setCode(Status.StatusCode.STATUS_CODE_ERROR).setMessage("Some message").build(); - final Status st2 = Status.newBuilder().setMessage("error message").build(); - final Status st3 = Status.newBuilder().setCode(Status.StatusCode.STATUS_CODE_UNSET).build(); - final Status st4 = Status.newBuilder().build(); - - assertThat(OTelProtoHelper.getSpanStatusAttributes(st1).size() == 2).isTrue(); - assertThat(Status.StatusCode.forNumber((Integer) OTelProtoHelper.getSpanStatusAttributes(st1).get(OTelProtoHelper.STATUS_CODE)).equals(st1.getCode())).isTrue(); - assertThat(OTelProtoHelper.getSpanStatusAttributes(st1).get(OTelProtoHelper.STATUS_MESSAGE).equals(st1.getMessage())).isTrue(); - - assertThat(OTelProtoHelper.getSpanStatusAttributes(st2).size() == 2).isTrue(); - assertThat(Status.StatusCode.forNumber((Integer) OTelProtoHelper.getSpanStatusAttributes(st2).get(OTelProtoHelper.STATUS_CODE)).equals(st2.getCode())).isTrue(); - - assertThat(OTelProtoHelper.getSpanStatusAttributes(st3).size() == 1).isTrue(); - assertThat(Status.StatusCode.forNumber((Integer) OTelProtoHelper.getSpanStatusAttributes(st3).get(OTelProtoHelper.STATUS_CODE)).equals(st3.getCode())).isTrue(); - - assertThat(OTelProtoHelper.getSpanStatusAttributes(st4).size() == 1).isTrue(); - assertThat(Status.StatusCode.forNumber((Integer) OTelProtoHelper.getSpanStatusAttributes(st4).get(OTelProtoHelper.STATUS_CODE)).equals(st4.getCode())).isTrue(); - - } - - @Test - public void testISO8601() { - final long NANO_MULTIPLIER = 1_000 * 1_000 * 1_000; - final Span startTimeUnixNano = Span.newBuilder().setStartTimeUnixNano(651242400000000321L).build(); - final Span endTimeUnixNano = Span.newBuilder().setEndTimeUnixNano(1598013600000000321L).build(); - final Span emptyTimeSpan = Span.newBuilder().build(); - - final String startTime = OTelProtoHelper.getStartTimeISO8601(startTimeUnixNano); - assertThat(Instant.parse(startTime).getEpochSecond() * NANO_MULTIPLIER + Instant.parse(startTime).getNano() == startTimeUnixNano.getStartTimeUnixNano()).isTrue(); - final String endTime = OTelProtoHelper.getEndTimeISO8601(endTimeUnixNano); - assertThat(Instant.parse(endTime).getEpochSecond() * NANO_MULTIPLIER + Instant.parse(endTime).getNano() == endTimeUnixNano.getEndTimeUnixNano()).isTrue(); - final String emptyTime = OTelProtoHelper.getStartTimeISO8601(endTimeUnixNano); - assertThat(Instant.parse(emptyTime).getEpochSecond() * NANO_MULTIPLIER + Instant.parse(emptyTime).getNano() == emptyTimeSpan.getStartTimeUnixNano()).isTrue(); - - } - - @Test - public void testTraceGroup() { - final Span span1 = Span.newBuilder().setParentSpanId(ByteString.copyFrom("PArentIdExists", StandardCharsets.UTF_8)).build(); - assertThat(OTelProtoHelper.getTraceGroup(span1)).isEqualTo(new TraceGroup.TraceGroupBuilder().build()); - final Span span2 = Span.newBuilder().setName("TraceGroup").build(); - assertThat(OTelProtoHelper.getTraceGroup(span2)).isEqualTo(new TraceGroup.TraceGroupBuilder().setFromSpan(span2).build()); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java deleted file mode 100644 index fbeda1e091..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawBuilderTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.google.protobuf.ByteString; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.trace.v1.Span; -import io.opentelemetry.proto.trace.v1.Status; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.junit.Test; - -import java.util.Collections; - -import static org.assertj.core.api.Assertions.assertThat; - -public class RawBuilderTest { - @Test - public void testRawSpan() throws DecoderException { - final Span.Event event1 = Span.Event.newBuilder().setName("event-1").setTimeUnixNano(651242400000000321L + 1000).build(); - final Span.Event event2 = Span.Event.newBuilder().setName("event-2").setTimeUnixNano(651242400000000321L + 2000).setDroppedAttributesCount(0).build(); - final Span.Link link1 = Span.Link.newBuilder().setTraceId(ByteString.copyFrom(TestUtils.getRandomBytes(16))) - .setSpanId(ByteString.copyFrom(TestUtils.getRandomBytes(8))) - .build(); - final Span span = Span.newBuilder() - .setTraceId(ByteString.copyFrom(TestUtils.getRandomBytes(16))) - .setSpanId(ByteString.copyFrom(TestUtils.getRandomBytes(8))) - .setParentSpanId(ByteString.copyFrom(TestUtils.getRandomBytes(8))) - .setTraceState("some state") - .setName("test-span") - .setKind(Span.SpanKind.SPAN_KIND_CONSUMER) - .setStartTimeUnixNano(651242400000000321L) - .setEndTimeUnixNano(651242400000000321L + 3000) - .setStatus(Status.newBuilder().setCodeValue(Status.StatusCode.STATUS_CODE_UNSET_VALUE).setMessage("status-description").build()) - .addAttributes(KeyValue.newBuilder() - .setKey("some-key") - .build() - ) - .setDroppedAttributesCount(1) - .addEvents(event1) - .addEvents(event2) - .addLinks(link1) - .build(); - final RawSpan rawSpan = new RawSpanBuilder().setFromSpan(span, InstrumentationLibrary.newBuilder().build(), "some-service", Collections.EMPTY_MAP).build(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawSpan.getTraceId())).equals(span.getTraceId())).isTrue(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawSpan.getSpanId())).equals(span.getSpanId())).isTrue(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawSpan.getParentSpanId())).equals(span.getParentSpanId())).isTrue(); - assertThat(rawSpan.getTraceState().equals(span.getTraceState())).isTrue(); - assertThat(rawSpan.getName().equals(span.getName())).isTrue(); - assertThat(rawSpan.getKind().equals(span.getKind().name())).isTrue(); - assertThat(rawSpan.getDurationInNanos()).isEqualTo(3000); - assertThat(rawSpan.getStartTime().equals(OTelProtoHelper.getStartTimeISO8601(span))).isTrue(); - assertThat(rawSpan.getEndTime().equals(OTelProtoHelper.getEndTimeISO8601(span))).isTrue(); - assertThat(rawSpan.getAttributes().get(OTelProtoHelper.STATUS_MESSAGE)).isEqualTo("status-description"); - assertThat(rawSpan.getAttributes().get(OTelProtoHelper.STATUS_CODE)).isEqualTo(0); - assertThat(rawSpan.getAttributes().get(OTelProtoHelper.SPAN_ATTRIBUTES_REPLACE_DOT_WITH_AT.apply("some-key")).equals("")).isTrue(); - assertThat(rawSpan.getDroppedAttributesCount()).isEqualTo(1); - assertThat(rawSpan.getDroppedLinksCount()).isEqualTo(0); - assertThat(rawSpan.getDroppedEventsCount()).isEqualTo(0); - assertThat(rawSpan.getEvents().size()).isEqualTo(2); - assertThat(rawSpan.getLinks().size()).isEqualTo(1); - assertThat(rawSpan.getServiceName()).isEqualTo("some-service"); - assertThat(rawSpan.getTraceGroup()).isEqualTo(new TraceGroup.TraceGroupBuilder().build()); - } - - @Test - public void testForTraceGroupRawSpan() throws DecoderException { - final Span span = Span.newBuilder() - .setTraceId(ByteString.copyFrom(TestUtils.getRandomBytes(16))) - .setSpanId(ByteString.copyFrom(TestUtils.getRandomBytes(8))) - .setName("test-span") - .build(); - final RawSpan rawSpan = new RawSpanBuilder().setFromSpan(span, InstrumentationLibrary.newBuilder().build(), "some-service", Collections.EMPTY_MAP).build(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawSpan.getTraceId())).equals(span.getTraceId())).isTrue(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawSpan.getSpanId())).equals(span.getSpanId())).isTrue(); - assertThat(rawSpan.getTraceGroup()).isEqualTo(new TraceGroup.TraceGroupBuilder().setFromSpan(span).build()); - } - - /** - * You can submit empty object and the prepper will process it without any error - */ - @Test - public void testRawSpanEmpty() { - final RawSpan rawSpan = new RawSpanBuilder().setFromSpan(Span.newBuilder().build(), InstrumentationLibrary.newBuilder().build(), null, Collections.EMPTY_MAP).build(); - assertThat(rawSpan.getSpanId()).isEmpty(); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEventTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEventTest.java deleted file mode 100644 index d8abcb1be5..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawEventTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.trace.v1.Span; -import org.junit.Test; - -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; - -public class RawEventTest { - - - @Test - public void testRawEventEmpty() { - final Span.Event spanEventEmpty = Span.Event.newBuilder().build(); - final RawEvent rawEventWithEmptyAttri = RawEvent.buildRawEvent(spanEventEmpty); - assertThat(rawEventWithEmptyAttri.getAttributes().isEmpty()).isTrue(); - assertThat(rawEventWithEmptyAttri.getName()).isEmpty(); - assertThat(rawEventWithEmptyAttri.getTime().equals("1970-01-01T00:00:00Z")).isTrue(); - assertThat(rawEventWithEmptyAttri.getDroppedAttributesCount() == 0).isTrue(); - - } - - @Test - public void testRawEvent() { - final Span.Event spanEventWithEmptyAttri = Span.Event.newBuilder().setName("Error").setTimeUnixNano(1598013600000000321L).build(); - final RawEvent rawEventWithEmptyAttri = RawEvent.buildRawEvent(spanEventWithEmptyAttri); - assertThat(rawEventWithEmptyAttri.getAttributes().isEmpty()).isTrue(); - assertThat(rawEventWithEmptyAttri.getName().equals("Error")).isTrue(); - assertThat(rawEventWithEmptyAttri.getTime().equals("2020-08-21T12:40:00.000000321Z")).isTrue(); - } - - @Test - public void testRawEventWithAttributes() { - final KeyValue childAttr1 = KeyValue.newBuilder().setKey("statement").setValue(AnyValue.newBuilder() - .setIntValue(1_000).build()).build(); - final KeyValue childAttr2 = KeyValue.newBuilder().setKey("statement.params").setValue(AnyValue.newBuilder() - .setStringValue("us-east-1").build()).build(); - final Span.Event spanEvent = Span.Event.newBuilder().setName("Error").setTimeUnixNano(1598013600000000321L) - .addAllAttributes(Arrays.asList(childAttr1, childAttr2)).build(); - final RawEvent rawEvent = RawEvent.buildRawEvent(spanEvent); - assertThat(rawEvent.getAttributes().size() == 2).isTrue(); - assertThat((Long) rawEvent.getAttributes().get(childAttr1.getKey()) == childAttr1.getValue().getIntValue()).isTrue(); - assertThat(rawEvent.getAttributes().get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr2.getKey())).equals(childAttr2.getValue().getStringValue())).isTrue(); - assertThat(rawEvent.getName().equals("Error")).isTrue(); - assertThat(rawEvent.getTime().equals("2020-08-21T12:40:00.000000321Z")).isTrue(); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLinkTest.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLinkTest.java deleted file mode 100644 index e0d4d51c96..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/RawLinkTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import com.google.protobuf.ByteString; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.trace.v1.Span; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.junit.Test; - -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; - -public class RawLinkTest { - @Test - public void testRawLinkEmpty() { - final Span.Link spanLinkEmpty = Span.Link.newBuilder().build(); - final RawLink rawLink = RawLink.buildRawLink(spanLinkEmpty); - assertThat(rawLink.getAttributes().isEmpty()).isTrue(); - assertThat(rawLink.getSpanId()).isEmpty(); - assertThat(rawLink.getTraceId()).isEmpty(); - assertThat(rawLink.getTraceState()).isEmpty(); - assertThat(rawLink.getDroppedAttributesCount() == 0).isTrue(); - } - - @Test - public void testRawEventWithAttributes() throws DecoderException { - final KeyValue childAttr1 = KeyValue.newBuilder().setKey("statement").setValue(AnyValue.newBuilder() - .setIntValue(1_000).build()).build(); - final KeyValue childAttr2 = KeyValue.newBuilder().setKey("statement.params").setValue(AnyValue.newBuilder() - .setStringValue("us-east-1").build()).build(); - final Span.Link spanLink = Span.Link.newBuilder() - .setSpanId(ByteString.copyFrom(TestUtils.getRandomBytes(8))) - .setTraceId(ByteString.copyFrom(TestUtils.getRandomBytes(16))) - .setTraceState("Some State") - .addAllAttributes(Arrays.asList(childAttr1, childAttr2)).build(); - final RawLink rawLink = RawLink.buildRawLink(spanLink); - assertThat(rawLink.getAttributes().size() == 2).isTrue(); - assertThat((Long) rawLink.getAttributes().get(childAttr1.getKey()) == childAttr1.getValue().getIntValue()).isTrue(); - assertThat(rawLink.getAttributes().get(OTelProtoHelper.REPLACE_DOT_WITH_AT.apply(childAttr2.getKey())).equals(childAttr2.getValue().getStringValue())).isTrue(); - assertThat(rawLink.getTraceState().equals("Some State")).isTrue(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawLink.getTraceId())).equals(spanLink.getTraceId())).isTrue(); - assertThat(ByteString.copyFrom(Hex.decodeHex(rawLink.getSpanId())).equals(spanLink.getSpanId())).isTrue(); - } -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TestUtils.java b/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TestUtils.java deleted file mode 100644 index 0011d41b33..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/java/com/amazon/dataprepper/plugins/prepper/oteltrace/model/TestUtils.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.prepper.oteltrace.model; - -import java.util.Random; - -public class TestUtils { - - private static final Random RANDOM = new Random(); - - public static byte[] getRandomBytes(int len) { - byte[] bytes = new byte[len]; - RANDOM.nextBytes(bytes); - return bytes; - } - -} diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-full-trace-group.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-full-trace-group.json deleted file mode 100644 index 299c1abe42..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-full-trace-group.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "G4PRdsC1+0Y=", - "traceState": "", - "parentSpanId": "CFrAgv/Pv40=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902043175204700", - "endTimeUnixNano": "1597902043205117100", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.servlet-3.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "yxwHNNFJQP0=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902043168010200", - "endTimeUnixNano": "1597902043217170200", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41164" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-trace-group-missing-root.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-trace-group-missing-root.json deleted file mode 100644 index 40feea879c..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-one-trace-group-missing-root.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "WsV2GFh9pxw=", - "traceState": "", - "parentSpanId": "mnO/qUT5ye4=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902046041803300", - "endTimeUnixNano": "1597902046088892200", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "na9fwApQsYs=", - "traceState": "", - "parentSpanId": "WsV2GFh9pxw=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902046052809200", - "endTimeUnixNano": "1597902046084822500", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-full-trace-group.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-full-trace-group.json deleted file mode 100644 index 8bce844f03..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-full-trace-group.json +++ /dev/null @@ -1,353 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - }, - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "WsV2GFh9pxw=", - "traceState": "", - "parentSpanId": "mnO/qUT5ye4=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902046041803300", - "endTimeUnixNano": "1597902046088892200", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "G4PRdsC1+0Y=", - "traceState": "", - "parentSpanId": "CFrAgv/Pv40=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902043175204700", - "endTimeUnixNano": "1597902043205117100", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - }, - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "na9fwApQsYs=", - "traceState": "", - "parentSpanId": "WsV2GFh9pxw=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902046052809200", - "endTimeUnixNano": "1597902046084822500", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.servlet-3.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "yxwHNNFJQP0=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902043168010200", - "endTimeUnixNano": "1597902043217170200", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41164" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - }, - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "mnO/qUT5ye4=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902046041011600", - "endTimeUnixNano": "1597902046089556800", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41168" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-1.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-1.json deleted file mode 100644 index 9fc192306f..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-1.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "WsV2GFh9pxw=", - "traceState": "", - "parentSpanId": "mnO/qUT5ye4=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902046041803300", - "endTimeUnixNano": "1597902046088892200", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "G4PRdsC1+0Y=", - "traceState": "", - "parentSpanId": "CFrAgv/Pv40=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902043175204700", - "endTimeUnixNano": "1597902043205117100", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.servlet-3.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "yxwHNNFJQP0=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902043168010200", - "endTimeUnixNano": "1597902043217170200", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41164" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-2.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-2.json deleted file mode 100644 index 170a068e38..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-interleaved-2.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "na9fwApQsYs=", - "traceState": "", - "parentSpanId": "WsV2GFh9pxw=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902046052809200", - "endTimeUnixNano": "1597902046084822500", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.servlet-3.0", - "version": "" - }, - "spans": [ - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "mnO/qUT5ye4=", - "traceState": "", - "parentSpanId": "", - "name": "/logs", - "kind": "SPAN_KIND_SERVER", - "startTimeUnixNano": "1597902046041011600", - "endTimeUnixNano": "1597902046089556800", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "net.peer.port", - "value": { - "intValue": "41168" - } - }, - { - "key": "servlet.path", - "value": { - "stringValue": "/logs" - } - }, - { - "key": "http.response_content_length", - "value": { - "intValue": "7" - } - }, - { - "key": "http.user_agent", - "value": { - "stringValue": "curl/7.54.0" - } - }, - { - "key": "http.flavor", - "value": { - "stringValue": "HTTP/1.1" - } - }, - { - "key": "servlet.context", - "value": { - "stringValue": "" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "http://0.0.0.0:8087/logs" - } - }, - { - "key": "net.peer.ip", - "value": { - "stringValue": "172.29.0.1" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "POST" - } - }, - { - "key": "http.client_ip", - "value": { - "stringValue": "172.29.0.1" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-missing-roots.json b/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-missing-roots.json deleted file mode 100644 index ede2703bf6..0000000000 --- a/data-prepper-plugins/otel-trace-raw-prepper/src/test/resources/sample-request-two-trace-group-missing-roots.json +++ /dev/null @@ -1,171 +0,0 @@ -{ - "resourceSpans": [ - { - "resource": { - "attributes": [ - { - "key": "service.name", - "value": { - "stringValue": "analytics-service" - } - }, - { - "key": "telemetry.sdk.language", - "value": { - "stringValue": "java" - } - }, - { - "key": "telemetry.sdk.name", - "value": { - "stringValue": "opentelemetry" - } - }, - { - "key": "telemetry.sdk.version", - "value": { - "stringValue": "0.8.0-SNAPSHOT" - } - } - ], - "droppedAttributesCount": 0 - }, - "instrumentationLibrarySpans": [ - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.spring-webmvc-3.1", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "CFrAgv/Pv40=", - "traceState": "", - "parentSpanId": "yxwHNNFJQP0=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902043168792500", - "endTimeUnixNano": "1597902043215953100", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - }, - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "WsV2GFh9pxw=", - "traceState": "", - "parentSpanId": "mnO/qUT5ye4=", - "name": "LoggingController.save", - "kind": "SPAN_KIND_INTERNAL", - "startTimeUnixNano": "1597902046041803300", - "endTimeUnixNano": "1597902046088892200", - "attributes": [], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - }, - { - "instrumentationLibrary": { - "name": "io.opentelemetry.auto.apache-httpasyncclient-4.0", - "version": "" - }, - "spans": [ - { - "traceId": "/6V20yEXOsbO82Acj0vedQ==", - "spanId": "G4PRdsC1+0Y=", - "traceState": "", - "parentSpanId": "CFrAgv/Pv40=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902043175204700", - "endTimeUnixNano": "1597902043205117100", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - }, - { - "traceId": "bQ/2NNEmtuwsGAOR5ntCNw==", - "spanId": "na9fwApQsYs=", - "traceState": "", - "parentSpanId": "WsV2GFh9pxw=", - "name": "HTTP PUT", - "kind": "SPAN_KIND_CLIENT", - "startTimeUnixNano": "1597902046052809200", - "endTimeUnixNano": "1597902046084822500", - "attributes": [ - { - "key": "http.status_code", - "value": { - "intValue": "200" - } - }, - { - "key": "http.url", - "value": { - "stringValue": "/logs/_doc/service_1?timeout\\u003d1m" - } - }, - { - "key": "http.method", - "value": { - "stringValue": "PUT" - } - } - ], - "droppedAttributesCount": 0, - "events": [], - "droppedEventsCount": 0, - "links": [], - "droppedLinksCount": 0, - "status": { - "code": "STATUS_CODE_OK", - "message": "" - } - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/data-prepper-plugins/otel-trace-source/README.md b/data-prepper-plugins/otel-trace-source/README.md index 94717ba1b6..609c012565 100644 --- a/data-prepper-plugins/otel-trace-source/README.md +++ b/data-prepper-plugins/otel-trace-source/README.md @@ -21,9 +21,6 @@ source: * thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200`. * max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500`. * authentication(Optional) => An authentication configuration. By default, this runs an unauthenticated server. See below for more information. -* record_type(Optional) => A string represents the supported record data type that will be written into the buffer plugin. Its value takes either `otlp` or `event`. Default is `otlp`. - * `otlp`: otel-trace-source will write each incoming ExportTraceServiceRequest as record data type into the buffer. - * `event`: otel-trace-source will decode each incoming ExportTraceServiceRequest into collection of Data Prepper internal spans serving as buffer items. To achieve better performance in this mode, it is recommended to set the buffer capacity proportional to the estimated number of spans in the incoming request payload. ### Authentication Configurations diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java index a946cbcbfd..cde2c20c83 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcService.java @@ -41,7 +41,6 @@ public class OTelTraceGrpcService extends TraceServiceGrpc.TraceServiceImplBase public static final String REQUEST_PROCESS_DURATION = "requestProcessDuration"; private final int bufferWriteTimeoutInMillis; - private final RecordType recordType; private final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder; private final Buffer> buffer; @@ -56,12 +55,10 @@ public class OTelTraceGrpcService extends TraceServiceGrpc.TraceServiceImplBase public OTelTraceGrpcService(int bufferWriteTimeoutInMillis, - final RecordType recordType, final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder, final Buffer> buffer, final PluginMetrics pluginMetrics) { this.bufferWriteTimeoutInMillis = bufferWriteTimeoutInMillis; - this.recordType = recordType; this.buffer = buffer; this.oTelProtoDecoder = oTelProtoDecoder; @@ -78,36 +75,7 @@ public OTelTraceGrpcService(int bufferWriteTimeoutInMillis, @Override public void export(ExportTraceServiceRequest request, StreamObserver responseObserver) { - if (recordType == RecordType.OTLP) { - requestProcessDuration.record(() -> processRequestWithoutDecoding(request, responseObserver)); - } else { - requestProcessDuration.record(() -> processRequest(request, responseObserver)); - } - } - - // TODO: remove in 2.0 - private void processRequestWithoutDecoding( - final ExportTraceServiceRequest request, final StreamObserver responseObserver) { - requestsReceivedCounter.increment(); - payloadSizeSummary.record(request.getSerializedSize()); - - if (Context.current().isCancelled()) { - requestTimeoutCounter.increment(); - responseObserver.onError(Status.CANCELLED.withDescription("Cancelled by client").asRuntimeException()); - return; - } - - try { - buffer.write(new Record<>(request), bufferWriteTimeoutInMillis); - responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build()); - responseObserver.onCompleted(); - } catch (TimeoutException e) { - LOG.error("Buffer is full, unable to write"); - requestTimeoutCounter.increment(); - responseObserver - .onError(Status.RESOURCE_EXHAUSTED.withDescription("Buffer is full, request timed out.") - .asException()); - } + requestProcessDuration.record(() -> processRequest(request, responseObserver)); } private void processRequest( diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index bcdacc51c8..97d7b6860d 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -80,7 +80,6 @@ public void start(Buffer> buffer) { final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( oTelTraceSourceConfig.getRequestTimeoutInMillis(), - oTelTraceSourceConfig.getRecordType(), new OTelProtoCodec.OTelProtoDecoder(), buffer, pluginMetrics diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java index f30a7cbecf..6bb522f0a7 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceConfig.java @@ -25,7 +25,6 @@ public class OTelTraceSourceConfig { static final String THREAD_COUNT = "thread_count"; static final String MAX_CONNECTION_COUNT = "max_connection_count"; static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; - static final String RECORD_TYPE = "record_type"; static final String UNAUTHENTICATED_HEALTH_CHECK = "unauthenticated_health_check"; static final int DEFAULT_REQUEST_TIMEOUT_MS = 10000; static final int DEFAULT_PORT = 21890; @@ -37,7 +36,6 @@ public class OTelTraceSourceConfig { static final boolean DEFAULT_PROTO_REFLECTION_SERVICE = false; static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; - static final RecordType DEFAULT_RECORD_TYPE = RecordType.OTLP; private static final String S3_PREFIX = "s3://"; @JsonProperty(REQUEST_TIMEOUT) @@ -90,9 +88,6 @@ public class OTelTraceSourceConfig { @JsonProperty("authentication") private PluginModel authentication; - @JsonProperty(RECORD_TYPE) - private RecordType recordType = DEFAULT_RECORD_TYPE; - @JsonProperty(UNAUTHENTICATED_HEALTH_CHECK) private boolean unauthenticatedHealthCheck = false; @@ -197,10 +192,6 @@ public int getMaxConnectionCount() { public PluginModel getAuthentication() { return authentication; } - public RecordType getRecordType() { - return recordType; - } - public boolean isUnauthenticatedHealthCheck() { return unauthenticatedHealthCheck; } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/RecordType.java b/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/RecordType.java deleted file mode 100644 index 44854af7ea..0000000000 --- a/data-prepper-plugins/otel-trace-source/src/main/java/com/amazon/dataprepper/plugins/source/oteltrace/RecordType.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.plugins.source.oteltrace; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -/** - * An enum to represent the record data types supported in {@link OTelTraceSource}. - * @since 1.4 - * TODO: remove in 2.0 - */ -enum RecordType { - @JsonProperty("otlp") - OTLP("otlp"), - @JsonProperty("event") - EVENT("event"); - - private static final Map NAMES_MAP = new HashMap<>(); - - static { - Arrays.stream(RecordType.values()).forEach(recordType -> NAMES_MAP.put(recordType.name(), recordType)); - } - - private final String name; - - RecordType(final String name) { - this.name = name; - } - - public String toString() { - return this.name; - } -} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java index c839e33942..45eb42a853 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceGrpcServiceTest.java @@ -40,7 +40,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -127,23 +126,8 @@ public void setup() { } @Test - public void export_Success_responseObserverOnCompleted_withOTLPRecordType() throws Exception { - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.OTLP); - objectUnderTest.export(SUCCESS_REQUEST, responseObserver); - - verify(buffer, times(1)).write(recordCaptor.capture(), anyInt()); - verify(responseObserver, times(1)).onNext(ExportTraceServiceResponse.newBuilder().build()); - verify(responseObserver, times(1)).onCompleted(); - verify(requestsReceivedCounter, times(1)).increment(); - verifyNoInteractions(timeoutCounter); - - Record capturedRecord = recordCaptor.getValue(); - assertEquals(SUCCESS_REQUEST, capturedRecord.getData()); - } - - @Test - public void export_Success_responseObserverOnCompleted_withEventRecordType() throws Exception { - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.EVENT); + public void export_Success_responseObserverOnCompleted() throws Exception { + objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder()); objectUnderTest.export(SUCCESS_REQUEST, responseObserver); verify(buffer, times(1)).writeAll(recordsCaptor.capture(), anyInt()); @@ -165,32 +149,8 @@ public void export_Success_responseObserverOnCompleted_withEventRecordType() thr } @Test - public void export_BufferTimeout_responseObserverOnError_withOTLPRecordType() throws Exception { - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.OTLP); - doThrow(new TimeoutException()).when(buffer).write(any(Record.class), anyInt()); - - objectUnderTest.export(SUCCESS_REQUEST, responseObserver); - - verify(buffer, times(1)).write(any(Record.class), anyInt()); - verify(responseObserver, times(0)).onNext(any()); - verify(responseObserver, times(0)).onCompleted(); - verify(responseObserver, times(1)).onError(statusExceptionArgumentCaptor.capture()); - verify(timeoutCounter, times(1)).increment(); - verify(requestsReceivedCounter, times(1)).increment(); - verifyNoInteractions(successRequestsCounter); - verifyNoInteractions(badRequestsCounter); - verifyNoInteractions(requestsTooLargeCounter); - final ArgumentCaptor payloadLengthCaptor = ArgumentCaptor.forClass(Double.class); - verify(payloadSizeSummary, times(1)).record(payloadLengthCaptor.capture()); - assertThat(payloadLengthCaptor.getValue().intValue(), equalTo(SUCCESS_REQUEST.getSerializedSize())); - verify(requestProcessDuration, times(1)).record(ArgumentMatchers.any()); - StatusException capturedStatusException = statusExceptionArgumentCaptor.getValue(); - assertThat(capturedStatusException.getStatus().getCode(), equalTo(Status.RESOURCE_EXHAUSTED.getCode())); - } - - @Test - public void export_BufferTimeout_responseObserverOnError_withEventRecordType() throws Exception { - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.EVENT); + public void export_BufferTimeout_responseObserverOnError() throws Exception { + objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder()); doThrow(new TimeoutException()).when(buffer).writeAll(any(Collection.class), anyInt()); objectUnderTest.export(SUCCESS_REQUEST, responseObserver); @@ -217,7 +177,7 @@ public void export_BadRequest_responseObserverOnError() throws Exception { final String testMessage = "test message"; final RuntimeException testException = new RuntimeException(testMessage); when(mockOTelProtoDecoder.parseExportTraceServiceRequest(any())).thenThrow(testException); - objectUnderTest = generateOTelTraceGrpcService(mockOTelProtoDecoder, RecordType.EVENT); + objectUnderTest = generateOTelTraceGrpcService(mockOTelProtoDecoder); objectUnderTest.export(SUCCESS_REQUEST, responseObserver); verifyNoInteractions(buffer); @@ -242,7 +202,7 @@ public void export_BadRequest_responseObserverOnError() throws Exception { public void export_RequestTooLarge_responseObserverOnError() throws Exception { final String testMessage = "test message"; doThrow(new SizeOverflowException(testMessage)).when(buffer).writeAll(any(Collection.class), anyInt()); - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.EVENT); + objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder()); objectUnderTest.export(SUCCESS_REQUEST, responseObserver); verify(buffer, times(1)).writeAll(any(Collection.class), anyInt()); @@ -266,7 +226,7 @@ public void export_RequestTooLarge_responseObserverOnError() throws Exception { public void export_BufferInternalException_responseObserverOnError() throws Exception { final String testMessage = "test message"; doThrow(new IOException(testMessage)).when(buffer).writeAll(any(Collection.class), anyInt()); - objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder(), RecordType.EVENT); + objectUnderTest = generateOTelTraceGrpcService(new OTelProtoCodec.OTelProtoDecoder()); objectUnderTest.export(SUCCESS_REQUEST, responseObserver); verify(buffer, times(1)).writeAll(any(Collection.class), anyInt()); @@ -286,8 +246,8 @@ public void export_BufferInternalException_responseObserverOnError() throws Exce assertThat(capturedStatusException.getStatus().getCode(), equalTo(Status.INTERNAL.getCode())); } - private OTelTraceGrpcService generateOTelTraceGrpcService(final OTelProtoCodec.OTelProtoDecoder decoder, final RecordType recordType) { + private OTelTraceGrpcService generateOTelTraceGrpcService(final OTelProtoCodec.OTelProtoDecoder decoder) { return new OTelTraceGrpcService( - bufferWriteTimeoutInMillis, recordType, decoder, buffer, mockPluginMetrics); + bufferWriteTimeoutInMillis, decoder, buffer, mockPluginMetrics); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 46d0379bf9..566ae7585e 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -47,9 +47,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockedStatic; @@ -70,7 +67,6 @@ import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; import static org.hamcrest.MatcherAssert.assertThat; @@ -164,9 +160,8 @@ private BlockingBuffer> getBuffer() { return new BlockingBuffer<>(new PluginSetting("blocking_buffer", integerHashMap)); } - private void configureObjectUnderTest(final String recordType) { + private void configureObjectUnderTest() { final Map settingsMap = new HashMap<>(); - settingsMap.put("record_type", recordType); settingsMap.put("request_timeout", 5); settingsMap.put(SSL, false); pluginSetting = new PluginSetting("otel_trace", settingsMap); @@ -177,10 +172,6 @@ private void configureObjectUnderTest(final String recordType) { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory); } - private static Stream recordTypeArguments() { - return Stream.of(Arguments.of(RecordType.OTLP.toString()), Arguments.of(RecordType.EVENT.toString())); - } - @BeforeEach public void beforeEach() { lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); @@ -199,7 +190,7 @@ public void beforeEach() { when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(authenticationProvider); - configureObjectUnderTest(RecordType.EVENT.toString()); + configureObjectUnderTest(); buffer = getBuffer(); } @@ -208,10 +199,9 @@ public void afterEach() { SOURCE.stop(); } - @ParameterizedTest - @MethodSource("recordTypeArguments") - void testHttpFullJson(final String recordType) throws InvalidProtocolBufferException { - configureObjectUnderTest(recordType); + @Test + void testHttpFullJson() throws InvalidProtocolBufferException { + configureObjectUnderTest(); SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) @@ -237,12 +227,10 @@ void testHttpFullJson(final String recordType) throws InvalidProtocolBufferExcep .join(); } - @ParameterizedTest - @MethodSource("recordTypeArguments") - void testHttpsFullJson(final String recordType) throws InvalidProtocolBufferException { + @Test + void testHttpsFullJson() throws InvalidProtocolBufferException { final Map settingsMap = new HashMap<>(); - settingsMap.put("record_type", recordType); settingsMap.put("request_timeout", 5); settingsMap.put(SSL, true); settingsMap.put("useAcmCertForSSL", false); @@ -281,10 +269,9 @@ void testHttpsFullJson(final String recordType) throws InvalidProtocolBufferExce .join(); } - @ParameterizedTest - @MethodSource("recordTypeArguments") - void testHttpFullBytes(final String recordType) { - configureObjectUnderTest(recordType); + @Test + void testHttpFullBytes() { + configureObjectUnderTest(); SOURCE.start(buffer); WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java index 872a07e87e..c80f91f24b 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/com/amazon/dataprepper/plugins/source/oteltrace/OtelTraceSourceConfigTests.java @@ -8,8 +8,6 @@ import com.amazon.dataprepper.model.configuration.PluginSetting; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import java.util.HashMap; import java.util.Map; @@ -49,7 +47,6 @@ public void testDefault() { assertEquals(OTelTraceSourceConfig.DEFAULT_PORT, otelTraceSourceConfig.getPort()); assertEquals(OTelTraceSourceConfig.DEFAULT_THREAD_COUNT, otelTraceSourceConfig.getThreadCount()); assertEquals(OTelTraceSourceConfig.DEFAULT_MAX_CONNECTION_COUNT, otelTraceSourceConfig.getMaxConnectionCount()); - assertEquals(OTelTraceSourceConfig.DEFAULT_RECORD_TYPE, otelTraceSourceConfig.getRecordType()); assertFalse(otelTraceSourceConfig.hasHealthCheck()); assertFalse(otelTraceSourceConfig.hasProtoReflectionService()); assertFalse(otelTraceSourceConfig.enableHttpHealthCheck()); @@ -107,7 +104,6 @@ public void testValidConfigWithoutS3CertAndKey() { true, TEST_KEY_CERT, TEST_KEY, - RecordType.OTLP.toString(), TEST_THREAD_COUNT, TEST_MAX_CONNECTION_COUNT); @@ -141,7 +137,6 @@ public void testValidConfigWithS3CertAndKey() { true, TEST_KEY_CERT_S3, TEST_KEY_S3, - RecordType.OTLP.toString(), TEST_THREAD_COUNT, TEST_MAX_CONNECTION_COUNT); @@ -174,7 +169,6 @@ public void testInvalidConfigWithNullKeyCert() { false, true, null, TEST_KEY, - RecordType.OTLP.toString(), DEFAULT_THREAD_COUNT, DEFAULT_MAX_CONNECTION_COUNT); @@ -197,7 +191,6 @@ public void testInvalidConfigWithEmptyKeyCert() { true, "", TEST_KEY, - RecordType.OTLP.toString(), DEFAULT_THREAD_COUNT, DEFAULT_MAX_CONNECTION_COUNT); @@ -220,7 +213,6 @@ public void testInvalidConfigWithEmptyKeyFile() { true, TEST_KEY_CERT, "", - RecordType.OTLP.toString(), DEFAULT_THREAD_COUNT, DEFAULT_MAX_CONNECTION_COUNT); @@ -230,45 +222,6 @@ public void testInvalidConfigWithEmptyKeyFile() { assertThrows(IllegalArgumentException.class, otelTraceSourceConfig::validateAndInitializeCertAndKeyFileInS3); } - @ParameterizedTest - @ValueSource(strings = {"otlp", "event"}) - public void testConfigWithValidRecordType(final String recordType) { - final PluginSetting pluginSetting = completePluginSettingForOtelTraceSource( - TEST_REQUEST_TIMEOUT_MS, - TEST_PORT, - false, - false, - false, - false, - null, - null, - recordType, - TEST_THREAD_COUNT, - TEST_MAX_CONNECTION_COUNT); - - final OTelTraceSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class); - assertEquals(recordType, otelTraceSourceConfig.getRecordType().toString()); - } - - @Test - public void testConfigWithInValidRecordType() { - final String invalidRecordType = "unknown"; - final PluginSetting pluginSetting = completePluginSettingForOtelTraceSource( - TEST_REQUEST_TIMEOUT_MS, - TEST_PORT, - false, - false, - false, - false, - null, - null, - invalidRecordType, - TEST_THREAD_COUNT, - TEST_MAX_CONNECTION_COUNT); - - assertThrows(IllegalArgumentException.class, () -> OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class)); - } - private PluginSetting completePluginSettingForOtelTraceSource(final int requestTimeoutInMillis, final int port, final boolean healthCheck, @@ -277,7 +230,6 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT final boolean isSSL, final String sslKeyCertChainFile, final String sslKeyFile, - final String recordType, final int threadCount, final int maxConnectionCount) { final Map settings = new HashMap<>(); @@ -289,7 +241,6 @@ private PluginSetting completePluginSettingForOtelTraceSource(final int requestT settings.put(OTelTraceSourceConfig.SSL, isSSL); settings.put(OTelTraceSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); settings.put(OTelTraceSourceConfig.SSL_KEY_FILE, sslKeyFile); - settings.put(OTelTraceSourceConfig.RECORD_TYPE, recordType); settings.put(OTelTraceSourceConfig.THREAD_COUNT, threadCount); settings.put(OTelTraceSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); return new PluginSetting(PLUGIN_NAME, settings); diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java index 6bdebeda60..e8c4efb40d 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarder.java @@ -7,6 +7,7 @@ import com.amazon.dataprepper.model.annotations.DataPrepperPlugin; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.event.Event; import com.amazon.dataprepper.model.processor.AbstractProcessor; import com.amazon.dataprepper.model.processor.Processor; import com.amazon.dataprepper.model.record.Record; @@ -38,10 +39,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; -import java.util.stream.Stream; @DataPrepperPlugin(name = "peer_forwarder", pluginType = Processor.class) -public class PeerForwarder extends AbstractProcessor, Record> { +public class PeerForwarder extends AbstractProcessor, Record> { public static final String REQUESTS = "requests"; public static final String LATENCY = "latency"; public static final String ERRORS = "errors"; @@ -96,88 +96,18 @@ public PeerForwarder(final PluginSetting pluginSetting, final PeerForwarderConfi } @Override - public List> doExecute(final Collection> records) { + public List> doExecute(final Collection> records) { final List spans = new ArrayList<>(); - final List exportTraceServiceRequests = new ArrayList<>(); records.forEach(record -> { - final Object recordData = record.getData(); - // TODO: remove support for ExportTraceServiceRequest in 2.0 - if (recordData instanceof ExportTraceServiceRequest) { - exportTraceServiceRequests.add((ExportTraceServiceRequest) recordData); - } else if (recordData instanceof Span) { + final Event recordData = record.getData(); + if (recordData instanceof Span) { spans.add((Span) recordData); } else { throw new RuntimeException("Unsupported record data type: " + recordData.getClass()); } }); - final List requestsToProcessLocally = executeExportTraceServiceRequests(exportTraceServiceRequests); final List spansToProcessLocally = executeSpans(spans); - return Stream.concat(requestsToProcessLocally.stream(), spansToProcessLocally.stream()).map(Record::new).collect(Collectors.toList()); - } - - private List executeExportTraceServiceRequests(final List requests) { - final Map> groupedRS = new HashMap<>(); - - // Group ResourceSpans by consistent hashing of traceId - for (final ExportTraceServiceRequest request : requests) { - for (final ResourceSpans rs : request.getResourceSpansList()) { - final List> rsBatch = PeerForwarderUtils.splitByTrace(rs); - for (final Map.Entry entry : rsBatch) { - final String traceId = entry.getKey(); - final ResourceSpans newRS = entry.getValue(); - final String dataPrepperIp = hashRing.getServerIp(traceId).orElse(StaticPeerListProvider.LOCAL_ENDPOINT); - groupedRS.computeIfAbsent(dataPrepperIp, x -> new ArrayList<>()).add(newRS); - } - } - } - - final List requestsToProcessLocally = new ArrayList<>(); - final List> forwardedRequestFutures = new ArrayList<>(); - - for (final Map.Entry> entry : groupedRS.entrySet()) { - final TraceServiceGrpc.TraceServiceBlockingStub client = getClient(entry.getKey()); - - // Create ExportTraceRequest for storing single batch of spans - ExportTraceServiceRequest.Builder currRequestBuilder = ExportTraceServiceRequest.newBuilder(); - int currSpansCount = 0; - for (final ResourceSpans rs : entry.getValue()) { - final int rsSize = PeerForwarderUtils.getResourceSpansSize(rs); - if (currSpansCount >= maxNumSpansPerRequest) { - final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - if (isLocalClient(client)) { - requestsToProcessLocally.add(currRequest); - } else { - forwardedRequestFutures.add(processRequest(client, currRequest)); - } - currRequestBuilder = ExportTraceServiceRequest.newBuilder(); - currSpansCount = 0; - } - currRequestBuilder.addResourceSpans(rs); - currSpansCount += rsSize; - } - // Dealing with the last batch request - if (currSpansCount > 0) { - final ExportTraceServiceRequest currRequest = currRequestBuilder.build(); - if (client == null) { - requestsToProcessLocally.add(currRequest); - } else { - forwardedRequestFutures.add(processRequest(client, currRequest)); - } - } - } - - for (final CompletableFuture future : forwardedRequestFutures) { - try { - final ExportTraceServiceRequest request = future.get(); - if (request != null) { - requestsToProcessLocally.add(request); - } - } catch (InterruptedException | ExecutionException e) { - LOG.error("Problem with asynchronous peer forwarding", e); - } - } - - return requestsToProcessLocally; + return spansToProcessLocally.stream().map(span -> new Record(span)).collect(Collectors.toList()); } private List executeSpans(final List spans) { diff --git a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java index 7b6f1f55fa..2cad3c2b43 100644 --- a/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java +++ b/data-prepper-plugins/peer-forwarder/src/main/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderUtils.java @@ -6,45 +6,13 @@ package com.amazon.dataprepper.plugins.prepper.peerforwarder; import com.amazon.dataprepper.model.trace.Span; -import com.linecorp.armeria.internal.shaded.bouncycastle.util.encoders.Hex; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; -import io.opentelemetry.proto.trace.v1.ResourceSpans; -import java.util.AbstractMap; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; public final class PeerForwarderUtils { - public static int getResourceSpansSize(final ResourceSpans rs) { - return rs.getInstrumentationLibrarySpansList().stream().mapToInt(InstrumentationLibrarySpans::getSpansCount).sum(); - } - - public static List> splitByTrace(final ResourceSpans rs) { - final List> result = new ArrayList<>(); - for (final InstrumentationLibrarySpans ils: rs.getInstrumentationLibrarySpansList()) { - final Map batches = new HashMap<>(); - for (final io.opentelemetry.proto.trace.v1.Span span: ils.getSpansList()) { - final String sTraceId = Hex.toHexString(span.getTraceId().toByteArray()); - if (!batches.containsKey(sTraceId)) { - final ResourceSpans.Builder newRSBuilder = ResourceSpans.newBuilder() - .setResource(rs.getResource()); - newRSBuilder.addInstrumentationLibrarySpansBuilder().setInstrumentationLibrary(ils.getInstrumentationLibrary()); - batches.put(sTraceId, newRSBuilder); - } - - // there is only one instrumentation library per batch - batches.get(sTraceId).getInstrumentationLibrarySpansBuilder(0).addSpans(span); - } - - batches.forEach((traceId, rsBuilder) -> result.add(new AbstractMap.SimpleEntry<>(traceId, rsBuilder.build()))); - } - - return result; - } public static Map> splitByTrace(final Collection spans) { return spans.stream().collect(Collectors.groupingBy(Span::getTraceId)); diff --git a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java index bc0da1c291..cbd16a73f7 100644 --- a/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java +++ b/data-prepper-plugins/peer-forwarder/src/test/java/com/amazon/dataprepper/plugins/prepper/peerforwarder/PeerForwarderTest.java @@ -8,18 +8,16 @@ import com.amazon.dataprepper.metrics.MetricNames; import com.amazon.dataprepper.metrics.MetricsTestUtil; import com.amazon.dataprepper.model.configuration.PluginSetting; +import com.amazon.dataprepper.model.event.Event; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.trace.DefaultTraceGroupFields; import com.amazon.dataprepper.model.trace.JacksonSpan; import com.amazon.dataprepper.model.trace.Span; import com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec; -import com.google.protobuf.ByteString; import io.grpc.Channel; import io.micrometer.core.instrument.Measurement; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import io.opentelemetry.proto.common.v1.InstrumentationLibrary; -import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; import io.opentelemetry.proto.trace.v1.ResourceSpans; import org.apache.commons.codec.DecoderException; @@ -66,23 +64,6 @@ public class PeerForwarderTest { private static final String TEST_PIPELINE_NAME = "testPipeline"; private static final String LOCAL_IP = "127.0.0.1"; - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_1 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdA")).setSpanId(ByteString.copyFromUtf8("spanId1")).build(); - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_2 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdA")).setSpanId(ByteString.copyFromUtf8("spanId2")).build(); - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_3 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdA")).setSpanId(ByteString.copyFromUtf8("spanId3")).build(); - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_4 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdB")).setSpanId(ByteString.copyFromUtf8("spanId4")).build(); - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_5 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdB")).setSpanId(ByteString.copyFromUtf8("spanId5")).build(); - private static final io.opentelemetry.proto.trace.v1.Span OTLP_SPAN_6 = io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setTraceId(ByteString.copyFromUtf8("traceIdB")).setSpanId(ByteString.copyFromUtf8("spanId6")).build(); - - private static final ExportTraceServiceRequest REQUEST_1 = generateRequest(OTLP_SPAN_1, OTLP_SPAN_2, OTLP_SPAN_4); - private static final ExportTraceServiceRequest REQUEST_2 = generateRequest(OTLP_SPAN_3, OTLP_SPAN_5, OTLP_SPAN_6); - private static final ExportTraceServiceRequest REQUEST_3 = generateRequest(OTLP_SPAN_1, OTLP_SPAN_2, OTLP_SPAN_3); - private static final ExportTraceServiceRequest REQUEST_4 = generateRequest(OTLP_SPAN_4, OTLP_SPAN_5, OTLP_SPAN_6); private static final String TEST_TRACE_ID_1 = "b1"; private static final String TEST_TRACE_ID_2 = "b2"; @@ -137,24 +118,6 @@ public void tearDown() { peerClientPoolClassMock.close(); } - private static ExportTraceServiceRequest generateRequest(final io.opentelemetry.proto.trace.v1.Span... spans) { - return ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .setResource(Resource.newBuilder().build()) - .addInstrumentationLibrarySpans( - InstrumentationLibrarySpans.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder().build()) - .addAllSpans(Arrays.asList(spans))) - .build()).build(); - } - - private static ResourceSpans generateResourceSpans(final io.opentelemetry.proto.trace.v1.Span... spans) { - return ResourceSpans.newBuilder().setResource(Resource.newBuilder().build()).addInstrumentationLibrarySpans( - InstrumentationLibrarySpans.newBuilder() - .setInstrumentationLibrary(InstrumentationLibrary.newBuilder().build()) - .addAllSpans(Arrays.asList(spans))).build(); - } - public static Span getSpan(final String traceId, final String spanId, final String parentId, final String serviceName, final String spanName, final io.opentelemetry.proto.trace.v1.Span.SpanKind spanKind) { final long startTimeNanos = System.nanoTime(); @@ -203,44 +166,20 @@ private void reflectivelySetEncoder(final PeerForwarder peerForwarder, final OTe } } - // TODO: remove in 2.0 - @Test - public void testLocalIpOnlyWithExportTraceServiceRequestRecordData() { - final PeerForwarder testPeerForwarder = generatePeerForwarder(Collections.singletonList(LOCAL_IP), 2); - final List> exportedRecords = - testPeerForwarder.doExecute(Arrays.asList(new Record<>(REQUEST_1), new Record<>(REQUEST_2))); - assertTrue(exportedRecords.size() >= 3); - final List exportedResourceSpans = new ArrayList<>(); - for (final Record record: exportedRecords) { - final Object recordData = record.getData(); - assertTrue(recordData instanceof ExportTraceServiceRequest); - exportedResourceSpans.addAll(((ExportTraceServiceRequest) recordData).getResourceSpansList()); - } - final List expectedResourceSpans = Arrays.asList( - generateResourceSpans(OTLP_SPAN_1, OTLP_SPAN_2), - generateResourceSpans(OTLP_SPAN_3), - generateResourceSpans(OTLP_SPAN_4), - generateResourceSpans(OTLP_SPAN_5, OTLP_SPAN_6) - ); - assertTrue(exportedResourceSpans.containsAll(expectedResourceSpans)); - assertTrue(expectedResourceSpans.containsAll(exportedResourceSpans)); - } - @Test - public void testLocalIpOnlyWithEventRecordData() { + public void testLocalIpOnly() { final PeerForwarder testPeerForwarder = generatePeerForwarder(Collections.singletonList(LOCAL_IP), 2); final List inputSpans = Arrays.asList(SPAN_1, SPAN_2, SPAN_4, SPAN_5); - final List> exportedRecords = testPeerForwarder.doExecute( - inputSpans.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder.doExecute( + inputSpans.stream().map(span -> new Record(span)).collect(Collectors.toList())); assertEquals(4, exportedRecords.size()); final List exportedSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList()); assertTrue(exportedSpans.containsAll(inputSpans)); assertTrue(inputSpans.containsAll(exportedSpans)); } - // TODO: remove in 2.0 @Test - public void testSingleRemoteIpBothLocalAndForwardedRequestWithExportTraceServiceRequestRecordData() { + public void testSingleRemoteIpBothLocalAndForwardedRequest() throws DecoderException { final List testIps = generateTestIps(2); final Channel channel = mock(Channel.class); final String peerIp = testIps.get(1); @@ -258,50 +197,8 @@ public void testSingleRemoteIpBothLocalAndForwardedRequestWithExportTraceService MetricsTestUtil.initMetrics(); final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - final List> exportedRecords = testPeerForwarder - .doExecute(Arrays.asList(new Record<>(REQUEST_1), new Record<>(REQUEST_2))); - - final List expectedLocalResourceSpans = Arrays.asList( - generateResourceSpans(OTLP_SPAN_1, OTLP_SPAN_2), - generateResourceSpans(OTLP_SPAN_3) - ); - final List expectedForwardedResourceSpans = Arrays.asList( - generateResourceSpans(OTLP_SPAN_5, OTLP_SPAN_6), - generateResourceSpans(OTLP_SPAN_4) - ); - assertEquals(1, exportedRecords.size()); - final ExportTraceServiceRequest localRequest = (ExportTraceServiceRequest) exportedRecords.get(0).getData(); - final List localResourceSpans = localRequest.getResourceSpansList(); - assertTrue(localResourceSpans.containsAll(expectedLocalResourceSpans)); - assertTrue(expectedLocalResourceSpans.containsAll(localResourceSpans)); - assertEquals(1, requestsByIp.get(peerIp).size()); - final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0); - final List forwardedResourceSpans = forwardedRequest.getResourceSpansList(); - assertTrue(forwardedResourceSpans.containsAll(expectedForwardedResourceSpans)); - assertTrue(expectedForwardedResourceSpans.containsAll(forwardedResourceSpans)); - } - - @Test - public void testSingleRemoteIpBothLocalAndForwardedRequestWithEventRecordData() throws DecoderException { - final List testIps = generateTestIps(2); - final Channel channel = mock(Channel.class); - final String peerIp = testIps.get(1); - when(channel.authority()).thenReturn(String.format("%s:21890", peerIp)); - when(peerClientPool.getClient(peerIp)).thenReturn(client); - when(client.getChannel()).thenReturn(channel); - final Map> requestsByIp = testIps.stream() - .collect(Collectors.toMap(ip-> ip, ip-> new ArrayList<>())); - doAnswer(invocation -> { - final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0); - requestsByIp.get(peerIp).add(exportTraceServiceRequest); - return null; - }).when(client).export(any(ExportTraceServiceRequest.class)); - - MetricsTestUtil.initMetrics(); - final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_ALL.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_ALL.stream().map(span -> new Record(span)).collect(Collectors.toList())); final List expectedLocalSpans = Arrays.asList(SPAN_1, SPAN_2, SPAN_3); Assert.assertEquals(3, exportedRecords.size()); @@ -322,31 +219,13 @@ public void testSingleRemoteIpBothLocalAndForwardedRequestWithEventRecordData() }); } - // TODO: remove in 2.0 - @Test - public void testSingleRemoteIpLocalRequestOnlyWithExportTraceServiceRequestRecordData() throws Exception { - final List testIps = generateTestIps(2); - final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - - final List> exportedRecords = testPeerForwarder - .doExecute(Collections.singletonList(new Record<>(REQUEST_3))); - - final List expectedLocalResourceSpans = Collections.singletonList( - generateResourceSpans(OTLP_SPAN_1, OTLP_SPAN_2, OTLP_SPAN_3)); - assertEquals(1, exportedRecords.size()); - final ExportTraceServiceRequest localRequest = (ExportTraceServiceRequest) exportedRecords.get(0).getData(); - final List localResourceSpans = localRequest.getResourceSpansList(); - assertTrue(localResourceSpans.containsAll(expectedLocalResourceSpans)); - assertTrue(expectedLocalResourceSpans.containsAll(localResourceSpans)); - } - @Test - public void testSingleRemoteIpLocalRequestOnlyWithEventRecordData() throws Exception { + public void testSingleRemoteIpLocalRequestOnly() throws Exception { final List testIps = generateTestIps(2); final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_A.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_A.stream().map(span -> new Record(span)).collect(Collectors.toList())); Assert.assertEquals(3, exportedRecords.size()); final List localSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList()); @@ -354,64 +233,8 @@ public void testSingleRemoteIpLocalRequestOnlyWithEventRecordData() throws Excep assertTrue(TEST_SPANS_A.containsAll(localSpans)); } - // TODO: remove in 2.0 - @Test - public void testSingleRemoteIpForwardedRequestOnlyWithExportTraceServiceRequestRecordData() throws Exception { - final List testIps = generateTestIps(2); - final Channel channel = mock(Channel.class); - final String peerIp = testIps.get(1); - final String fullPeerIp = String.format("%s:21890", peerIp); - when(channel.authority()).thenReturn(fullPeerIp); - when(peerClientPool.getClient(peerIp)).thenReturn(client); - when(client.getChannel()).thenReturn(channel); - final Map> requestsByIp = testIps.stream() - .collect(Collectors.toMap(ip-> ip, ip-> new ArrayList<>())); - doAnswer(invocation -> { - final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0); - requestsByIp.get(peerIp).add(exportTraceServiceRequest); - return null; - }).when(client).export(any(ExportTraceServiceRequest.class)); - - MetricsTestUtil.initMetrics(); - final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - - final List> exportedRecords = testPeerForwarder - .doExecute(Collections.singletonList(new Record<>(REQUEST_4))); - - final List expectedForwardedResourceSpans = Collections.singletonList( - generateResourceSpans(OTLP_SPAN_4, OTLP_SPAN_5, OTLP_SPAN_6)); - assertEquals(1, requestsByIp.get(peerIp).size()); - final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0); - final List forwardedResourceSpans = forwardedRequest.getResourceSpansList(); - assertTrue(forwardedResourceSpans.containsAll(expectedForwardedResourceSpans)); - assertTrue(expectedForwardedResourceSpans.containsAll(forwardedResourceSpans)); - assertEquals(0, exportedRecords.size()); - - // Verify metrics - final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.ERRORS).toString()); - assertEquals(1, forwardRequestErrorMeasurements.size()); - assertEquals(0.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); - final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.REQUESTS).toString()); - assertEquals(1, forwardRequestSuccessMeasurements.size()); - assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); - final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.LATENCY).toString()); - assertEquals(3, forwardRequestLatencyMeasurements.size()); - // COUNT - assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); - // TOTAL_TIME - assertTrue(forwardRequestLatencyMeasurements.get(1).getValue() > 0.0); - // MAX - assertTrue(forwardRequestLatencyMeasurements.get(2).getValue() > 0.0); - } - @Test - public void testSingleRemoteIpForwardedRequestOnlyWithEventRecordData() throws Exception { + public void testSingleRemoteIpForwardedRequestOnly() throws Exception { final List testIps = generateTestIps(2); final Channel channel = mock(Channel.class); final String peerIp = testIps.get(1); @@ -430,8 +253,8 @@ public void testSingleRemoteIpForwardedRequestOnlyWithEventRecordData() throws E MetricsTestUtil.initMetrics(); final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); Assert.assertEquals(1, requestsByIp.get(peerIp).size()); final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0); @@ -470,57 +293,8 @@ public void testSingleRemoteIpForwardedRequestOnlyWithEventRecordData() throws E assertTrue(forwardRequestLatencyMeasurements.get(2).getValue() > 0.0); } - // TODO: remove in 2.0 - @Test - public void testSingleRemoteIpForwardRequestClientErrorWithExportTraceServiceRequestRecordData() { - final List testIps = generateTestIps(2); - final Channel channel = mock(Channel.class); - final String peerIp = testIps.get(1); - final String fullPeerIp = String.format("%s:21890", peerIp); - when(channel.authority()).thenReturn(fullPeerIp); - when(peerClientPool.getClient(peerIp)).thenReturn(client); - when(client.export(any(ExportTraceServiceRequest.class))).thenThrow(new RuntimeException()); - when(client.getChannel()).thenReturn(channel); - - MetricsTestUtil.initMetrics(); - final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - - final List> exportedRecords = testPeerForwarder - .doExecute(Collections.singletonList(new Record<>(REQUEST_4))); - - final List expectedLocalResourceSpans = Collections.singletonList( - generateResourceSpans(OTLP_SPAN_4, OTLP_SPAN_5, OTLP_SPAN_6)); - assertEquals(1, exportedRecords.size()); - final ExportTraceServiceRequest exportedRequest = (ExportTraceServiceRequest) exportedRecords.get(0).getData(); - final List forwardedResourceSpans = exportedRequest.getResourceSpansList(); - assertTrue(forwardedResourceSpans.containsAll(expectedLocalResourceSpans)); - assertTrue(expectedLocalResourceSpans.containsAll(forwardedResourceSpans)); - - // Verify metrics - final List forwardRequestErrorMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.ERRORS).toString()); - assertEquals(1, forwardRequestErrorMeasurements.size()); - assertEquals(1.0, forwardRequestErrorMeasurements.get(0).getValue(), 0); - final List forwardRequestSuccessMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.REQUESTS).toString()); - assertEquals(1, forwardRequestSuccessMeasurements.size()); - assertEquals(1.0, forwardRequestSuccessMeasurements.get(0).getValue(), 0); - final List forwardRequestLatencyMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(TEST_PIPELINE_NAME).add("peer_forwarder") - .add(PeerForwarder.LATENCY).toString()); - assertEquals(3, forwardRequestLatencyMeasurements.size()); - // COUNT - assertEquals(1.0, forwardRequestLatencyMeasurements.get(0).getValue(), 0); - // TOTAL_TIME - assertTrue(forwardRequestLatencyMeasurements.get(1).getValue() > 0.0); - // MAX - assertTrue(forwardRequestLatencyMeasurements.get(2).getValue() > 0.0); - } - @Test - public void testSingleRemoteIpForwardRequestClientErrorWithEventRecordData() { + public void testSingleRemoteIpForwardRequestClientError() { final List testIps = generateTestIps(2); final Channel channel = mock(Channel.class); final String peerIp = testIps.get(1); @@ -533,8 +307,8 @@ public void testSingleRemoteIpForwardRequestClientErrorWithEventRecordData() { MetricsTestUtil.initMetrics(); final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); Assert.assertEquals(3, exportedRecords.size()); final List exportedSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList()); @@ -582,8 +356,8 @@ public void testSingleRemoteIpForwardRequestFutureError() throws ExecutionExcept MetricsTestUtil.initMetrics(); final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3); - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); verify(completableFuture, times(1)).get(); Assert.assertEquals(3, exportedRecords.size()); @@ -605,8 +379,8 @@ public void testSingleRemoteIpForwardRequestEncodeError() throws NoSuchFieldExce when(oTelProtoEncoder.convertToResourceSpans(any(Span.class))).thenThrow(new DecoderException()); reflectivelySetEncoder(testPeerForwarder, oTelProtoEncoder); - final List> exportedRecords = testPeerForwarder - .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); + final List> exportedRecords = testPeerForwarder + .doExecute(TEST_SPANS_B.stream().map(span -> new Record(span)).collect(Collectors.toList())); verifyNoInteractions(client); Assert.assertEquals(3, exportedRecords.size()); diff --git a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java index 60d5b5369c..62ec093975 100644 --- a/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java +++ b/data-prepper-plugins/service-map-stateful/src/main/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepper.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.google.common.primitives.SignedBytes; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import org.apache.commons.codec.binary.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +37,7 @@ @SingleThread @DataPrepperPlugin(name = "service_map_stateful", pluginType = Processor.class) -public class ServiceMapStatefulPrepper extends AbstractProcessor, Record> { +public class ServiceMapStatefulPrepper extends AbstractProcessor, Record> { public static final String SPANS_DB_SIZE = "spansDbSize"; public static final String TRACE_GROUP_DB_SIZE = "traceGroupDbSize"; @@ -126,20 +125,10 @@ private static File createPath(File path) { * added to the service map index. Otherwise, returns an empty set. */ @Override - public Collection> doExecute(Collection> records) { + public Collection> doExecute(Collection> records) { final Collection> relationships = windowDurationHasPassed() ? evaluateEdges() : EMPTY_COLLECTION; final Map batchStateData = new TreeMap<>(SignedBytes.lexicographicalComparator()); - records.forEach(i -> { - final Object recordData = i.getData(); - // TODO: remove support for ExportTraceServiceRequest in 2.0 - if (recordData instanceof ExportTraceServiceRequest) { - processExportTraceServiceRequest((ExportTraceServiceRequest) recordData, batchStateData); - } else if (recordData instanceof Span) { - processSpan((Span) recordData, batchStateData); - } else { - throw new RuntimeException("Unsupported record data type: " + recordData.getClass()); - } - }); + records.forEach(i -> processSpan((Span) i.getData(), batchStateData)); try { currentWindow.putAll(batchStateData); } catch (RuntimeException e) { @@ -148,42 +137,6 @@ public Collection> doExecute(Collection> records) { return relationships; } - private void processExportTraceServiceRequest( - final ExportTraceServiceRequest exportTraceServiceRequest, final Map batchStateData) { - exportTraceServiceRequest.getResourceSpansList().forEach(resourceSpans -> { - OTelHelper.getServiceName(resourceSpans.getResource()).ifPresent(serviceName -> resourceSpans.getInstrumentationLibrarySpansList().forEach( - instrumentationLibrarySpans -> { - instrumentationLibrarySpans.getSpansList().forEach( - span -> { - if (OTelHelper.checkValidSpan(span)) { - try { - batchStateData.put( - span.getSpanId().toByteArray(), - new ServiceMapStateData( - serviceName, - span.getParentSpanId().isEmpty() ? null : span.getParentSpanId().toByteArray(), - span.getTraceId().toByteArray(), - span.getKind().name(), - span.getName())); - } catch (RuntimeException e) { - LOG.error("Caught exception trying to put service map state data into batch", e); - } - if (span.getParentSpanId().isEmpty()) { - try { - currentTraceGroupWindow.put(span.getTraceId().toByteArray(), span.getName()); - } catch (RuntimeException e) { - LOG.error("Caught exception trying to put trace group name", e); - } - } - } else { - LOG.warn("Invalid span received"); - } - }); - } - )); - }); - } - private void processSpan(final Span span, final Map batchStateData) { if (span.getServiceName() != null) { final String serviceName = span.getServiceName(); diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java index 4165c2cb76..a041baec84 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapStatefulPrepperTest.java @@ -12,7 +12,6 @@ import com.amazon.dataprepper.model.trace.Span; import com.google.common.collect.Sets; import io.micrometer.core.instrument.Measurement; -import io.opentelemetry.proto.trace.v1.ResourceSpans; import org.apache.commons.codec.binary.Hex; import org.junit.Assert; import org.junit.Before; @@ -125,146 +124,6 @@ public void testPluginSettingConstructor() { final ServiceMapStatefulPrepper serviceMapStatefulPrepper = new ServiceMapStatefulPrepper(pluginSetting); } - // TODO: remove in 2.0 - @Test - public void testTraceGroupsWithExportTraceServiceRequestRecordData() throws Exception { - final Clock clock = Mockito.mock(Clock.class); - Mockito.when(clock.millis()).thenReturn(1L); - Mockito.when(clock.instant()).thenReturn(Instant.now()); - ExecutorService threadpool = Executors.newCachedThreadPool(); - final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful1 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); - final ServiceMapStatefulPrepper serviceMapStateful2 = new ServiceMapStatefulPrepper(100, path, clock, 2, PLUGIN_SETTING); - - final byte[] rootSpanId1 = ServiceMapTestUtils.getRandomBytes(8); - final byte[] rootSpanId2 = ServiceMapTestUtils.getRandomBytes(8); - final byte[] traceId1 = ServiceMapTestUtils.getRandomBytes(16); - final byte[] traceId2 = ServiceMapTestUtils.getRandomBytes(16); - final String traceGroup1 = "reset_password"; - final String traceGroup2 = "checkout"; - - final ResourceSpans frontendSpans1 = ServiceMapTestUtils.getResourceSpans(FRONTEND_SERVICE, traceGroup1, rootSpanId1, null, traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans authenticationSpansServer = ServiceMapTestUtils.getResourceSpans(AUTHENTICATION_SERVICE, "reset", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(frontendSpans1), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - final ResourceSpans authenticationSpansClient = ServiceMapTestUtils.getResourceSpans(AUTHENTICATION_SERVICE, "reset", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(authenticationSpansServer), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans passwordDbSpans = ServiceMapTestUtils.getResourceSpans(PASSWORD_DATABASE, "update", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(authenticationSpansClient), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - - final ResourceSpans frontendSpans2 = ServiceMapTestUtils.getResourceSpans(FRONTEND_SERVICE, traceGroup2, rootSpanId2, null, traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans checkoutSpansServer = ServiceMapTestUtils.getResourceSpans(CHECKOUT_SERVICE, "checkout", ServiceMapTestUtils.getRandomBytes(8), rootSpanId2, traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - final ResourceSpans checkoutSpansClient = ServiceMapTestUtils.getResourceSpans(CHECKOUT_SERVICE, "checkout", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(checkoutSpansServer), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans cartSpans = ServiceMapTestUtils.getResourceSpans(CART_SERVICE, "get_items", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(checkoutSpansClient), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - final ResourceSpans paymentSpans = ServiceMapTestUtils.getResourceSpans(PAYMENT_SERVICE, "charge", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(checkoutSpansClient), traceId2, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - - - //Expected relationships - final ServiceMapRelationship frontendAuth = ServiceMapRelationship.newDestinationRelationship(FRONTEND_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), AUTHENTICATION_SERVICE, "reset", traceGroup1); - final ServiceMapRelationship authPassword = ServiceMapRelationship.newDestinationRelationship(AUTHENTICATION_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), PASSWORD_DATABASE, "update", traceGroup1); - final ServiceMapRelationship frontendCheckout = ServiceMapRelationship.newDestinationRelationship(FRONTEND_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), CHECKOUT_SERVICE, "checkout", traceGroup2); - final ServiceMapRelationship checkoutCart = ServiceMapRelationship.newDestinationRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), CART_SERVICE, "get_items", traceGroup2); - final ServiceMapRelationship checkoutPayment = ServiceMapRelationship.newDestinationRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT.name(), PAYMENT_SERVICE, "charge", traceGroup2); - - final ServiceMapRelationship checkoutTarget = ServiceMapRelationship.newTargetRelationship(CHECKOUT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), CHECKOUT_SERVICE, "checkout", traceGroup2); - final ServiceMapRelationship authTarget = ServiceMapRelationship.newTargetRelationship(AUTHENTICATION_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), AUTHENTICATION_SERVICE, "reset", traceGroup1); - final ServiceMapRelationship passwordTarget = ServiceMapRelationship.newTargetRelationship(PASSWORD_DATABASE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), PASSWORD_DATABASE, "update", traceGroup1); - final ServiceMapRelationship paymentTarget = ServiceMapRelationship.newTargetRelationship(PAYMENT_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), PAYMENT_SERVICE, "charge", traceGroup2); - final ServiceMapRelationship cartTarget = ServiceMapRelationship.newTargetRelationship(CART_SERVICE, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER.name(), CART_SERVICE, "get_items", traceGroup2); - - final Set relationshipsFound = new HashSet<>(); - - //First batch - Mockito.when(clock.millis()).thenReturn(110L); - Future> r1 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, - Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(frontendSpans1, checkoutSpansServer)))); - Future> r2 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, - Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(frontendSpans2, checkoutSpansClient)))); - relationshipsFound.addAll(r1.get()); - relationshipsFound.addAll(r2.get()); - - //Shouldn't find any relationships - Assert.assertEquals(0, relationshipsFound.size()); - - //Second batch - Mockito.when(clock.millis()).thenReturn(220L); - Future> r3 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, - Arrays.asList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(authenticationSpansServer, authenticationSpansClient)), - new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(cartSpans)))); - Future> r4 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, - Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(passwordDbSpans, paymentSpans)))); - relationshipsFound.addAll(r3.get()); - relationshipsFound.addAll(r4.get()); - - //Should find the frontend->checkout relationship indicated in the first batch - Assert.assertEquals(2, relationshipsFound.size()); - assertTrue(relationshipsFound.containsAll(Arrays.asList( - frontendCheckout, - checkoutTarget - ))); - - //Third batch - Mockito.when(clock.millis()).thenReturn(340L); - Future> r5 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList()); - Future> r6 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList()); - relationshipsFound.addAll(r5.get()); - relationshipsFound.addAll(r6.get()); - - //Should find the rest of the relationships - Assert.assertEquals(10, relationshipsFound.size()); - assertTrue(relationshipsFound.containsAll(Arrays.asList( - frontendAuth, - authTarget, - authPassword, - passwordTarget, - checkoutCart, - cartTarget, - checkoutPayment, - paymentTarget - ))); - - // Extra validation - final List expectedSourceDests = Arrays.asList( - new ServiceMapSourceDest(FRONTEND_SERVICE, AUTHENTICATION_SERVICE), - new ServiceMapSourceDest(AUTHENTICATION_SERVICE, PASSWORD_DATABASE), - new ServiceMapSourceDest(FRONTEND_SERVICE, CHECKOUT_SERVICE), - new ServiceMapSourceDest(CHECKOUT_SERVICE, CART_SERVICE), - new ServiceMapSourceDest(CHECKOUT_SERVICE, PAYMENT_SERVICE) - ); - - assertTrue(evaluateEdges(relationshipsFound).containsAll(expectedSourceDests)); - - // Verify gauges - final List spansDbSizeMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") - .add(ServiceMapStatefulPrepper.SPANS_DB_SIZE).toString()); - Assert.assertEquals(1, spansDbSizeMeasurement.size()); - - final List traceGroupDbSizeMeasurement = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add("testPipelineName").add("testServiceMapPrepper") - .add(ServiceMapStatefulPrepper.TRACE_GROUP_DB_SIZE).toString()); - Assert.assertEquals(1, traceGroupDbSizeMeasurement.size()); - - - //Make sure that future relationships that are equivalent are caught by cache - final byte[] rootSpanId3 = ServiceMapTestUtils.getRandomBytes(8); - final byte[] traceId3 = ServiceMapTestUtils.getRandomBytes(16); - final ResourceSpans frontendSpans3 = ServiceMapTestUtils.getResourceSpans(FRONTEND_SERVICE, traceGroup1, rootSpanId3, rootSpanId3, traceId3, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans authenticationSpansServer2 = ServiceMapTestUtils.getResourceSpans(AUTHENTICATION_SERVICE, "reset", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(frontendSpans3), traceId3, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - - when(clock.millis()).thenReturn(450L); - Future> r7 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, - Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(frontendSpans3)))); - Future> r8 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, - Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(authenticationSpansServer2)))); - assertTrue(r7.get().isEmpty()); - assertTrue(r8.get().isEmpty()); - - when(clock.millis()).thenReturn(560L); - Future> r9 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful1, Arrays.asList()); - Future> r10 = ServiceMapTestUtils.startExecuteAsync(threadpool, serviceMapStateful2, Arrays.asList()); - assertTrue(r9.get().isEmpty()); - assertTrue(r10.get().isEmpty()); - serviceMapStateful1.shutdown(); - serviceMapStateful2.shutdown(); - } - @Test public void testTraceGroupsWithEventRecordData() throws Exception { final Clock clock = Mockito.mock(Clock.class); @@ -439,31 +298,6 @@ public void testTraceGroupsWithEventRecordData() throws Exception { serviceMapStateful2.shutdown(); } - // TODO: remove in 2.0 - @Test - public void testPrepareForShutdownWithExportTraceServiceRequestRecordData() throws Exception { - final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); - final ServiceMapStatefulPrepper serviceMapStateful = new ServiceMapStatefulPrepper(100, path, Clock.systemUTC(), 1, PLUGIN_SETTING); - - final byte[] rootSpanId1 = ServiceMapTestUtils.getRandomBytes(8); - final byte[] traceId1 = ServiceMapTestUtils.getRandomBytes(16); - final String traceGroup1 = "reset_password"; - - final ResourceSpans frontendSpans1 = ServiceMapTestUtils.getResourceSpans(FRONTEND_SERVICE, traceGroup1, rootSpanId1, null, traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_CLIENT); - final ResourceSpans authenticationSpansServer = ServiceMapTestUtils.getResourceSpans(AUTHENTICATION_SERVICE, "reset", ServiceMapTestUtils.getRandomBytes(8), ServiceMapTestUtils.getSpanId(frontendSpans1), traceId1, io.opentelemetry.proto.trace.v1.Span.SpanKind.SPAN_KIND_SERVER); - - serviceMapStateful.execute(Collections.singletonList(new Record<>(ServiceMapTestUtils.getExportTraceServiceRequest(frontendSpans1, authenticationSpansServer)))); - - assertFalse(serviceMapStateful.isReadyForShutdown()); - - serviceMapStateful.prepareForShutdown(); - serviceMapStateful.execute(Collections.emptyList()); - - assertTrue(serviceMapStateful.isReadyForShutdown()); - - serviceMapStateful.shutdown(); - } - @Test public void testPrepareForShutdownWithEventRecordData() { final File path = new File(ServiceMapPrepperConfig.DEFAULT_DB_PATH); diff --git a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapTestUtils.java b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapTestUtils.java index cff68dbf86..adca0aa2c0 100644 --- a/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapTestUtils.java +++ b/data-prepper-plugins/service-map-stateful/src/test/java/com/amazon/dataprepper/plugins/prepper/ServiceMapTestUtils.java @@ -5,23 +5,15 @@ package com.amazon.dataprepper.plugins.prepper; +import com.amazon.dataprepper.model.event.Event; import com.amazon.dataprepper.model.record.Record; import com.amazon.dataprepper.model.trace.DefaultTraceGroupFields; import com.amazon.dataprepper.model.trace.JacksonSpan; import com.amazon.dataprepper.model.trace.Span; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; -import io.opentelemetry.proto.trace.v1.ResourceSpans; -import java.io.UnsupportedEncodingException; import java.time.Instant; -import java.util.Arrays; import java.util.Collection; import java.util.Random; import java.util.Set; @@ -42,12 +34,8 @@ public static byte[] getRandomBytes(int len) { return bytes; } - public static byte[] getSpanId(ResourceSpans resourceSpans) { - return resourceSpans.getInstrumentationLibrarySpans(0).getSpans(0).getSpanId().toByteArray(); - } - public static Future> startExecuteAsync(ExecutorService threadpool, ServiceMapStatefulPrepper prepper, - Collection> records) { + Collection> records) { return threadpool.submit(() -> { return prepper.execute(records) .stream() @@ -61,50 +49,6 @@ public static Future> startExecuteAsync(ExecutorServ }); } - /** - * Creates a ResourceSpans object with the given parameters, with a single span - * @param serviceName Resource name for the ResourceSpans object - * @param spanName Span name for the single span in the ResourceSpans object - * @param spanId Span id for the single span in the ResourceSpans object - * @param parentId Parent id for the single span in the ResourceSpans object - * @param spanKind Span kind for the single span in the ResourceSpans object - * @return ResourceSpans object with a single span constructed according to the parameters - * @throws UnsupportedEncodingException - */ - public static ResourceSpans getResourceSpans(final String serviceName, final String spanName, final byte[] - spanId, final byte[] parentId, final byte[] traceId, final io.opentelemetry.proto.trace.v1.Span.SpanKind spanKind) throws UnsupportedEncodingException { - final ByteString parentSpanId = parentId != null ? ByteString.copyFrom(parentId) : ByteString.EMPTY; - return ResourceSpans.newBuilder() - .setResource( - Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey(OTelHelper.SERVICE_NAME_KEY) - .setValue(AnyValue.newBuilder().setStringValue(serviceName).build()).build()) - .build() - ) - .addInstrumentationLibrarySpans( - 0, - InstrumentationLibrarySpans.newBuilder() - .addSpans( - io.opentelemetry.proto.trace.v1.Span.newBuilder() - .setName(spanName) - .setKind(spanKind) - .setSpanId(ByteString.copyFrom(spanId)) - .setParentSpanId(parentSpanId) - .setTraceId(ByteString.copyFrom(traceId)) - .build() - ) - .build() - ) - .build(); - } - - public static ExportTraceServiceRequest getExportTraceServiceRequest(ResourceSpans...spans){ - return ExportTraceServiceRequest.newBuilder() - .addAllResourceSpans(Arrays.asList(spans)) - .build(); - } - /** * Creates a {@link com.amazon.dataprepper.model.trace.Span} object with the given parameters * @param serviceName Resource name for the ResourceSpans object diff --git a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml index a6dde191f0..984a02d971 100644 --- a/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml +++ b/deployment-template/ec2/data-prepper-ec2-deployment-cfn.yaml @@ -127,7 +127,7 @@ Resources: pipeline: name: "entry-pipeline" processor: - - otel_trace_raw_prepper: + - otel_trace_raw: sink: - opensearch: ${rawSpanConfig} service-map-pipeline: diff --git a/deployment-template/k8s/data-prepper-k8s.yaml b/deployment-template/k8s/data-prepper-k8s.yaml index c21ebc1165..c0b09b2a2c 100644 --- a/deployment-template/k8s/data-prepper-k8s.yaml +++ b/deployment-template/k8s/data-prepper-k8s.yaml @@ -27,7 +27,7 @@ data: pipeline: name: "entry-pipeline" processor: - - otel_trace_raw_prepper: + - otel_trace_raw: sink: - stdout: service-map-pipeline: diff --git a/docs/configuration.md b/docs/configuration.md index 38b7ed69ff..b846478723 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,7 +34,7 @@ raw-pipeline: pipeline: name: "entry-pipeline" processor: - - otel_trace_raw_prepper: + - otel_trace_raw: sink: - stdout: service-map-pipeline: diff --git a/docs/trace_analytics.md b/docs/trace_analytics.md index 1e38a1fadf..e0f50b4d44 100644 --- a/docs/trace_analytics.md +++ b/docs/trace_analytics.md @@ -33,7 +33,8 @@ The [OpenTelemetry source](../data-prepper-plugins/otel-trace-source/README.md) ### Processor We have two processor for the Trace Analytics feature, -* *otel_trace_raw_prepper* - This processor is responsible for converting the trace data in [OpenTelemetry specification](https://github.com/open-telemetry/opentelemetry-proto/tree/master/opentelemetry/proto/trace/v1) to OpenSearch-friendly (JSON) docs. These OpenSearch-friendly docs have certain additional fields like duration which are not part of the original OpenTelemetry specification. These additional fields are to make the instant OpenSearch Dashboards dashboards user-friendly. +* *otel_trace_raw* - This is a processor that receives collection of [Span](../../data-prepper-api/src/main/java/com/amazon/dataprepper/model/trace/Span.java) records sent from [otel-trace-source](../dataPrepper-plugins/otel-trace-source), does stateful processing on extracting and filling-in trace group related fields. +* *otel_trace_group* - This is a processor that fills in the missing trace group related fields in the collection of [Span](../../data-prepper-api/src/main/java/com/amazon/dataprepper/model/trace/Span.java) records by looking up the opensearch backend. * *service_map_stateful* - This processor performs the required preprocessing on the trace data and build metadata to display the service-map OpenSearch Dashboards dashboards. @@ -114,7 +115,19 @@ raw-pipeline: # With 64 as batch size each worker thread could process upto 3200 spans (64 * 50) batch_size: 64 processor: - - otel_trace_raw_prepper: + - otel_trace_raw: + - otel_trace_group: + hosts: [ "https://localhost:9200" ] + # Change to your credentials + username: "admin" + password: "admin" + # Add a certificate file if you are accessing an OpenSearch cluster with a self-signed certificate + #cert: /path/to/cert + # If you are connecting to an Amazon OpenSearch Service domain without + # Fine-Grained Access Control, enable these settings. Comment out the + # username and password above. + #aws_sigv4: true + #aws_region: us-east-1 sink: - opensearch: hosts: [ "https://localhost:9200" ] diff --git a/e2e-test/trace/README.md b/e2e-test/trace/README.md index 93a6010002..cb164acbcc 100644 --- a/e2e-test/trace/README.md +++ b/e2e-test/trace/README.md @@ -4,72 +4,33 @@ This module includes e2e tests for trace data ingestion supported by data-preppe ## Raw Span Ingestion Pipeline End-to-end test -### OTLP record type - -Run from current directory -``` -./gradlew :rawSpanOTLPEndToEndTest -``` -or from project root directory -``` -./gradlew :e2e-test:trace:rawSpanOTLPEndToEndTest -``` - -### Event record type compatibility with OTLP record type - Run from current directory ``` -./gradlew :rawSpanOTLPAndEventEndToEndTest +./gradlew :rawSpanEndToEndTest ``` or from project root directory ``` -./gradlew :e2e-test:trace:rawSpanOTLPAndEventEndToEndTest +./gradlew :e2e-test:trace:rawSpanEndToEndTest ``` ## Raw Span Ingestion Pipelines Latest Release Compatibility End-to-end test -### OTLP record type compatibility with latest release - Run from current directory ``` -./gradlew :rawSpanOTLPLatestReleaseCompatibilityEndToEndTest +./gradlew :rawSpanLatestReleaseCompatibilityEndToEndTest ``` or from project root directory ``` -./gradlew :e2e-test:trace:rawSpanOTLPLatestReleaseCompatibilityEndToEndTest -``` - -### Event record type compatibility with latest release - -Run from current directory -``` -./gradlew :rawSpanEventLatestReleaseCompatibilityEndToEndTest -``` -or from project root directory -``` -./gradlew :e2e-test:trace:rawSpanEventLatestReleaseCompatibilityEndToEndTest +./gradlew :e2e-test:trace:rawSpanLatestReleaseCompatibilityEndToEndTest ``` ## Service Map Ingestion Pipelines End-to-end test -### OTLP record type - -Run from current directory -``` -./gradlew :serviceMapOTLPEndToEndTest -``` -or from project root directory -``` -./gradlew :e2e-test:trace:serviceMapOTLPEndToEndTest -``` - -### Event record type compatibility with OTLP record type - Run from current directory ``` -./gradlew :serviceMapOTLPAndEventEndToEndTest +./gradlew :serviceMapEndToEndTest ``` or from project root directory ``` -./gradlew :e2e-test:trace:serviceMapOTLPAndEventEndToEndTest +./gradlew :e2e-test:trace:serviceMapEndToEndTest ``` diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index 7e1452e40c..be5f1f7afc 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -31,11 +31,11 @@ task removeDataPrepperNetwork(type: DockerRemoveNetwork) { } def RAW_SPAN_PIPELINE_YAML = "raw-span-e2e-pipeline.yml" -def RAW_SPAN_PIPELINE_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-event-type.yml" -def RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML = "raw-span-e2e-pipeline-peer-forwarder-event-type.yml" +// TODO: refactor this file once core-peerforwarder is ready +def RAW_SPAN_PIPELINE_PEER_FORWARDER_YAML = "raw-span-e2e-pipeline-peer-forwarder.yml" +// TODO: change or remove the file once core-peerforwarder is ready def RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML = "raw-span-e2e-pipeline-latest-release.yml" def SERVICE_MAP_PIPELINE_YAML = "service-map-e2e-pipeline.yml" -def SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML = "service-map-e2e-pipeline-event-type.yml" /** * DataPrepper Docker tasks @@ -49,11 +49,9 @@ task createDataPrepperDockerFile(type: Dockerfile) { workingDir("/app") copyFile("${dataPrepperJarFilepath}", "/app/data-prepper") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_YAML}", "/app/${RAW_SPAN_PIPELINE_YAML}") - copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}") - copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}", "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}") + copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_PEER_FORWARDER_YAML}", "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_YAML}") copyFile("src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}", "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_YAML}", "/app/${SERVICE_MAP_PIPELINE_YAML}") - copyFile("src/integrationTest/resources/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}", "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}") copyFile("src/integrationTest/resources/data_prepper.yml", "/app/data_prepper.yml") defaultCommand('java', '-cp', 'data-prepper/*', 'org.opensearch.dataprepper.DataPrepperExecute', "/app/${RAW_SPAN_PIPELINE_YAML}", "/app/data_prepper.yml") } @@ -193,62 +191,32 @@ def createEndToEndTest(final String testName, final String includeTestsMatchPatt def includeRawSpanTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndRawSpanTest.testPipelineEndToEnd*" -def createRawSpanDataPrepperOTLP1Task = createDataPrepperDockerContainer( +def createRawSpanDataPrepper1Task = createDataPrepperDockerContainer( "rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") -def createRawSpanDataPrepperOTLP2Task = createDataPrepperDockerContainer( +def createRawSpanDataPrepper2Task = createDataPrepperDockerContainer( "rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}") -// TODO: replace with Event type in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) -def rawSpanOTLPEndToEndTest = createEndToEndTest("rawSpanOTLPEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanDataPrepperOTLP1Task, createRawSpanDataPrepperOTLP2Task) - -def createRawSpanDataPrepperOTLPTask = createDataPrepperDockerContainer( - "rawSpanDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}") -def createRawSpanDataPrepperEventTask = createDataPrepperDockerContainer( - "rawSpanDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_EVENT_TYPE_YAML}") - -// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) -def rawSpanOTLPAndEventEndToEndTest = createEndToEndTest("rawSpanOTLPAndEventEndToEndTest", includeRawSpanTestsMatchPattern, - createRawSpanDataPrepperOTLPTask, createRawSpanDataPrepperEventTask) - -def rawSpanDataPrepperOTLPFromBuild = createDataPrepperDockerContainer( - "rawSpanDataPrepperOTLPFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") -def rawSpanDataPrepperOTLPFromPull = createDataPrepperDockerContainerFromPullImage( - "rawSpanDataPrepperOTLPFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") - -// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) -def rawSpanOTLPLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanOTLPLatestReleaseCompatibilityEndToEndTest", - includeRawSpanTestsMatchPattern, - rawSpanDataPrepperOTLPFromBuild, rawSpanDataPrepperOTLPFromPull) +def rawSpanEndToEndTest = createEndToEndTest("rawSpanEndToEndTest", includeRawSpanTestsMatchPattern, + createRawSpanDataPrepper1Task, createRawSpanDataPrepper2Task) def rawSpanDataPrepperEventFromBuild = createDataPrepperDockerContainer( - "rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_EVENT_TYPE_YAML}") + "rawSpanDataPrepperEventFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_PEER_FORWARDER_YAML}") def rawSpanDataPrepperLatestFromPull = createDataPrepperDockerContainerFromPullImage( "rawSpanDataPrepperLatestFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}") -def rawSpanEventLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanEventLatestReleaseCompatibilityEndToEndTest", +def rawSpanLatestReleaseCompatibilityEndToEndTest = createEndToEndTest("rawSpanLatestReleaseCompatibilityEndToEndTest", includeRawSpanTestsMatchPattern, rawSpanDataPrepperEventFromBuild, rawSpanDataPrepperLatestFromPull) def includeServiceMapTestsMatchPattern = "com.amazon.dataprepper.integration.trace.EndToEndServiceMapTest*" -def createServiceMapDataPrepperOTLP1Task = createDataPrepperDockerContainer( +def createServiceMapDataPrepper1Task = createDataPrepperDockerContainer( "serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") -def createServiceMapDataPrepperOTLP2Task = createDataPrepperDockerContainer( +def createServiceMapDataPrepper2Task = createDataPrepperDockerContainer( "serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}") -// TODO: replace with Event type in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) -def serviceMapOTLPEndToEndTest = createEndToEndTest("serviceMapOTLPEndToEndTest", includeServiceMapTestsMatchPattern, - createServiceMapDataPrepperOTLP1Task, createServiceMapDataPrepperOTLP2Task) - -def serviceMapDataPrepperOTLP = createDataPrepperDockerContainer( - "serviceMapDataPrepperOTLP", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}") -def serviceMapDataPrepperEvent = createDataPrepperDockerContainer( - "serviceMapDataPrepperEvent", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_EVENT_TYPE_YAML}") - -// TODO: remove in 2.0 (https://github.com/opensearch-project/data-prepper/issues/1272) -def serviceMapOTLPAndEventEndToEndTest = createEndToEndTest("serviceMapOTLPAndEventEndToEndTest", includeServiceMapTestsMatchPattern, - serviceMapDataPrepperOTLP, serviceMapDataPrepperEvent) +def serviceMapEndToEndTest = createEndToEndTest("serviceMapEndToEndTest", includeServiceMapTestsMatchPattern, + createServiceMapDataPrepper1Task, createServiceMapDataPrepper2Task) dependencies { integrationTestImplementation project(':data-prepper-api') diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-event-type.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-event-type.yml deleted file mode 100644 index f6f237082e..0000000000 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-event-type.yml +++ /dev/null @@ -1,25 +0,0 @@ -entry-pipeline: - source: - otel_trace_source: - ssl: false - record_type: "event" - sink: - - pipeline: - name: "raw-pipeline" -raw-pipeline: - source: - pipeline: - name: "entry-pipeline" - processor: - - otel_trace_raw: - trace_flush_interval: 5 - - otel_trace_group: - hosts: [ "https://node-0.example.com:9200" ] - username: "admin" - password: "admin" - sink: - - opensearch: - hosts: [ "https://node-0.example.com:9200" ] - username: "admin" - password: "admin" - index_type: trace-analytics-raw \ No newline at end of file diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder-event-type.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder.yml similarity index 95% rename from e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder-event-type.yml rename to e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder.yml index c97356d4a6..6a08f62e77 100644 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder-event-type.yml +++ b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline-peer-forwarder.yml @@ -2,7 +2,6 @@ entry-pipeline: source: otel_trace_source: ssl: false - record_type: "event" processor: - peer_forwarder: discovery_mode: "static" diff --git a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml index cdca3cc74e..551ad9dd4e 100644 --- a/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml +++ b/e2e-test/trace/src/integrationTest/resources/raw-span-e2e-pipeline.yml @@ -9,10 +9,10 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: trace_flush_interval: 5 - - otel_trace_group_prepper: + - otel_trace_group: hosts: [ "https://node-0.example.com:9200" ] username: "admin" password: "admin" diff --git a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline-event-type.yml b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline-event-type.yml deleted file mode 100644 index feb4ed2a8a..0000000000 --- a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline-event-type.yml +++ /dev/null @@ -1,26 +0,0 @@ -entry-pipeline: - source: - otel_trace_source: - ssl: false - record_type: "event" - processor: - - peer_forwarder: - discovery_mode: "static" - static_endpoints: ["dataprepper1", "dataprepper2"] - ssl: false - sink: - - pipeline: - name: "service-map-pipeline" -service-map-pipeline: - source: - pipeline: - name: "entry-pipeline" - processor: - - service_map_stateful: - window_duration: 4 - sink: - - opensearch: - hosts: ["https://node-0.example.com:9200"] - username: "admin" - password: "admin" - index_type: trace-analytics-service-map \ No newline at end of file diff --git a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml index f1e3b65a18..cd53f19191 100644 --- a/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml +++ b/e2e-test/trace/src/integrationTest/resources/service-map-e2e-pipeline.yml @@ -2,7 +2,7 @@ entry-pipeline: source: otel_trace_source: ssl: false - prepper: + processor: - peer_forwarder: discovery_mode: "static" static_endpoints: ["dataprepper1", "dataprepper2"] @@ -14,7 +14,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: window_duration: 4 sink: diff --git a/examples/config/example-pipelines.yaml b/examples/config/example-pipelines.yaml index d73e1a3bac..986345de9c 100644 --- a/examples/config/example-pipelines.yaml +++ b/examples/config/example-pipelines.yaml @@ -12,8 +12,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://..es.amazonaws.com" ] @@ -25,7 +25,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/examples/dev/data-prepper-emf-monitoring/data-prepper-emf-demo-cfn.yaml b/examples/dev/data-prepper-emf-monitoring/data-prepper-emf-demo-cfn.yaml index 2e0fe579d3..718cb1de85 100644 --- a/examples/dev/data-prepper-emf-monitoring/data-prepper-emf-demo-cfn.yaml +++ b/examples/dev/data-prepper-emf-monitoring/data-prepper-emf-demo-cfn.yaml @@ -51,8 +51,8 @@ Resources: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - stdout: - Name: DATA_PREPPER_CONFIG_YAML diff --git a/examples/dev/data-prepper-emf-monitoring/pipelines-raw-trace-stdout.yaml b/examples/dev/data-prepper-emf-monitoring/pipelines-raw-trace-stdout.yaml index 0f96499234..e71252a71c 100644 --- a/examples/dev/data-prepper-emf-monitoring/pipelines-raw-trace-stdout.yaml +++ b/examples/dev/data-prepper-emf-monitoring/pipelines-raw-trace-stdout.yaml @@ -10,8 +10,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: # trace_flush_interval: 6 sink: - stdout: diff --git a/examples/dev/k8s/data-prepper.yaml b/examples/dev/k8s/data-prepper.yaml index 0dc09bdc77..f120ad360d 100644 --- a/examples/dev/k8s/data-prepper.yaml +++ b/examples/dev/k8s/data-prepper.yaml @@ -15,7 +15,7 @@ data: source: otel_trace_source: ssl: false - prepper: + processor: - peer_forwarder: discovery_mode: "dns" domain_name: "data-prepper-headless" @@ -29,8 +29,8 @@ data: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://opensearch:9200" ] @@ -43,7 +43,7 @@ data: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/examples/dev/trace-analytics-sample-app/Dockerfile b/examples/dev/trace-analytics-sample-app/Dockerfile index daa5c3b378..1c511d350a 100644 --- a/examples/dev/trace-analytics-sample-app/Dockerfile +++ b/examples/dev/trace-analytics-sample-app/Dockerfile @@ -1,4 +1,4 @@ -FROM gradle:jdk14 AS builder +FROM gradle:jdk11 AS builder COPY . /home/gradle/src WORKDIR /home/gradle/src RUN gradle -p data-prepper-core clean jar --no-daemon diff --git a/examples/dev/trace-analytics-sample-app/resources/pipelines.yaml b/examples/dev/trace-analytics-sample-app/resources/pipelines.yaml index 85b8a9a314..78b2e7ee65 100644 --- a/examples/dev/trace-analytics-sample-app/resources/pipelines.yaml +++ b/examples/dev/trace-analytics-sample-app/resources/pipelines.yaml @@ -5,7 +5,7 @@ entry-pipeline: ssl: true sslKeyFile: /usr/share/data-prepper/demo-data-prepper.key sslKeyCertChainFile: /usr/share/data-prepper/demo-data-prepper.crt - prepper: + processor: - peer_forwarder: discovery_mode: "dns" domain_name: "prepper-cluster" @@ -19,8 +19,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://node-0.example.com:9200" ] @@ -33,7 +33,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/examples/trace_analytics.yml b/examples/trace_analytics.yml index beeb0f1a1c..b3f9be8b31 100644 --- a/examples/trace_analytics.yml +++ b/examples/trace_analytics.yml @@ -14,8 +14,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://node-0.example.com:9200" ] @@ -28,7 +28,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/examples/trace_analytics_no_ssl.yml b/examples/trace_analytics_no_ssl.yml index af343ccbe0..c6b813cffd 100644 --- a/examples/trace_analytics_no_ssl.yml +++ b/examples/trace_analytics_no_ssl.yml @@ -12,8 +12,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://node-0.example.com:9200" ] @@ -26,7 +26,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/release/smoke-tests/data-prepper/config/pipelines.yaml b/release/smoke-tests/data-prepper/config/pipelines.yaml index d573f0f5ae..28a1da0547 100644 --- a/release/smoke-tests/data-prepper/config/pipelines.yaml +++ b/release/smoke-tests/data-prepper/config/pipelines.yaml @@ -26,8 +26,8 @@ raw-pipeline: source: pipeline: name: "entry-pipeline" - prepper: - - otel_trace_raw_prepper: + processor: + - otel_trace_raw: sink: - opensearch: hosts: [ "https://node-0.example.com:9200" ] @@ -40,7 +40,7 @@ service-map-pipeline: source: pipeline: name: "entry-pipeline" - prepper: + processor: - service_map_stateful: sink: - opensearch: diff --git a/research/build.gradle b/research/build.gradle deleted file mode 100644 index 9e182dcd1f..0000000000 --- a/research/build.gradle +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - diff --git a/research/zipkin-opensearch-to-otel/README.md b/research/zipkin-opensearch-to-otel/README.md deleted file mode 100644 index d8b310364b..0000000000 --- a/research/zipkin-opensearch-to-otel/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Zipkin trace data from OpenSearch to Otel Trace Source - -This is a demo on reading zipkin trace analytics data from local OpenSearch cluster to Otel Trace source. - -## Prerequisite - -* OpenSearch docker: listening on https://localhost:9200 with `admin` as user and password. -* Otel Trace Source: listening on http://127.0.0.1:21890. - -## Run - -``` -./gradlew :research:zipkin-opensearch-to-otel:run --args YOUR_INDEX_PATTERN FIELD VALUE -``` - -Notice that `FIELD` and `VALUE` are optional arguments. - -* Test mode: with environment variable test set to `true`. A test Otel trace source will be launched listening on http://127.0.0.1:21890. - -``` -./gradlew :research:zipkin-opensearch-to-otel:run -Dtest=true --args YOUR_INDEX_PATTERN -``` diff --git a/research/zipkin-opensearch-to-otel/build.gradle b/research/zipkin-opensearch-to-otel/build.gradle deleted file mode 100644 index 66aa06931e..0000000000 --- a/research/zipkin-opensearch-to-otel/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -plugins { - id 'java' - id 'application' -} - -group 'com.amazon' -version '0.1-beta' - -repositories { - mavenCentral() -} - -application { - mainClass = 'com.amazon.dataprepper.research.zipkin.ZipkinOpenSearchToOtel' -} - -run { - if (System.getProperty("test") != null) { - systemProperty "test", System.getProperty("test") - } -} - -dependencies { - implementation project(':data-prepper-plugins:blocking-buffer') - implementation project(':data-prepper-api') - implementation project(':data-prepper-plugins:common') - implementation project(':data-prepper-plugins:opensearch') - implementation project(':data-prepper-plugins:otel-trace-source') - implementation project(':data-prepper-plugins:otel-trace-raw-prepper') - implementation "org.apache.commons:commons-lang3:3.11" - implementation "com.linecorp.armeria:armeria:${versionMap.armeria}" - implementation "com.linecorp.armeria:armeria-grpc:${versionMap.armeriaGrpc}" - implementation "org.opensearch.client:opensearch-rest-high-level-client:${versionMap.opensearchVersion}" - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation "io.opentelemetry:opentelemetry-proto:${versionMap.opentelemetryProto}" -} diff --git a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/OpenSearchReader.java b/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/OpenSearchReader.java deleted file mode 100644 index 5342c4e1e1..0000000000 --- a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/OpenSearchReader.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.research.zipkin; - -import org.opensearch.action.search.ClearScrollRequest; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class OpenSearchReader { - private String scrollId = null; - private final String indexPattern; - private final SearchSourceBuilder searchSourceBuilder; - private final TimeValue keepAlive = TimeValue.timeValueMinutes(1); - private long total; - - public OpenSearchReader(final String indexPattern, final String field, final String value) { - this.indexPattern = indexPattern; - if (field != null && value != null) { - searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchQuery(field, value)); - } else { - searchSourceBuilder = null; - } - } - - public long getTotal() { - return total; - } - - public List> nextBatch(final RestHighLevelClient restHighLevelClient) throws IOException { - if (scrollId == null) { - final SearchRequest searchRequest = new SearchRequest(indexPattern); - searchRequest.scroll(keepAlive); - if (searchSourceBuilder != null) { - searchRequest.source(searchSourceBuilder); - } - final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - // Update total hits - total = searchResponse.getHits().getTotalHits().value; - updateScrollId(searchResponse); - final SearchHits hits = searchResponse.getHits(); - return Arrays.stream(hits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.toList()); - } else { - final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(keepAlive); - final SearchResponse searchScrollResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT); - updateScrollId(searchScrollResponse); - final SearchHits hits = searchScrollResponse.getHits(); - return Arrays.stream(hits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.toList()); - } - } - - private void updateScrollId(final SearchResponse searchResponse) { - scrollId = searchResponse.getScrollId(); - } - - public void clearScroll(final RestHighLevelClient restHighLevelClient) throws IOException { - final ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - } -} diff --git a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtel.java b/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtel.java deleted file mode 100644 index 51cecc799b..0000000000 --- a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtel.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.research.zipkin; - -import com.amazon.dataprepper.model.configuration.PluginSetting; -import com.amazon.dataprepper.model.record.Record; -import com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; -import com.amazon.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import com.amazon.dataprepper.plugins.source.oteltrace.OTelTraceSource; -import com.linecorp.armeria.client.Clients; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import org.opensearch.client.RestHighLevelClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ZipkinOpenSearchToOtel { - public static final Logger LOG = LoggerFactory.getLogger(ZipkinOpenSearchToOtel.class); - - public static void main(final String[] args) throws IOException { - if (args.length < 1) { - System.err.println("Missing indexPattern as arg"); - System.exit(1); - } - final String indexPattern = args[0]; - final String field = args.length >= 2? args[1] : null; - final String value = args.length >= 3? args[2] : null; - final boolean isTest = !System.getProperty("test", "false").equalsIgnoreCase("false"); - OTelTraceSource oTelTraceSource = null; - if (isTest) { - System.out.println("Setting up testing OtelTraceSource"); - oTelTraceSource = setUpOtelTraceSource(); - } - final ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration.Builder( - Collections.singletonList("https://localhost:9200")) - .withUsername("admin") - .withPassword("admin") - .build(); - final RestHighLevelClient restHighLevelClient = connectionConfiguration.createClient(); - final OpenSearchReader reader = new OpenSearchReader(indexPattern, field, value); - final TraceServiceGrpc.TraceServiceBlockingStub client = createGRPCClient(); - System.out.println("Reading batch 0"); - List> sources = reader.nextBatch(restHighLevelClient); - System.out.println(String.format("Batch size: %d", sources.size())); - System.out.println(String.format("Total number of hits: %d", reader.getTotal())); - int i = 0; - while (sources.size() > 0) { - System.out.println(String.format("Processing batch %d as ExportTraceServiceRequest", i)); - try { - final ExportTraceServiceRequest exportTraceServiceRequest = ZipkinOpenSearchToOtelPrepper.sourcesToRequest(sources); - client.export(exportTraceServiceRequest); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - System.out.println(String.format("Reading batch %d", i+1)); - sources = reader.nextBatch(restHighLevelClient); - i++; - } - System.out.println("Clearing reader scroll context ..."); - reader.clearScroll(restHighLevelClient); - System.out.println("Closing REST client"); - restHighLevelClient.close(); - if (isTest) { - closeOtelTraceSource(oTelTraceSource); - } - } - - public static TraceServiceGrpc.TraceServiceBlockingStub createGRPCClient() { - return Clients.newClient("gproto+http://127.0.0.1:21890/", TraceServiceGrpc.TraceServiceBlockingStub.class); - } - - public static OTelTraceSource setUpOtelTraceSource() { - final HashMap integerHashMap = new HashMap<>(); - integerHashMap.put("request_timeout", 1); - final OTelTraceSource SOURCE = new OTelTraceSource(null, null, null); - SOURCE.start(getBuffer()); - return SOURCE; - } - - public static void closeOtelTraceSource(final OTelTraceSource oTelTraceSource) { - oTelTraceSource.stop(); - } - - private static BlockingBuffer> getBuffer() { - final HashMap integerHashMap = new HashMap<>(); - integerHashMap.put("buffer_size", 5); - return new BlockingBuffer<>(new PluginSetting("blocking_buffer", integerHashMap)); - } -} diff --git a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtelPrepper.java b/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtelPrepper.java deleted file mode 100644 index 9c4379b20d..0000000000 --- a/research/zipkin-opensearch-to-otel/src/main/java/com/amazon/dataprepper/research/zipkin/ZipkinOpenSearchToOtelPrepper.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.amazon.dataprepper.research.zipkin; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.ByteString; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.resource.v1.Resource; -import io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans; -import io.opentelemetry.proto.trace.v1.ResourceSpans; -import io.opentelemetry.proto.trace.v1.Span; -import io.opentelemetry.proto.trace.v1.Status; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@SuppressWarnings("unchecked") -public class ZipkinOpenSearchToOtelPrepper { - public static String SPAN_ID = "id"; - public static String NAME = "name"; - public static String TRACE_ID = "traceId"; - public static String TIME_STAMP = "timestamp"; - public static String DURATION = "duration"; - public static String SPAN_KIND = "kind"; - public static String PARENT_ID = "parentId"; - public static String LOCAL_ENDPOINT = "localEndpoint"; - public static String SERVICE_NAME = "serviceName"; - public static String TAGS = "tags"; - public static String HTTP_STATUS_CODE = "http.status_code"; - public static String RESPONSE_STATUS = "response.status"; - public static String ERROR = "error"; - public static String RETURN_VALUE = "return value"; - public static String STATUS_CODE_VALUE = "statusCodeValue"; - - private static final ObjectMapper mapper = new ObjectMapper(); - private static final TypeReference> typeRef = new TypeReference>() {}; - - public static Span sourceToSpan(final Map source) { - final String traceID = (String) source.get(TRACE_ID); - final String spanID = (String) source.get(SPAN_ID); - final String name = (String) source.get(NAME); - final Long startTime = (Long) source.get(TIME_STAMP); - final Long duration = Long.valueOf((Integer) source.get(DURATION)); - final long endTime = startTime + duration; - final String parentID = (String) source.get(PARENT_ID); - final String spanKind = (String) source.get(SPAN_KIND); - // TODO: read span status from tags - final Map tags = (Map) source.get(TAGS); - Integer statusCode = null; - if (tags != null) { - statusCode = extractStatusCodeFromTags(tags); - } - - final Span.Builder spanBuilder = Span.newBuilder() - .setStartTimeUnixNano(startTime * 1000) // Convert to UnixNano - .setEndTimeUnixNano(endTime * 1000); // Convert to UnixNano - - if (traceID != null) { - spanBuilder.setTraceId(ByteString.copyFromUtf8(traceID)); - } - if (spanID != null) { - spanBuilder.setSpanId(ByteString.copyFromUtf8(spanID)); - } - if (name != null) { - spanBuilder.setName(name); - } - if (parentID != null) { - spanBuilder.setParentSpanId(ByteString.copyFromUtf8(parentID)); - } - if (spanKind != null) { - spanBuilder.setKind(Span.SpanKind.valueOf(spanKind)); - } - if (statusCode != null) { - spanBuilder.setStatus(Status.newBuilder().setCodeValue(statusCode)); - } - - return spanBuilder.build(); - } - - public static Integer extractStatusCodeFromTags(final Map tags) { - for (final Map.Entry entry:tags.entrySet()) { - final String key = entry.getKey(); - if (key.equals(HTTP_STATUS_CODE)) { - final String value = (String) entry.getValue(); - if (StringUtils.isNumeric(value)) { - return Integer.parseInt(value); - } - } else if (key.equals(RESPONSE_STATUS)) { - final String value = (String) entry.getValue(); - if (StringUtils.isNumeric(value)) { - return Integer.parseInt(value); - } - } else if (key.equals(ERROR)) { - final String value = (String) entry.getValue(); - if (StringUtils.isNumeric(value)) { - return Integer.parseInt(value); - } - } else if (key.equals(RETURN_VALUE)) { - // Extract only if value is json string - try { - final Map value = mapper.readValue((String) entry.getValue(), typeRef); - if (value != null) { - final Integer statusCodeValue = (Integer) value.get(STATUS_CODE_VALUE); - if (statusCodeValue != null) { - return statusCodeValue; - } - } - } catch (final JsonProcessingException ignored) { } - } - } - - return null; - } - - public static ExportTraceServiceRequest sourcesToRequest(final List> sources) { - final ExportTraceServiceRequest.Builder exportTraceServiceRequestBuilder = ExportTraceServiceRequest.newBuilder(); - final Map>> sourceByService = groupSourceByService(sources); - for (final Map.Entry>> entry : sourceByService.entrySet()) { - final String serviceName = entry.getKey(); - final List> sourceGroup = entry.getValue(); - final ResourceSpans.Builder rsBuilder = ResourceSpans.newBuilder().setResource(Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey(SERVICE_NAME) - .setValue(AnyValue.newBuilder().setStringValue(serviceName)) - .build() - ) - .build() - ); - // TODO: add library name and version - final InstrumentationLibrarySpans.Builder isBuilder = - InstrumentationLibrarySpans.newBuilder(); - final List spanGroup = sourceGroup.stream() - .map(ZipkinOpenSearchToOtelPrepper::sourceToSpan).collect(Collectors.toList()); - isBuilder.addAllSpans(spanGroup); - rsBuilder.addInstrumentationLibrarySpans(isBuilder); - exportTraceServiceRequestBuilder.addResourceSpans(rsBuilder); - } - - return exportTraceServiceRequestBuilder.build(); - } - - public static Map>> groupSourceByService(final List> sources) { - final Map>> sourceByService = new HashMap<>(); - for (final Map source: sources) { - String serviceName = null; - final Map localEndpoint = (Map) source.get(LOCAL_ENDPOINT); - if (localEndpoint != null) { - serviceName = (String) localEndpoint.get(SERVICE_NAME); - } - if (sourceByService.containsKey(serviceName)) { - sourceByService.get(serviceName).add(source); - } else { - sourceByService.put(serviceName, new ArrayList<>(Arrays.asList(source))); - } - } - return sourceByService; - } -} diff --git a/settings.gradle b/settings.gradle index 9fc8633303..b637184ebf 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,12 +28,8 @@ include 'data-prepper-plugins:armeria-common' include 'data-prepper-plugins:opensearch' include 'data-prepper-plugins:service-map-stateful' include 'data-prepper-plugins:mapdb-prepper-state' -include 'research' -include 'research:zipkin-opensearch-to-otel' include 'data-prepper-plugins:otel-proto-common' -include 'data-prepper-plugins:otel-trace-raw-prepper' include 'data-prepper-plugins:otel-trace-raw-processor' -include 'data-prepper-plugins:otel-trace-group-prepper' include 'data-prepper-plugins:otel-trace-group-processor' include 'data-prepper-plugins:otel-trace-source' include 'data-prepper-plugins:otel-metrics-source'