From 60739ab3f9cfd0053c4b0f78d4410526bf58f382 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 18 Apr 2024 09:28:02 -0700 Subject: [PATCH] Final changes from POC, adds agg support Signed-off-by: Chase Engelbrecht --- .../model/document/JacksonDocument.java | 2 +- .../dataprepper/model/event/Event.java | 2 +- .../dataprepper/model/event/JacksonEvent.java | 6 +- .../dataprepper/model/log/JacksonLog.java | 2 +- .../dataprepper/model/log/JacksonOtelLog.java | 2 +- .../metric/JacksonExponentialHistogram.java | 2 +- .../model/metric/JacksonGauge.java | 2 +- .../model/metric/JacksonHistogram.java | 2 +- .../dataprepper/model/metric/JacksonSum.java | 2 +- .../model/metric/JacksonSummary.java | 2 +- .../dataprepper/model/trace/JacksonSpan.java | 4 +- .../model/document/JacksonDocumentTest.java | 4 +- .../model/event/JacksonEventTest.java | 28 ++-- .../JacksonEvent_JavaSerializationTest.java | 8 +- .../dataprepper/model/log/JacksonLogTest.java | 4 +- .../model/trace/JacksonSpanTest.java | 22 +-- .../codec/JacksonPeerForwarderCodec.java | 6 +- .../router/RouterCopyRecordStrategy.java | 3 +- .../DefaultEventBuilderFactoryTests.java | 2 +- .../DefaultLogEventBuilderFactoryTests.java | 2 +- .../GetMetadataExpressionFunction.java | 2 +- .../expression/HasTagsExpressionFunction.java | 2 +- ...ericExpressionEvaluator_ConditionalIT.java | 4 +- .../HasTagsExpressionFunctionTest.java | 2 +- .../actions/AppendAggregateActionTest.java | 2 +- .../actions/PutAllAggregateActionTest.java | 2 +- .../codec/avro/AvroInputCodecTest.java | 4 +- .../plugins/processor/StringProcessor.java | 2 +- .../processor/StringProcessorTests.java | 12 +- .../plugins/sink/FileSinkTests.java | 2 +- .../plugins/codec/csv/CsvCodecTest.java | 52 +++--- data-prepper-plugins/date-processor/README.md | 2 +- .../plugins/processor/date/DateProcessor.java | 4 +- .../processor/date/DateProcessorTests.java | 4 +- .../decompress/DecompressProcessor.java | 6 +- .../decompress/DecompressProcessorTest.java | 18 +-- .../dynamodb/converter/RecordConverter.java | 4 +- .../converter/ExportRecordConverterTest.java | 28 ++-- .../converter/StreamRecordConverterTest.java | 92 +++++------ .../plugins/processor/GeoIPProcessor.java | 2 +- .../plugins/processor/GeoIPProcessorTest.java | 2 +- .../plugins/processor/grok/GrokProcessor.java | 6 +- .../processor/grok/GrokProcessorIT.java | 4 +- .../processor/grok/GrokProcessorTests.java | 20 +-- .../kafka/source/KafkaSourceJsonTypeIT.java | 12 +- .../kafka/consumer/KafkaCustomConsumer.java | 2 +- .../processor/keyvalue/KeyValueProcessor.java | 6 +- .../keyvalue/KeyValueProcessorTests.java | 2 +- .../mutateevent/AddEntryProcessor.java | 6 +- .../ConvertEntryTypeProcessor.java | 4 +- .../mutateevent/ListToMapProcessor.java | 10 +- .../mutateevent/MapToListProcessor.java | 4 +- .../mutateevent/AddEntryProcessorTests.java | 18 +-- .../ConvertEntryTypeProcessorTests.java | 8 +- .../mutateevent/ListToMapProcessorTest.java | 2 +- .../mutateevent/MapToListProcessorTest.java | 2 +- .../newline/NewlineDelimitedCodecTest.java | 16 +- .../obfuscation/ObfuscationProcessor.java | 2 +- .../obfuscation/ObfuscationProcessorTest.java | 6 +- .../sink/opensearch/OpenSearchSinkIT.java | 10 +- .../sink/opensearch/OpenSearchSink.java | 6 + .../worker/NoSearchContextWorker.java | 4 +- .../source/opensearch/worker/PitWorker.java | 4 +- .../opensearch/worker/ScrollWorker.java | 4 +- .../opensearch/index/DocumentBuilderTest.java | 2 +- .../codec/parquet/ParquetInputCodecTest.java | 4 +- .../parse/AbstractParseProcessor.java | 2 +- .../codec/json/JsonInputCodecTest.java | 16 +- .../parse/json/ParseJsonProcessorTest.java | 2 +- data-prepper-plugins/rule-engine/build.gradle | 1 + .../rule-engine/src/main/antlr/Aggregation.g4 | 56 +++++++ .../plugins/processor/RuleEngine.java | 19 +++ .../plugins/processor/RuleEngineConfig.java | 34 +--- .../processor/RuleEngineProcessor.java | 18 ++- .../processor/RuleEngineProcessorConfig.java | 7 + .../converters/FindingConverter.java | 38 +++-- .../evaluator/CompositeRuleEvaluator.java | 19 +++ .../evaluator/CorrelationEvaluator.java | 9 ++ .../DefaultCorrelationEvaluator.java | 147 +++++++++++++++++ .../evaluator/DefaultRuleEvaluator.java | 21 ++- .../formats/accessors/FieldAccessor.java | 6 +- .../processor/model/datatypes/CloudTrail.java | 5 + .../processor/model/datatypes/DataType.java | 113 ++----------- .../processor/model/datatypes/ocsf/OCSF.java | 102 +++++++++++- .../processor/model/matches/Match.java | 30 ++-- .../processor/model/matches/SubMatch.java | 14 ++ .../processor/model/rule/RuleSchema.java | 2 - .../parser/OpenSearchSigmaV1RuleParser.java | 152 ++++++++++++++++-- .../plugins/processor/parser/RuleParser.java | 7 +- .../processor/parser/SigmaV1RuleParser.java | 35 ---- .../parser/SigmaV1RuleStatefulConverter.java | 67 ++++++++ ...ava => SigmaV1RuleStatelessConverter.java} | 29 ++-- .../parser/aggregation/AggregationItem.java | 62 +++++++ .../AggregationTraverseVisitor.java | 61 +++++++ .../parser/objects/SigmaCondition.java | 23 ++- .../opensearch/OpenSearchRuleProvider.java | 2 +- .../registrar/SubMatchAccessorRegistrar.java | 28 ++++ .../OpenSearchSubMatchAccessor.java | 119 ++++++++++++++ .../retrievers/SubMatchAccessor.java | 12 ++ .../rules/OpenSearchRuleMetadata.java | 12 ++ .../rules/OpenSearchSigmaV1Rule.java | 33 ---- .../rules/OpenSearchSigmaV1StatefulRule.java | 11 ++ .../rules/OpenSearchSigmaV1StatelessRule.java | 11 ++ .../plugins/processor/rules/Rule.java | 29 ++-- .../processor/rules/RuleRefresher.java | 11 +- .../plugins/processor/rules/RuleStore.java | 19 ++- .../plugins/processor/rules/SigmaV1Rule.java | 32 ---- .../processor/rules/SigmaV1RuleMetadata.java | 13 ++ .../plugins/processor/rules/StatefulRule.java | 15 ++ .../processor/rules/StatelessRule.java | 11 ++ ...va => SigmaV1RuleConditionParserTest.java} | 0 .../plugins/sink/s3/S3SinkServiceIT.java | 4 +- .../useragent/UserAgentProcessor.java | 2 +- .../useragent/UserAgentProcessorTest.java | 4 +- 114 files changed, 1376 insertions(+), 608 deletions(-) create mode 100644 data-prepper-plugins/rule-engine/src/main/antlr/Aggregation.g4 create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CompositeRuleEvaluator.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CorrelationEvaluator.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultCorrelationEvaluator.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/SubMatch.java delete mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleParser.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatefulConverter.java rename data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/{SigmaV1RuleConditionParser.java => SigmaV1RuleStatelessConverter.java} (84%) create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationItem.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationTraverseVisitor.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/registrar/SubMatchAccessorRegistrar.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/OpenSearchSubMatchAccessor.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/SubMatchAccessor.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchRuleMetadata.java delete mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1Rule.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatefulRule.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatelessRule.java delete mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1Rule.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1RuleMetadata.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatefulRule.java create mode 100644 data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatelessRule.java rename data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/{SigmaV1SigmaV1RuleConditionParserTest.java => SigmaV1RuleConditionParserTest.java} (100%) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/document/JacksonDocument.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/document/JacksonDocument.java index 64e1ac8bc2..c06b11596f 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/document/JacksonDocument.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/document/JacksonDocument.java @@ -18,7 +18,7 @@ public class JacksonDocument extends JacksonEvent implements Document { protected JacksonDocument(final Builder builder) { super(builder); - checkArgument(this.getMetadata().getEventType().equals("DOCUMENT"), "eventType must be of type Document"); + checkArgument(this.getEventMetadata().getEventType().equals("DOCUMENT"), "eventType must be of type Document"); } public static Builder builder() { diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java index a27906b2cc..0ac6101ec9 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java @@ -96,7 +96,7 @@ public interface Event extends Serializable { * @return EventMetadata for the event * @since 1.2 */ - EventMetadata getMetadata(); + EventMetadata getEventMetadata(); /** * Checks if the key exists. diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index 62b31ad0c5..3b9c2ca636 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -367,7 +367,7 @@ private String formatStringInternal(final String format, final ExpressionEvaluat } @Override - public EventMetadata getMetadata() { + public EventMetadata getEventMetadata() { return eventMetadata; } @@ -477,7 +477,7 @@ public static JacksonEvent fromEvent(final Event event) { } else { return JacksonEvent.builder() .withData(event.toMap()) - .withEventMetadata(event.getMetadata()) + .withEventMetadata(event.getEventMetadata()) .build(); } } @@ -609,7 +609,7 @@ public String toJsonString() { final String tagsKey = getTagsKey(); if (tagsKey != null) { - final JsonNode tagsNode = mapper.valueToTree(event.getMetadata().getTags()); + final JsonNode tagsNode = mapper.valueToTree(event.getEventMetadata().getTags()); return jsonString.substring(0, jsonString.length() - 1) + ",\"" + tagsKey + "\":" + tagsNode.toString() + "}"; } return jsonString; diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonLog.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonLog.java index d05ebd90e4..276bb06101 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonLog.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonLog.java @@ -18,7 +18,7 @@ public class JacksonLog extends JacksonEvent implements Log { protected JacksonLog(final Builder builder) { super(builder); - checkArgument(this.getMetadata().getEventType().equals("LOG"), "eventType must be of type Log"); + checkArgument(this.getEventMetadata().getEventType().equals("LOG"), "eventType must be of type Log"); } /** diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonOtelLog.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonOtelLog.java index bb0c02be6d..a00fb27d28 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonOtelLog.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/log/JacksonOtelLog.java @@ -40,7 +40,7 @@ public class JacksonOtelLog extends JacksonEvent implements OpenTelemetryLog { protected JacksonOtelLog(final JacksonOtelLog.Builder builder) { super(builder); - checkArgument(this.getMetadata().getEventType().equals("LOG"), "eventType must be of type Log"); + checkArgument(this.getEventMetadata().getEventType().equals("LOG"), "eventType must be of type Log"); } @Override diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonExponentialHistogram.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonExponentialHistogram.java index b865ce0eb5..a80ce46420 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonExponentialHistogram.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonExponentialHistogram.java @@ -42,7 +42,7 @@ public class JacksonExponentialHistogram extends JacksonMetric implements Expone protected JacksonExponentialHistogram(JacksonExponentialHistogram.Builder builder, boolean flattenAttributes) { super(builder, flattenAttributes); - checkArgument(this.getMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); + checkArgument(this.getEventMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); } public static JacksonExponentialHistogram.Builder builder() { diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonGauge.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonGauge.java index 4df5bf4793..1745c1507f 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonGauge.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonGauge.java @@ -30,7 +30,7 @@ public class JacksonGauge extends JacksonMetric implements Gauge { protected JacksonGauge(Builder builder, boolean flattenAttributes) { super(builder, flattenAttributes); - checkArgument(this.getMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); + checkArgument(this.getEventMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); } public static Builder builder() { diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonHistogram.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonHistogram.java index f9e066875d..d5273bc1dd 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonHistogram.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonHistogram.java @@ -41,7 +41,7 @@ public class JacksonHistogram extends JacksonMetric implements Histogram { protected JacksonHistogram(JacksonHistogram.Builder builder, boolean flattenAttributes) { super(builder, flattenAttributes); - checkArgument(this.getMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); + checkArgument(this.getEventMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); } public static JacksonHistogram.Builder builder() { return new JacksonHistogram.Builder(); diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSum.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSum.java index 9835650dd0..cc4f17e593 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSum.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSum.java @@ -32,7 +32,7 @@ public class JacksonSum extends JacksonMetric implements Sum { protected JacksonSum(JacksonSum.Builder builder, boolean flattenAttributes) { super(builder, flattenAttributes); - checkArgument(this.getMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); + checkArgument(this.getEventMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); } public static JacksonSum.Builder builder() { diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSummary.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSummary.java index 2196e59075..47c6cc8adc 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSummary.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/metric/JacksonSummary.java @@ -34,7 +34,7 @@ public class JacksonSummary extends JacksonMetric implements Summary { protected JacksonSummary(JacksonSummary.Builder builder, boolean flattenAttributes) { super(builder, flattenAttributes); - checkArgument(this.getMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); + checkArgument(this.getEventMetadata().getEventType().equals(EventType.METRIC.toString()), "eventType must be of type Metric"); } public static JacksonSummary.Builder builder() { diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java index d6bb211ca2..e039c31a91 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/trace/JacksonSpan.java @@ -65,7 +65,7 @@ public class JacksonSpan extends JacksonEvent implements Span { protected JacksonSpan(final Builder builder) { super(builder); - checkArgument(this.getMetadata().getEventType().equals("TRACE"), "eventType must be of type Trace"); + checkArgument(this.getEventMetadata().getEventType().equals("TRACE"), "eventType must be of type Trace"); } private JacksonSpan(final JacksonSpan otherSpan) { @@ -182,7 +182,7 @@ public static JacksonSpan fromSpan(final Span span) { } else { return JacksonSpan.builder() .withData(span.toMap()) - .withEventMetadata(span.getMetadata()) + .withEventMetadata(span.getEventMetadata()) .build(); } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/document/JacksonDocumentTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/document/JacksonDocumentTest.java index 2c7ea5b061..b6a997201a 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/document/JacksonDocumentTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/document/JacksonDocumentTest.java @@ -24,7 +24,7 @@ public void testBuilderUsesDocumentEventType() { final Document document = JacksonDocument.builder().build(); assertThat(document, is(notNullValue())); - assertThat(document.getMetadata().getEventType(), is(equalTo("DOCUMENT"))); + assertThat(document.getEventMetadata().getEventType(), is(equalTo("DOCUMENT"))); } @Test @@ -35,7 +35,7 @@ public void testBuilderUsesCustomEventType() { .build(); assertThat(document, is(notNullValue())); - assertThat(document.getMetadata().getEventType(), is(equalTo("DOCUMENT"))); + assertThat(document.getEventMetadata().getEventType(), is(equalTo("DOCUMENT"))); } @Test diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index 99ffd71259..428b353695 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -416,7 +416,7 @@ public void testBuild_withEventType() { .withEventType(eventType) .build(); - assertThat(event.getMetadata().getEventType(), is(equalTo(eventType))); + assertThat(event.getEventMetadata().getEventType(), is(equalTo(eventType))); assertThat(event.getEventHandle(), is(notNullValue())); assertThat(event.getEventHandle().getInternalOriginationTime(), is(notNullValue())); } @@ -431,7 +431,7 @@ public void testBuild_withTimeReceived() { .withTimeReceived(now) .build(); - assertThat(event.getMetadata().getTimeReceived(), is(equalTo(now))); + assertThat(event.getEventMetadata().getTimeReceived(), is(equalTo(now))); assertThat(event.getEventHandle(), is(notNullValue())); assertThat(event.getEventHandle().getInternalOriginationTime(), is(equalTo(now))); } @@ -461,7 +461,7 @@ public void testBuild_withAttributes() { .withEventMetadataAttributes(testAttributes) .build(); - assertThat(event.getMetadata().getAttributes(), is(equalTo(testAttributes))); + assertThat(event.getEventMetadata().getAttributes(), is(equalTo(testAttributes))); } @Test @@ -484,9 +484,9 @@ public void testBuild_withAllMetadataFields() { .withEventMetadata(metadata) .build(); - assertThat(event.getMetadata().getAttributes(), is(not(equalTo(testAttributes)))); - assertThat(event.getMetadata().getTimeReceived(), is(not(equalTo(now)))); - assertThat(event.getMetadata().getEventType(), is(equalTo(emEventType))); + assertThat(event.getEventMetadata().getAttributes(), is(not(equalTo(testAttributes)))); + assertThat(event.getEventMetadata().getTimeReceived(), is(not(equalTo(now)))); + assertThat(event.getEventMetadata().getEventType(), is(equalTo(emEventType))); } @Test @@ -500,7 +500,7 @@ public void testBuild_withEventMetadata() { .withEventMetadata(metadata) .build(); - assertThat(event.getMetadata(), is(equalTo(metadata))); + assertThat(event.getEventMetadata(), is(equalTo(metadata))); } @@ -690,9 +690,9 @@ void fromEvent_with_a_JacksonEvent() { assertThat(createdEvent.toMap(), equalTo(dataObject)); assertThat(createdEvent.getJsonNode(), not(sameInstance(originalEvent.getJsonNode()))); - assertThat(createdEvent.getMetadata(), notNullValue()); - assertThat(createdEvent.getMetadata(), not(sameInstance(originalEvent.getMetadata()))); - assertThat(createdEvent.getMetadata(), equalTo(originalEvent.getMetadata())); + assertThat(createdEvent.getEventMetadata(), notNullValue()); + assertThat(createdEvent.getEventMetadata(), not(sameInstance(originalEvent.getEventMetadata()))); + assertThat(createdEvent.getEventMetadata(), equalTo(originalEvent.getEventMetadata())); } @Test @@ -702,7 +702,7 @@ void fromEvent_with_a_non_JacksonEvent() { final EventMetadata eventMetadata = mock(EventMetadata.class); final Event originalEvent = mock(Event.class); when(originalEvent.toMap()).thenReturn(dataObject); - when(originalEvent.getMetadata()).thenReturn(eventMetadata); + when(originalEvent.getEventMetadata()).thenReturn(eventMetadata); final JacksonEvent createdEvent = JacksonEvent.fromEvent(originalEvent); @@ -711,8 +711,8 @@ void fromEvent_with_a_non_JacksonEvent() { assertThat(createdEvent.toMap(), equalTo(dataObject)); - assertThat(createdEvent.getMetadata(), notNullValue()); - assertThat(createdEvent.getMetadata(), equalTo(eventMetadata)); + assertThat(createdEvent.getEventMetadata(), notNullValue()); + assertThat(createdEvent.getEventMetadata(), equalTo(eventMetadata)); } @Test @@ -723,7 +723,7 @@ void testJsonStringBuilder() { .withEventType(eventType) .withData(jsonString) .build(); - final EventMetadata eventMetadata = event.getMetadata(); + final EventMetadata eventMetadata = event.getEventMetadata(); eventMetadata.addTags(List.of("tag1", "tag2")); final String expectedJsonString = "{\"foo\":\"bar\",\"tags\":[\"tag1\",\"tag2\"]}"; assertThat(event.jsonBuilder().includeTags("tags").toJsonString(), equalTo(expectedJsonString)); diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEvent_JavaSerializationTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEvent_JavaSerializationTest.java index b3ee46b55c..e6890554b2 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEvent_JavaSerializationTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEvent_JavaSerializationTest.java @@ -51,11 +51,11 @@ void serialize_without_acknowledgementSet_includes_data() throws IOException, Cl final JacksonEvent deserializedEvent = (JacksonEvent) deserializedObject; assertThat(deserializedEvent.toMap(), equalTo(objectUnderTest.toMap())); - assertThat(deserializedEvent.getMetadata(), equalTo(objectUnderTest.getMetadata())); + assertThat(deserializedEvent.getEventMetadata(), equalTo(objectUnderTest.getEventMetadata())); assertThat(deserializedEvent.getEventHandle(), instanceOf(InternalEventHandle.class)); assertThat(((InternalEventHandle) deserializedEvent.getEventHandle()).getAcknowledgementSet(), nullValue()); - assertThat(deserializedEvent.getEventHandle().getInternalOriginationTime(), equalTo(objectUnderTest.getMetadata().getTimeReceived())); + assertThat(deserializedEvent.getEventHandle().getInternalOriginationTime(), equalTo(objectUnderTest.getEventMetadata().getTimeReceived())); } @@ -71,11 +71,11 @@ void serialize_with_acknowledgementSet_does_not_include_old_acknowledgement_set( final JacksonEvent deserializedEvent = (JacksonEvent) deserializedObject; assertThat(deserializedEvent.toMap(), equalTo(objectUnderTest.toMap())); - assertThat(deserializedEvent.getMetadata(), equalTo(objectUnderTest.getMetadata())); + assertThat(deserializedEvent.getEventMetadata(), equalTo(objectUnderTest.getEventMetadata())); assertThat(deserializedEvent.getEventHandle(), instanceOf(InternalEventHandle.class)); assertThat(((InternalEventHandle) deserializedEvent.getEventHandle()).getAcknowledgementSet(), nullValue()); - assertThat(deserializedEvent.getEventHandle().getInternalOriginationTime(), equalTo(objectUnderTest.getMetadata().getTimeReceived())); + assertThat(deserializedEvent.getEventHandle().getInternalOriginationTime(), equalTo(objectUnderTest.getEventMetadata().getTimeReceived())); } private Object serializeAndDeserialize(final JacksonEvent objectUnderTest) throws IOException, ClassNotFoundException { diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/JacksonLogTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/JacksonLogTest.java index da930db7a3..4c4fe55910 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/JacksonLogTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/log/JacksonLogTest.java @@ -23,7 +23,7 @@ public void testBuilder_usesLogEventType() { final Log log = JacksonLog.builder().build(); assertThat(log, is(notNullValue())); - assertThat(log.getMetadata().getEventType(), is(equalTo("LOG"))); + assertThat(log.getEventMetadata().getEventType(), is(equalTo("LOG"))); } @Test @@ -34,7 +34,7 @@ public void testBuilder_usesLogEventType_withUserProvidedEventType() { .build(); assertThat(log, is(notNullValue())); - assertThat(log.getMetadata().getEventType(), is(equalTo("LOG"))); + assertThat(log.getEventMetadata().getEventType(), is(equalTo("LOG"))); } @Test diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java index 2648b172aa..90800f176d 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/trace/JacksonSpanTest.java @@ -609,8 +609,8 @@ void testBuilder_withEventMetadata_with_event_metadata_with_valid_metadata() { .build(); assertThat(jacksonSpan, is(notNullValue())); - assertThat(jacksonSpan.getMetadata(), is(notNullValue())); - assertThat(jacksonSpan.getMetadata().getTimeReceived(), equalTo(now)); + assertThat(jacksonSpan.getEventMetadata(), is(notNullValue())); + assertThat(jacksonSpan.getEventMetadata().getTimeReceived(), equalTo(now)); } @Test @@ -677,8 +677,8 @@ void testBuilder_withData_with_event_valid_data() { .build(); assertThat(jacksonSpan, is(notNullValue())); - assertThat(jacksonSpan.getMetadata(), is(notNullValue())); - assertThat(jacksonSpan.getMetadata().getTimeReceived(), equalTo(now)); + assertThat(jacksonSpan.getEventMetadata(), is(notNullValue())); + assertThat(jacksonSpan.getEventMetadata().getTimeReceived(), equalTo(now)); assertThat(jacksonSpan.toMap().get("traceId"), equalTo(traceId)); assertThat(jacksonSpan.toMap().get("kind"), equalTo(kind)); assertThat(jacksonSpan.toMap().get("traceGroup"), equalTo(traceGroup)); @@ -699,9 +699,9 @@ void fromSpan_with_a_Jackson_Span() { assertThat(createdEvent.toMap(), equalTo(jacksonSpan.toMap())); - assertThat(createdEvent.getMetadata(), notNullValue()); - assertThat(createdEvent.getMetadata(), not(sameInstance(jacksonSpan.getMetadata()))); - assertThat(createdEvent.getMetadata(), equalTo(jacksonSpan.getMetadata())); + assertThat(createdEvent.getEventMetadata(), notNullValue()); + assertThat(createdEvent.getEventMetadata(), not(sameInstance(jacksonSpan.getEventMetadata()))); + assertThat(createdEvent.getEventMetadata(), equalTo(jacksonSpan.getEventMetadata())); } @Test @@ -709,7 +709,7 @@ void fromSpan_with_a_non_JacksonSpan() { final EventMetadata eventMetadata = mock(EventMetadata.class); final Span originalSpan = mock(Span.class); when(originalSpan.toMap()).thenReturn(jacksonSpan.toMap()); - when(originalSpan.getMetadata()).thenReturn(eventMetadata); + when(originalSpan.getEventMetadata()).thenReturn(eventMetadata); when(eventMetadata.getEventType()).thenReturn("TRACE"); final JacksonSpan createdEvent = JacksonSpan.fromSpan(originalSpan); @@ -718,8 +718,8 @@ void fromSpan_with_a_non_JacksonSpan() { assertThat(createdEvent.toMap(), equalTo(jacksonSpan.toMap())); - assertThat(createdEvent.getMetadata(), notNullValue()); - assertThat(createdEvent.getMetadata(), not(sameInstance(jacksonSpan.getMetadata()))); - assertThat(createdEvent.getMetadata().getEventType(), equalTo("TRACE")); + assertThat(createdEvent.getEventMetadata(), notNullValue()); + assertThat(createdEvent.getEventMetadata(), not(sameInstance(jacksonSpan.getEventMetadata()))); + assertThat(createdEvent.getEventMetadata().getEventType(), equalTo("TRACE")); } } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/JacksonPeerForwarderCodec.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/JacksonPeerForwarderCodec.java index 4fac8a7c2d..db2e833017 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/JacksonPeerForwarderCodec.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/peerforwarder/codec/JacksonPeerForwarderCodec.java @@ -43,9 +43,9 @@ private WireEvents fromPeerForwardingEventsToWireEvents(final PeerForwardingEven List wireEventList = null; if (peerForwardingEvents.getEvents() != null) { wireEventList = peerForwardingEvents.getEvents().stream().map(event -> new WireEvent( - event.getMetadata().getEventType(), - event.getMetadata().getTimeReceived(), - event.getMetadata().getAttributes(), + event.getEventMetadata().getEventType(), + event.getEventMetadata().getTimeReceived(), + event.getEventMetadata().getAttributes(), event.toJsonString() )).collect(Collectors.toList()); } diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/pipeline/router/RouterCopyRecordStrategy.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/pipeline/router/RouterCopyRecordStrategy.java index 1bd2944c2e..5ad79256d7 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/pipeline/router/RouterCopyRecordStrategy.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/pipeline/router/RouterCopyRecordStrategy.java @@ -11,7 +11,6 @@ import org.opensearch.dataprepper.model.trace.Span; import org.opensearch.dataprepper.model.trace.JacksonSpan; -import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.event.EventFactory; @@ -98,7 +97,7 @@ public Record getRecord(final Record record) { Record newRecord; DefaultEventHandle eventHandle = (DefaultEventHandle)recordEvent.getEventHandle(); if (eventHandle != null && eventHandle.getAcknowledgementSet() != null) { - final EventMetadata eventMetadata = recordEvent.getMetadata(); + final EventMetadata eventMetadata = recordEvent.getEventMetadata(); final EventBuilder eventBuilder = (EventBuilder) eventFactory.eventBuilder(EventBuilder.class).withEventMetadata(eventMetadata).withData(recordEvent.toMap()); newRecordEvent = (JacksonEvent) eventBuilder.build(); diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultEventBuilderFactoryTests.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultEventBuilderFactoryTests.java index cbcebfb01f..a8685cfa19 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultEventBuilderFactoryTests.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultEventBuilderFactoryTests.java @@ -47,7 +47,7 @@ public void testBasic() { EventBuilder eventBuilder = (EventBuilder)baseEventBuilder.withEventMetadataAttributes(attributes).withData(data); JacksonEvent event = (JacksonEvent) eventBuilder.build(); - EventMetadata eventMetadata = event.getMetadata(); + EventMetadata eventMetadata = event.getEventMetadata(); assertThat(eventMetadata.getTimeReceived(), not(equalTo(null))); assertThat(eventMetadata.getEventType(), equalTo(DefaultEventBuilderFactory.EVENT_TYPE)); assertThat(eventMetadata.getAttributes(), equalTo(attributes)); diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultLogEventBuilderFactoryTests.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultLogEventBuilderFactoryTests.java index cfe719891b..950d3c9cc5 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultLogEventBuilderFactoryTests.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/event/DefaultLogEventBuilderFactoryTests.java @@ -47,7 +47,7 @@ public void testBasic() { LogEventBuilder eventBuilder = (LogEventBuilder)baseEventBuilder.withEventMetadataAttributes(attributes).withData(data); JacksonLog log = (JacksonLog) eventBuilder.build(); - EventMetadata eventMetadata = log.getMetadata(); + EventMetadata eventMetadata = log.getEventMetadata(); assertThat(eventMetadata.getTimeReceived(), not(equalTo(null))); assertThat(eventMetadata.getEventType(), equalTo(DefaultLogEventBuilderFactory.LOG_EVENT_TYPE)); assertThat(eventMetadata.getAttributes(), equalTo(attributes)); diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GetMetadataExpressionFunction.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GetMetadataExpressionFunction.java index d054dda4c6..63e4f2c80d 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GetMetadataExpressionFunction.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GetMetadataExpressionFunction.java @@ -35,7 +35,7 @@ public Object evaluate(final List args, Event event, Function args, Event event, Function validExpressionArguments() { String testTag3 = RandomStringUtils.randomAlphabetic(6); String testTag4 = RandomStringUtils.randomAlphabetic(7); - longEvent.getMetadata().addTags(List.of(testTag1, testTag2, testTag3)); + longEvent.getEventMetadata().addTags(List.of(testTag1, testTag2, testTag3)); Random random = new Random(); int testStringLength = random.nextInt(10); @@ -230,7 +230,7 @@ private static Stream invalidExpressionArguments() { .build(); String testTag1 = RandomStringUtils.randomAlphabetic(6); String testTag2 = RandomStringUtils.randomAlphabetic(7); - tagEvent.getMetadata().addTags(List.of(testTag1, testTag2)); + tagEvent.getEventMetadata().addTags(List.of(testTag1, testTag2)); String testMetadataKey = RandomStringUtils.randomAlphabetic(5); int testStringLength = random.nextInt(10); diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/HasTagsExpressionFunctionTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/HasTagsExpressionFunctionTest.java index 7b1e5fc83c..873e2fa030 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/HasTagsExpressionFunctionTest.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/HasTagsExpressionFunctionTest.java @@ -44,7 +44,7 @@ public void setUp() { public void generateTags(Event event, int numTags) { for (int i = 0; i < numTags; i++) { String tag = RandomStringUtils.randomAlphabetic(5); - testEvent.getMetadata().addTags(List.of(tag)); + testEvent.getEventMetadata().addTags(List.of(tag)); tags.add("\""+tag+"\""); } } diff --git a/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/AppendAggregateActionTest.java b/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/AppendAggregateActionTest.java index fc34d486e5..e8f9978a0a 100644 --- a/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/AppendAggregateActionTest.java +++ b/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/AppendAggregateActionTest.java @@ -292,7 +292,7 @@ void concludeGroup_should_return_groupState_As_An_Event_correctly() throws NoSuc final AggregateActionOutput actionOutput = appendAggregateAction.concludeGroup(aggregateActionInput); final List result = actionOutput.getEvents(); assertThat(result.size(), equalTo(1)); - assertThat(result.get(0).getMetadata().getEventType(), equalTo(AppendAggregateAction.EVENT_TYPE)); + assertThat(result.get(0).getEventMetadata().getEventType(), equalTo(AppendAggregateAction.EVENT_TYPE)); assertThat(result.get(0).toMap(), equalTo(groupState)); } } diff --git a/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/PutAllAggregateActionTest.java b/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/PutAllAggregateActionTest.java index 580feae8f1..4951265898 100644 --- a/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/PutAllAggregateActionTest.java +++ b/data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/actions/PutAllAggregateActionTest.java @@ -99,7 +99,7 @@ void concludeGroup_should_return_groupState_As_An_Event_correctly() { final AggregateActionOutput actionOutput = combineAggregateAction.concludeGroup(aggregateActionInput); final List result = actionOutput.getEvents(); assertThat(result.size(), equalTo(1)); - assertThat(result.get(0).getMetadata().getEventType(), equalTo(PutAllAggregateAction.EVENT_TYPE)); + assertThat(result.get(0).getEventMetadata().getEventType(), equalTo(PutAllAggregateAction.EVENT_TYPE)); assertThat(result.get(0).toMap(), equalTo(groupState)); } } diff --git a/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/avro/AvroInputCodecTest.java b/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/avro/AvroInputCodecTest.java index 5075ef3f62..4ac735b5a8 100644 --- a/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/avro/AvroInputCodecTest.java +++ b/data-prepper-plugins/avro-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/avro/AvroInputCodecTest.java @@ -176,8 +176,8 @@ private static void assertRecordsCorrect(final List> records) { assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(),notNullValue()); - assertThat(actualRecord.getData().getMetadata(),notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(),notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); Map expectedMap=new HashMap<>(); GenericRecord record=generateRecords(parseSchema()).get(index); diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java index aa2930e634..cc484b8737 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java @@ -73,7 +73,7 @@ public Collection> execute(final Collection> records try { final Map newData = processEventJson(eventJson); final Event newRecordEvent = JacksonEvent.builder() - .withEventMetadata(recordEvent.getMetadata()) + .withEventMetadata(recordEvent.getEventMetadata()) .withData(newData) .build(); modifiedRecords.add(new Record<>(newRecordEvent)); diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java index 8ddaf3da02..833927e31a 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/processor/StringProcessorTests.java @@ -63,10 +63,10 @@ public void testStringPrepperDefault() { final Event firstEvent = modifiedRecordEvents.get(0); final Event secondEvent = modifiedRecordEvents.get(1); assertTrue(firstEvent.containsKey(TEST_KEY)); - assertThat(firstEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(firstEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(firstEvent.get(TEST_KEY, String.class), equalTo(UPPERCASE_TEST_STRING.toUpperCase())); assertTrue(secondEvent.containsKey(TEST_KEY)); - assertThat(secondEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(secondEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(secondEvent.get(TEST_KEY, String.class), equalTo(LOWERCASE_TEST_STRING)); } @@ -83,10 +83,10 @@ public void testStringPrepperLowerCase() { final Event firstEvent = modifiedRecordEvents.get(0); final Event secondEvent = modifiedRecordEvents.get(1); assertTrue(firstEvent.containsKey(TEST_KEY)); - assertThat(firstEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(firstEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(firstEvent.get(TEST_KEY, String.class), equalTo(UPPERCASE_TEST_STRING)); assertTrue(secondEvent.containsKey(TEST_KEY)); - assertThat(secondEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(secondEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(secondEvent.get(TEST_KEY, String.class), equalTo(LOWERCASE_TEST_STRING.toLowerCase())); } @@ -103,10 +103,10 @@ public void testStringPrepperUpperCase() { final Event firstEvent = modifiedRecordEvents.get(0); final Event secondEvent = modifiedRecordEvents.get(1); assertTrue(firstEvent.containsKey(TEST_KEY)); - assertThat(firstEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(firstEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(firstEvent.get(TEST_KEY, String.class), equalTo(UPPERCASE_TEST_STRING.toUpperCase())); assertTrue(secondEvent.containsKey(TEST_KEY)); - assertThat(secondEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); + assertThat(secondEvent.getEventMetadata().getEventType(), equalTo(TEST_EVENT_TYPE)); assertThat(secondEvent.get(TEST_KEY, String.class), equalTo(LOWERCASE_TEST_STRING)); } diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/sink/FileSinkTests.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/sink/FileSinkTests.java index d6b0b71d70..8f497f188b 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/sink/FileSinkTests.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/sink/FileSinkTests.java @@ -64,7 +64,7 @@ void setUp() throws IOException { .withEventType("event") .withData(Map.of(TEST_KEY, TEST_DATA_1)) .build(); - event.getMetadata().addTags(List.of(tagStr1, tagStr2)); + event.getEventMetadata().addTags(List.of(tagStr1, tagStr2)); TEST_RECORD1 = new Record<>(event); TEST_RECORDS.add(TEST_RECORD1); event = JacksonEvent.builder() diff --git a/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/csv/CsvCodecTest.java b/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/csv/CsvCodecTest.java index 8dddd0f9ea..fb56f5cdee 100644 --- a/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/csv/CsvCodecTest.java +++ b/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/csv/CsvCodecTest.java @@ -121,8 +121,8 @@ void test_when_noAutoDetectHeaderOrUserSpecifiedHeader_then_autogeneratesHeaderA final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), autoGeneratedHeader); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -169,8 +169,8 @@ void test_inputFile_autogeneratesHeaderAndParsesCorrectly() throws IOException { final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), autoGeneratedHeader); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -200,8 +200,8 @@ void test_when_autoDetectHeaderHappyCase_then_callsConsumerWithParsedEvents(fina final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -233,8 +233,8 @@ void test_when_manualHeaderHappyCase_then_callsConsumerWithParsedEvents(final in final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -267,8 +267,8 @@ void test_when_manualHeaderTooFewColumns_then_autoGenerateRemainingColumnsOnPars final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), actualHeader); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -301,8 +301,8 @@ void test_when_autoDetectHeaderWithMoreColumnsThenBufferCapacity_then_parsesEnti final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -334,8 +334,8 @@ void test_when_manualHeaderTooManyColumns_then_omitsExtraColumnsOnParsedEvents(f final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), actualHeader); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -376,8 +376,8 @@ void test_when_autoDetectHeaderWrongNumberColumnsAndJaggedRows_then_skipsRemaini final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -421,8 +421,8 @@ void test_when_manualHeaderWrongNumberColumnsAndJaggedRows_then_skipsRemainingRo final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -485,8 +485,8 @@ void test_when_unrecoverableRow_then_logsExceptionAndSkipsRemainingRows(final in final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header, config.getDelimiter().charAt(0), config.getQuoteCharacter().charAt(0)); @@ -532,8 +532,8 @@ void test_when_emptyLineWithCorrectNumberDelimiters_then_parsesAsEmpty() throws final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -569,8 +569,8 @@ void test_when_manualHeaderDifferentDelimiterAndQuoteCharacter_then_parsesCorrec final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMapFromHeaderList(csvRowsExcludingHeader.get(i), header, delimiter.charAt(0), quoteCharacter.charAt(0)); @@ -606,8 +606,8 @@ void test_when_autodetectHeaderDifferentDelimiterAndQuoteCharacter_then_parsesCo final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = createExpectedMap(csvRowsExcludingHeader.get(i), header, delimiter.charAt(0), quoteCharacter.charAt(0)); diff --git a/data-prepper-plugins/date-processor/README.md b/data-prepper-plugins/date-processor/README.md index 3ba1647498..3af809db86 100644 --- a/data-prepper-plugins/date-processor/README.md +++ b/data-prepper-plugins/date-processor/README.md @@ -88,7 +88,7 @@ processor: > creating index template for [Data Streams](https://opensearch.org/docs/latest/opensearch/data-streams/#step-1-create-an-index-template). * `source_timezone` (Optional): Timezone used for parsing dates. It will be used in case of zone or offset cannot be extracted from value. If zone or offset is part of the value timezone will be ignored. - All the zone rules are provided to JVM by default provider defined by + All the zone statelessRules are provided to JVM by default provider defined by IANA Time Zone Database (TZDB). Find all the available timezones [here](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List) in "TZ database name" column. * Type: String diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java index a494cf5334..2b9c461338 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java @@ -86,7 +86,7 @@ public Collection> doExecute(Collection> records) { Instant timeStamp = result.getRight(); if (dateProcessorConfig.getToOriginationMetadata()) { Event event = (Event)record.getData(); - event.getMetadata().setExternalOriginationTime(timeStamp); + event.getEventMetadata().setExternalOriginationTime(timeStamp); event.getEventHandle().setExternalOriginationTime(timeStamp); } } @@ -139,7 +139,7 @@ private DateTimeFormatter getSourceFormatter(final String pattern) { } private String getDateTimeFromTimeReceived(final Record record) { - final Instant timeReceived = record.getData().getMetadata().getTimeReceived(); + final Instant timeReceived = record.getData().getEventMetadata().getTimeReceived(); return timeReceived.atZone(dateProcessorConfig.getDestinationZoneId()).format(getOutputFormatter()); } diff --git a/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java b/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java index c6688d08e3..bb2f454a83 100644 --- a/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java +++ b/data-prepper-plugins/date-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorTests.java @@ -489,10 +489,10 @@ void match_with_to_origination_metadata(String pattern) { final List> processedRecords = (List>) dateProcessor.doExecute(Collections.singletonList(record)); Event event = (Event)processedRecords.get(0).getData(); - Assertions.assertTrue(event.getMetadata().getExternalOriginationTime() != null); + Assertions.assertTrue(event.getEventMetadata().getExternalOriginationTime() != null); Assertions.assertTrue(event.getEventHandle().getExternalOriginationTime() != null); ZonedDateTime expectedZonedDatetime = expectedDateTime.atZone(mockDateProcessorConfig.getSourceZoneId()).truncatedTo(ChronoUnit.SECONDS); - Assertions.assertTrue(expectedZonedDatetime.equals(event.getMetadata().getExternalOriginationTime().atZone(mockDateProcessorConfig.getSourceZoneId()))); + Assertions.assertTrue(expectedZonedDatetime.equals(event.getEventMetadata().getExternalOriginationTime().atZone(mockDateProcessorConfig.getSourceZoneId()))); verify(dateProcessingMatchSuccessCounter, times(1)).increment(); } diff --git a/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessor.java b/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessor.java index 61e7b7e812..32133e3300 100644 --- a/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessor.java +++ b/data-prepper-plugins/decompress-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessor.java @@ -79,17 +79,17 @@ public Collection> doExecute(final Collection> recor } catch (final Exception e) { LOG.error("Unable to decompress key {} using decompression type {}:", key, decompressProcessorConfig.getDecompressionType(), e); - record.getData().getMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); + record.getData().getEventMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); decompressionProcessingErrors.increment(); } } } catch (final DecodingException e) { LOG.error("Unable to decode key with base64: {}", e.getMessage()); - record.getData().getMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); + record.getData().getEventMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); decompressionProcessingErrors.increment(); } catch (final Exception e) { LOG.error("An uncaught exception occurred while decompressing Events", e); - record.getData().getMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); + record.getData().getEventMetadata().addTags(decompressProcessorConfig.getTagsOnFailure()); decompressionProcessingErrors.increment(); } } diff --git a/data-prepper-plugins/decompress-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorTest.java b/data-prepper-plugins/decompress-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorTest.java index 8ad5cfc657..1159c37baa 100644 --- a/data-prepper-plugins/decompress-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorTest.java +++ b/data-prepper-plugins/decompress-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorTest.java @@ -133,9 +133,9 @@ void decompression_with_decoding_error_adds_tags_and_increments_error_metric() { assertThat(result.get(0), notNullValue()); assertThat(result.get(0).getData(), notNullValue()); assertThat(result.get(0).getData().get(key, String.class), equalTo(compressedValue)); - assertThat(result.get(0).getData().getMetadata().getTags(), notNullValue()); - assertThat(result.get(0).getData().getMetadata().getTags().size(), equalTo(1)); - assertThat(result.get(0).getData().getMetadata().getTags().contains(tagForFailure), equalTo(true)); + assertThat(result.get(0).getData().getEventMetadata().getTags(), notNullValue()); + assertThat(result.get(0).getData().getEventMetadata().getTags().size(), equalTo(1)); + assertThat(result.get(0).getData().getEventMetadata().getTags().contains(tagForFailure), equalTo(true)); verifyNoInteractions(decompressionEngine); verify(decompressionProcessingErrors).increment(); @@ -167,9 +167,9 @@ void exception_from_DecompressionEngine_adds_tags_and_increments_error_metric() assertThat(result.get(0), notNullValue()); assertThat(result.get(0).getData(), notNullValue()); assertThat(result.get(0).getData().get(key, String.class), equalTo(compressedValue)); - assertThat(result.get(0).getData().getMetadata().getTags(), notNullValue()); - assertThat(result.get(0).getData().getMetadata().getTags().size(), equalTo(1)); - assertThat(result.get(0).getData().getMetadata().getTags().contains(tagForFailure), equalTo(true)); + assertThat(result.get(0).getData().getEventMetadata().getTags(), notNullValue()); + assertThat(result.get(0).getData().getEventMetadata().getTags().size(), equalTo(1)); + assertThat(result.get(0).getData().getEventMetadata().getTags().contains(tagForFailure), equalTo(true)); verify(decompressionProcessingErrors).increment(); } @@ -196,9 +196,9 @@ void exception_from_expression_evaluator_adds_tags_and_increments_error_metric() assertThat(result.get(0), notNullValue()); assertThat(result.get(0).getData(), notNullValue()); assertThat(result.get(0).getData().get(key, String.class), equalTo(compressedValue)); - assertThat(result.get(0).getData().getMetadata().getTags(), notNullValue()); - assertThat(result.get(0).getData().getMetadata().getTags().size(), equalTo(1)); - assertThat(result.get(0).getData().getMetadata().getTags().contains(tagForFailure), equalTo(true)); + assertThat(result.get(0).getData().getEventMetadata().getTags(), notNullValue()); + assertThat(result.get(0).getData().getEventMetadata().getTags().size(), equalTo(1)); + assertThat(result.get(0).getData().getEventMetadata().getTags().contains(tagForFailure), equalTo(true)); verifyNoInteractions(decoderEngine, decompressionEngine); verify(decompressionProcessingErrors).increment(); diff --git a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/RecordConverter.java b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/RecordConverter.java index ce57f118c4..ddae92817f 100644 --- a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/RecordConverter.java +++ b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/RecordConverter.java @@ -93,9 +93,9 @@ public void addToBuffer(final AcknowledgementSet acknowledgementSet, if (eventName != null) { final Instant externalOriginationTime = Instant.ofEpochMilli(eventCreationTimeMillis); event.getEventHandle().setExternalOriginationTime(externalOriginationTime); - event.getMetadata().setExternalOriginationTime(externalOriginationTime); + event.getEventMetadata().setExternalOriginationTime(externalOriginationTime); } - EventMetadata eventMetadata = event.getMetadata(); + EventMetadata eventMetadata = event.getEventMetadata(); eventMetadata.setAttribute(EVENT_TABLE_NAME_METADATA_ATTRIBUTE, tableInfo.getTableName()); eventMetadata.setAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE, eventCreationTimeMillis); diff --git a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/ExportRecordConverterTest.java b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/ExportRecordConverterTest.java index e1808874a6..1fab00253b 100644 --- a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/ExportRecordConverterTest.java +++ b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/ExportRecordConverterTest.java @@ -156,17 +156,17 @@ void test_writeSingleRecordToBuffer() throws Exception { assertThat(recordArgumentCaptor.getValue().getData(), notNullValue()); JacksonEvent event = (JacksonEvent) recordArgumentCaptor.getValue().getData(); - assertThat(event.getMetadata(), notNullValue()); - - assertThat(event.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(pk)); - assertThat(event.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sk)); - assertThat(event.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(pk + "|" + sk)); - assertThat(event.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(event.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), notNullValue()); - assertThat(event.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), nullValue()); - assertThat(event.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), notNullValue()); + assertThat(event.getEventMetadata(), notNullValue()); + + assertThat(event.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(pk)); + assertThat(event.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sk)); + assertThat(event.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(pk + "|" + sk)); + assertThat(event.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(event.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), notNullValue()); + assertThat(event.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), nullValue()); + assertThat(event.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), notNullValue()); final long expectedVersion = (1700020441123L - VERSION_OVERLAP_TIME_FOR_EXPORT.toMillis()) * 1_000; - assertThat(event.getMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(expectedVersion)); + assertThat(event.getEventMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(expectedVersion)); assertThat(event.getEventHandle(), notNullValue()); assertThat(event.getEventHandle().getExternalOriginationTime(), nullValue()); verify(bytesReceivedSummary, times(1)).record(line.getBytes().length); @@ -194,11 +194,11 @@ void writing_record_to_buffer_with_ion_formatted_decimals_creates_expected_parti assertThat(recordArgumentCaptor.getValue().getData(), notNullValue()); JacksonEvent event = (JacksonEvent) recordArgumentCaptor.getValue().getData(); - assertThat(event.getMetadata(), notNullValue()); + assertThat(event.getEventMetadata(), notNullValue()); - assertThat(event.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(expectedPartitionKey)); - assertThat(event.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(expectedSortKey)); - assertThat(event.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(expectedPartitionKey + "|" + expectedSortKey)); + assertThat(event.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(expectedPartitionKey)); + assertThat(event.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(expectedSortKey)); + assertThat(event.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(expectedPartitionKey + "|" + expectedSortKey)); } private static Stream decimalFormatKeysArgumentProvider() { diff --git a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/StreamRecordConverterTest.java b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/StreamRecordConverterTest.java index 19b4e27b49..d218279ab5 100644 --- a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/StreamRecordConverterTest.java +++ b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/converter/StreamRecordConverterTest.java @@ -150,15 +150,15 @@ void test_writeSingleRecordToBuffer() throws Exception { assertThat(recordArgumentCaptor.getValue().getData(), notNullValue()); JacksonEvent event = (JacksonEvent) recordArgumentCaptor.getValue().getData(); - assertThat(event.getMetadata(), notNullValue()); + assertThat(event.getEventMetadata(), notNullValue()); String partitionKey = record.dynamodb().keys().get(partitionKeyAttrName).s(); String sortKey = record.dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(event.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); - assertThat(event.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); - assertThat(event.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); - assertThat(event.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(event.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(event.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(record.dynamodb().approximateCreationDateTime().toEpochMilli())); + assertThat(event.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); + assertThat(event.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); + assertThat(event.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); + assertThat(event.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(event.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(event.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(record.dynamodb().approximateCreationDateTime().toEpochMilli())); assertThat(event.get(partitionKeyAttrName, String.class), notNullValue()); assertThat(event.get(sortKeyAttrName, String.class), notNullValue()); @@ -196,15 +196,15 @@ void test_writeSingleRecordToBuffer_with_other_data(final String additionalStrin assertThat(recordArgumentCaptor.getValue().getData(), notNullValue()); JacksonEvent event = (JacksonEvent) recordArgumentCaptor.getValue().getData(); - assertThat(event.getMetadata(), notNullValue()); + assertThat(event.getEventMetadata(), notNullValue()); String partitionKey = record.dynamodb().keys().get(partitionKeyAttrName).s(); String sortKey = record.dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(event.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); - assertThat(event.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); - assertThat(event.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); - assertThat(event.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(event.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(event.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(record.dynamodb().approximateCreationDateTime().toEpochMilli())); + assertThat(event.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); + assertThat(event.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); + assertThat(event.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); + assertThat(event.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(event.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(event.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(record.dynamodb().approximateCreationDateTime().toEpochMilli())); assertThat(event.get(partitionKeyAttrName, String.class), notNullValue()); assertThat(event.get(sortKeyAttrName, String.class), notNullValue()); @@ -274,61 +274,61 @@ void writingToBuffer_with_nth_event_in_that_second_returns_expected_that_timesta JacksonEvent firstEventForSecond = (JacksonEvent) createdEvents.get(0).getData(); - assertThat(firstEventForSecond.getMetadata(), notNullValue()); + assertThat(firstEventForSecond.getEventMetadata(), notNullValue()); String partitionKey = records.get(0).dynamodb().keys().get(partitionKeyAttrName).s(); String sortKey = records.get(0).dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(firstEventForSecond.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); - assertThat(firstEventForSecond.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); - assertThat(firstEventForSecond.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); - assertThat(firstEventForSecond.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(firstEventForSecond.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(firstEventForSecond.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(timestamp.toEpochMilli())); - assertThat(firstEventForSecond.getMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(timestamp.toEpochMilli() * 1000)); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(partitionKey)); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(sortKey)); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(partitionKey + "|" + sortKey)); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(timestamp.toEpochMilli())); + assertThat(firstEventForSecond.getEventMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(timestamp.toEpochMilli() * 1000)); assertThat(firstEventForSecond.getEventHandle(), notNullValue()); assertThat(firstEventForSecond.getEventHandle().getExternalOriginationTime(), equalTo(timestamp)); JacksonEvent secondEventForSameSecond = (JacksonEvent) createdEvents.get(1).getData(); - assertThat(secondEventForSameSecond.getMetadata(), notNullValue()); + assertThat(secondEventForSameSecond.getEventMetadata(), notNullValue()); String secondPartitionKey = records.get(1).dynamodb().keys().get(partitionKeyAttrName).s(); String secondSortKey = records.get(1).dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(secondPartitionKey)); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(secondSortKey)); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(secondPartitionKey + "|" + secondSortKey)); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(timestamp.toEpochMilli())); - assertThat(secondEventForSameSecond.getMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(timestamp.toEpochMilli() * 1000 + 1)); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(secondPartitionKey)); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(secondSortKey)); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(secondPartitionKey + "|" + secondSortKey)); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(timestamp.toEpochMilli())); + assertThat(secondEventForSameSecond.getEventMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(timestamp.toEpochMilli() * 1000 + 1)); assertThat(secondEventForSameSecond.getEventHandle(), notNullValue()); assertThat(secondEventForSameSecond.getEventHandle().getExternalOriginationTime(), equalTo(timestamp)); JacksonEvent thirdEventWithOlderSecond = (JacksonEvent) createdEvents.get(2).getData(); - assertThat(thirdEventWithOlderSecond.getMetadata(), notNullValue()); + assertThat(thirdEventWithOlderSecond.getEventMetadata(), notNullValue()); String thirdPartitionKey = records.get(2).dynamodb().keys().get(partitionKeyAttrName).s(); String thirdSortKey = records.get(2).dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(thirdPartitionKey)); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(thirdSortKey)); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(thirdPartitionKey + "|" + thirdSortKey)); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(olderSecond.toEpochMilli())); - assertThat(thirdEventWithOlderSecond.getMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(olderSecond.toEpochMilli() * 1000)); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(thirdPartitionKey)); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(thirdSortKey)); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(thirdPartitionKey + "|" + thirdSortKey)); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(olderSecond.toEpochMilli())); + assertThat(thirdEventWithOlderSecond.getEventMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(olderSecond.toEpochMilli() * 1000)); assertThat(thirdEventWithOlderSecond.getEventHandle(), notNullValue()); assertThat(thirdEventWithOlderSecond.getEventHandle().getExternalOriginationTime(), equalTo(olderSecond)); JacksonEvent fourthEventWithNewerSecond = (JacksonEvent) createdEvents.get(3).getData(); - assertThat(fourthEventWithNewerSecond.getMetadata(), notNullValue()); + assertThat(fourthEventWithNewerSecond.getEventMetadata(), notNullValue()); String fourthPartitionKey = records.get(3).dynamodb().keys().get(partitionKeyAttrName).s(); String fourthSortKey = records.get(3).dynamodb().keys().get(sortKeyAttrName).s(); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(fourthPartitionKey)); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(fourthSortKey)); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(fourthPartitionKey + "|" + fourthSortKey)); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(newerSecond.toEpochMilli())); - assertThat(fourthEventWithNewerSecond.getMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(newerSecond.toEpochMilli() * 1000)); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(PARTITION_KEY_METADATA_ATTRIBUTE), equalTo(fourthPartitionKey)); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(SORT_KEY_METADATA_ATTRIBUTE), equalTo(fourthSortKey)); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(PRIMARY_KEY_DOCUMENT_ID_METADATA_ATTRIBUTE), equalTo(fourthPartitionKey + "|" + fourthSortKey)); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(EVENT_NAME_BULK_ACTION_METADATA_ATTRIBUTE), equalTo(OpenSearchBulkActions.INDEX.toString())); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(DDB_STREAM_EVENT_NAME_METADATA_ATTRIBUTE), equalTo("INSERT")); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(EVENT_TIMESTAMP_METADATA_ATTRIBUTE), equalTo(newerSecond.toEpochMilli())); + assertThat(fourthEventWithNewerSecond.getEventMetadata().getAttribute(EVENT_VERSION_FROM_TIMESTAMP), equalTo(newerSecond.toEpochMilli() * 1000)); assertThat(fourthEventWithNewerSecond.getEventHandle(), notNullValue()); assertThat(fourthEventWithNewerSecond.getEventHandle().getExternalOriginationTime(), equalTo(newerSecond)); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java index c81e2caa0f..d856729261 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java @@ -158,7 +158,7 @@ public Collection> doExecute(final Collection> recor if (isEventFailedLookup) { geoIpEventsFailedLookup.increment(); - event.getMetadata().addTags(tagsOnFailure); + event.getEventMetadata().addTags(tagsOnFailure); } } geoIPDatabaseReader.close(); diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java index 1ff06cfc99..cc64b6f192 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java @@ -330,7 +330,7 @@ void test_tags_when_enrich_fails() { for (final Record record : records) { Event event = record.getData(); - assertTrue(event.getMetadata().hasTags(testTags)); + assertTrue(event.getEventMetadata().hasTags(testTags)); verify(geoIpEventsFailedLookup).increment(); verify(geoIpEventsProcessed).increment(); } diff --git a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java index 1c3c9e6d15..11d8dbf7ae 100644 --- a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java @@ -133,11 +133,11 @@ public Collection> doExecute(final Collection> recor } } catch (final TimeoutException e) { - event.getMetadata().addTags(tagsOnTimeout); + event.getEventMetadata().addTags(tagsOnTimeout); LOG.error(EVENT, "Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokProcessorConfig.getTimeoutMillis()); grokProcessingTimeoutsCounter.increment(); } catch (final ExecutionException | InterruptedException | RuntimeException e) { - event.getMetadata().addTags(tagsOnMatchFailure); + event.getEventMetadata().addTags(tagsOnMatchFailure); LOG.error(EVENT, "An exception occurred when matching record [{}]", record.getData(), e); grokProcessingErrorsCounter.increment(); } @@ -256,7 +256,7 @@ private void matchAndMerge(final Event event) { if (grokkedCaptures.isEmpty()) { if (tagsOnMatchFailure != null && tagsOnMatchFailure.size() > 0) { - event.getMetadata().addTags(tagsOnMatchFailure); + event.getEventMetadata().addTags(tagsOnMatchFailure); } grokProcessingMismatchCounter.increment(); } else { diff --git a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java index 1c8d0036c2..3c7e8b444f 100644 --- a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorIT.java @@ -488,8 +488,8 @@ public void testMatchWithNoCapturesAndTags() throws JsonProcessingException { assertThat(grokkedRecords.size(), equalTo(1)); assertRecordsAreEqual(grokkedRecords.get(0), record); - assertTrue(((Event)record.getData()).getMetadata().getTags().contains(tagOnMatchFailure1)); - assertTrue(((Event)record.getData()).getMetadata().getTags().contains(tagOnMatchFailure2)); + assertTrue(((Event)record.getData()).getEventMetadata().getTags().contains(tagOnMatchFailure1)); + assertTrue(((Event)record.getData()).getEventMetadata().getTags().contains(tagOnMatchFailure2)); } @Test diff --git a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java index bb3b6857c7..ca8fdb0727 100644 --- a/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java +++ b/data-prepper-plugins/grok-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorTests.java @@ -548,10 +548,10 @@ public void testNoCapturesWithTag() throws JsonProcessingException { assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); assertRecordsAreEqual(grokkedRecords.get(0), record); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure1)); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure2)); - assertThat(record.getData().getMetadata().getTags(), not(hasItem(tagOnTimeout1))); - assertThat(record.getData().getMetadata().getTags(), not(hasItem(tagOnTimeout2))); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnMatchFailure1)); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnMatchFailure2)); + assertThat(record.getData().getEventMetadata().getTags(), not(hasItem(tagOnTimeout1))); + assertThat(record.getData().getEventMetadata().getTags(), not(hasItem(tagOnTimeout2))); verify(grokProcessingMismatchCounter, times(1)).increment(); verify(grokProcessingTime, times(1)).record(any(Runnable.class)); verifyNoInteractions(grokProcessingErrorsCounter, grokProcessingMatchCounter, grokProcessingTimeoutsCounter); @@ -576,10 +576,10 @@ public void timeout_exception_tags_the_event() throws JsonProcessingException, T assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); assertRecordsAreEqual(grokkedRecords.get(0), record); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnTimeout1)); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnTimeout2)); - assertThat(record.getData().getMetadata().getTags(), not(hasItem(tagOnMatchFailure1))); - assertThat(record.getData().getMetadata().getTags(), not(hasItem(tagOnMatchFailure2))); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnTimeout1)); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnTimeout2)); + assertThat(record.getData().getEventMetadata().getTags(), not(hasItem(tagOnMatchFailure1))); + assertThat(record.getData().getEventMetadata().getTags(), not(hasItem(tagOnMatchFailure2))); verify(grokProcessingTimeoutsCounter, times(1)).increment(); verify(grokProcessingTime, times(1)).record(any(Runnable.class)); verifyNoInteractions(grokProcessingErrorsCounter, grokProcessingMismatchCounter); @@ -605,8 +605,8 @@ public void execution_exception_tags_the_event(Class exceptionClass) assertThat(grokkedRecords.size(), equalTo(1)); assertThat(grokkedRecords.get(0), notNullValue()); assertRecordsAreEqual(grokkedRecords.get(0), record); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure1)); - assertThat(record.getData().getMetadata().getTags(), hasItem(tagOnMatchFailure2)); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnMatchFailure1)); + assertThat(record.getData().getEventMetadata().getTags(), hasItem(tagOnMatchFailure2)); verify(grokProcessingErrorsCounter, times(1)).increment(); verify(grokProcessingTime, times(1)).record(any(Runnable.class)); verifyNoInteractions(grokProcessingTimeoutsCounter, grokProcessingMismatchCounter); diff --git a/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/source/KafkaSourceJsonTypeIT.java b/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/source/KafkaSourceJsonTypeIT.java index 6cd7a5215f..b55508971d 100644 --- a/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/source/KafkaSourceJsonTypeIT.java +++ b/data-prepper-plugins/kafka-plugins/src/integrationTest/java/org/opensearch/dataprepper/plugins/kafka/source/KafkaSourceJsonTypeIT.java @@ -197,7 +197,7 @@ public void TestJsonRecordsWithNullKey() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); @@ -229,7 +229,7 @@ public void TestJsonRecordsWithNegativeAcknowledgements() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); @@ -247,7 +247,7 @@ public void TestJsonRecordsWithNegativeAcknowledgements() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); @@ -278,7 +278,7 @@ public void TestJsonRecordsWithKafkaKeyModeDiscard() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); @@ -308,7 +308,7 @@ public void TestJsonRecordsWithKafkaKeyModeAsField() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); @@ -339,7 +339,7 @@ public void TestJsonRecordsWithKafkaKeyModeAsMetadata() throws Exception { for (int i = 0; i < numRecords; i++) { Record record = receivedRecords.get(i); Event event = (Event) record.getData(); - EventMetadata metadata = event.getMetadata(); + EventMetadata metadata = event.getEventMetadata(); Map map = event.toMap(); assertThat(map.get("name"), equalTo("testName" + i)); assertThat(map.get("id"), equalTo(TEST_ID + i)); diff --git a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java index 0583dbddbe..aaf43c4301 100644 --- a/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java +++ b/data-prepper-plugins/kafka-plugins/src/main/java/org/opensearch/dataprepper/plugins/kafka/consumer/KafkaCustomConsumer.java @@ -421,7 +421,7 @@ private Record getRecord(ConsumerRecord consumerRecord, in data.put(key, value); } event = JacksonLog.builder().withData(data).build(); - EventMetadata eventMetadata = event.getMetadata(); + EventMetadata eventMetadata = event.getEventMetadata(); if (kafkaKeyMode == KafkaKeyMode.INCLUDE_AS_METADATA) { eventMetadata.setAttribute("kafka_key", key); } diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index 0ccfa90baa..817fb0e0d4 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -251,14 +251,14 @@ public Collection> doExecute(final Collection> recor outputMap.putAll(createRecursedMap(recursedTree, mapper)); } catch (Exception e) { LOG.error("Recursive parsing ran into an unexpected error, treating message as non-recursive", e); - recordEvent.getMetadata().addTags(tagsOnFailure); + recordEvent.getEventMetadata().addTags(tagsOnFailure); } } else { try { outputMap.putAll(createNonRecursedMap(groups)); } catch (Exception e) { LOG.error("Non-recursive parsing ran into an unexpected error", e); - recordEvent.getMetadata().addTags(tagsOnFailure); + recordEvent.getEventMetadata().addTags(tagsOnFailure); } } @@ -274,7 +274,7 @@ public Collection> doExecute(final Collection> recor } } catch (final Exception e) { LOG.error(EVENT, "There was an exception while processing on Event [{}]: ", recordEvent, e); - recordEvent.getMetadata().addTags(tagsOnFailure); + recordEvent.getEventMetadata().addTags(tagsOnFailure); } } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 429407877c..8dad4196ca 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -841,7 +841,7 @@ void testTagsAddedWhenParsingFails() { final LinkedHashMap parsed_message = getLinkedHashMap(editedRecords); assertThat(parsed_message.size(), equalTo(0)); - assertThat(record.getData().getMetadata().hasTags(List.of("tag1", "tag2")), is(true)); + assertThat(record.getData().getEventMetadata().hasTags(List.of("tag1", "tag2")), is(true)); } @Test diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java index e3e414f96d..add9257cdb 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java @@ -75,9 +75,9 @@ public Collection> doExecute(final Collection> recor mergeValueToEvent(recordEvent, key, value); } } else { - Map attributes = recordEvent.getMetadata().getAttributes(); + Map attributes = recordEvent.getEventMetadata().getAttributes(); if (!attributes.containsKey(metadataKey) || entry.getOverwriteIfKeyExists()) { - recordEvent.getMetadata().setAttribute(metadataKey, value); + recordEvent.getEventMetadata().setAttribute(metadataKey, value); } else if (attributes.containsKey(metadataKey) && entry.getAppendIfKeyExists()) { mergeValueToEventMetadata(recordEvent, metadataKey, value); } @@ -113,7 +113,7 @@ private void mergeValueToEvent(final Event recordEvent, final String key, final } private void mergeValueToEventMetadata(final Event recordEvent, final String key, final Object value) { - mergeValue(value, () -> recordEvent.getMetadata().getAttribute(key), newValue -> recordEvent.getMetadata().setAttribute(key, newValue)); + mergeValue(value, () -> recordEvent.getEventMetadata().getAttribute(key), newValue -> recordEvent.getEventMetadata().setAttribute(key, newValue)); } private void mergeValue(final Object value, Supplier getter, Consumer setter) { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java index 95cbd9f714..b81f4f210f 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java @@ -70,7 +70,7 @@ public Collection> doExecute(final Collection> recor recordEvent.put(key, converter.convert(keyVal)); } catch (final RuntimeException e) { LOG.error(EVENT, "Unable to convert key: {} with value: {} to {}", key, keyVal, type, e); - recordEvent.getMetadata().addTags(tagsOnFailure); + recordEvent.getEventMetadata().addTags(tagsOnFailure); } } else { recordEvent.delete(key); @@ -79,7 +79,7 @@ public Collection> doExecute(final Collection> recor } } catch (final Exception e) { LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); - recordEvent.getMetadata().addTags(tagsOnFailure); + recordEvent.getEventMetadata().addTags(tagsOnFailure); } } return records; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java index d042f8fa28..cedf21454c 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java @@ -57,7 +57,7 @@ public Collection> doExecute(final Collection> recor } catch (final Exception e) { LOG.warn(EVENT, "Given source path [{}] is not valid on record [{}]", config.getSource(), recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); continue; } @@ -67,11 +67,11 @@ public Collection> doExecute(final Collection> recor } catch (final IllegalArgumentException e) { LOG.warn(EVENT, "Cannot find a list at the given source path [{}} on record [{}]", config.getSource(), recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); continue; } catch (final Exception e) { LOG.error(EVENT, "Error converting source list to map on record [{}]", recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); continue; } @@ -79,11 +79,11 @@ public Collection> doExecute(final Collection> recor updateEvent(recordEvent, targetMap); } catch (final Exception e) { LOG.error(EVENT, "Error updating record [{}] after converting source list to map", recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); } } catch (final Exception e) { LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); } } return records; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java index d911cd6194..2c4853a0a8 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java @@ -85,11 +85,11 @@ public Collection> doExecute(final Collection> recor } } catch (Exception e) { LOG.error("Fail to perform Map to List operation", e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); } } catch (final Exception e) { LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); - recordEvent.getMetadata().addTags(config.getTagsOnFailure()); + recordEvent.getEventMetadata().addTags(config.getTagsOnFailure()); } } return records; diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java index 341bad79de..a3b035477a 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java @@ -407,7 +407,7 @@ public void testMetadataKeySetWithBadFormatThenEntryNotAdded() { final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); Event event = editedRecords.get(0).getData(); - Map attributes = event.getMetadata().getAttributes(); + Map attributes = event.getEventMetadata().getAttributes(); assertThat(event.get("date", Object.class), equalTo("date-value")); assertThat(event.get("time", Object.class), equalTo("time-value")); assertThat(attributes.containsKey("data-time"), equalTo(false)); @@ -444,7 +444,7 @@ public void testMetadataKeyIsNotAdded_when_addWhen_condition_is_false() { Event event = editedRecords.get(0).getData(); assertThat(event.containsKey("message"), is(true)); assertThat(event.get("message", Object.class), equalTo("thisisamessage")); - Map attributes = event.getMetadata().getAttributes(); + Map attributes = event.getEventMetadata().getAttributes(); assertThat(attributes.containsKey("newMessage"), is(false)); } @@ -460,7 +460,7 @@ public void testMetadataKeySetWithDifferentDataTypes() { final Record record = getEventWithMetadata("message", Map.of("key1", "value1")); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("newField"), equalTo("newValue")); assertThat(attributes.get("newIntField"), equalTo(123)); assertThat(attributes.get("newBooleanField"), equalTo(true)); @@ -476,7 +476,7 @@ public void testMetadataKeySetWithFormatNotOverwriteExistingEntry() { final Record record = getEventWithMetadata("message", Map.of("date", "date-value", "time", "time-value")); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("date"), equalTo("date-value")); assertThat(attributes.get("time"), equalTo("time-value")); assertThat(attributes.containsKey("date-time"), equalTo(false)); @@ -492,7 +492,7 @@ public void testMetadataKeySetWithFormatOverwriteExistingEntry() { final Record record = getEventWithMetadata("message", Map.of("date", "date-value", "time", "time-value")); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("date"), equalTo("date-value")); assertThat(attributes.get("time"), equalTo("date-value time-value")); assertThat(attributes.containsKey("date-time"), equalTo(false)); @@ -509,7 +509,7 @@ public void testMetadataKeySetAppendToExistingSimpleValue() { final Record record = getEventWithMetadata("message", Map.of("time", currentValue)); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("time"), equalTo(List.of(currentValue, "time-value2"))); } @@ -526,7 +526,7 @@ public void testMetadataKeySetAppendToExistingListValue() { final Record record = getEventWithMetadata("message", Map.of("time", listValue)); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("time"), equalTo(List.of(currentItem, "time-value2"))); } @@ -621,7 +621,7 @@ public void testValueExpressionWithIntegerFunctionsAndMetadataKey() { String randomString = UUID.randomUUID().toString(); when(expressionEvaluator.evaluate(valueExpression, record.getData())).thenReturn(randomString.length()); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("length_key"), equalTo(randomString.length())); } @@ -634,7 +634,7 @@ public void testValueExpressionWithStringExpressionWithMetadataKey() { String randomString = UUID.randomUUID().toString(); when(expressionEvaluator.evaluate(valueExpression, record.getData())).thenReturn(randomString); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); - Map attributes = editedRecords.get(0).getData().getMetadata().getAttributes(); + Map attributes = editedRecords.get(0).getData().getEventMetadata().getAttributes(); assertThat(attributes.get("newkey"), equalTo(randomString)); } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java index 6985c776a0..a833c37048 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java @@ -113,8 +113,8 @@ void testMapToStringConvertEntryTypeProcessorWithInvalidTypeWillAddTags() { Event event = executeAndGetProcessedEvent(testValue); assertThat(event.get(TEST_KEY, Object.class), equalTo(testValue)); - assertThat(event.getMetadata().getTags().size(), equalTo(1)); - assertThat(event.getMetadata().getTags(), containsInAnyOrder(tags.toArray())); + assertThat(event.getEventMetadata().getTags().size(), equalTo(1)); + assertThat(event.getEventMetadata().getTags(), containsInAnyOrder(tags.toArray())); } @Test @@ -194,8 +194,8 @@ void testDoubleToIntegerConvertEntryTypeProcessorWillAddTags() { Event event = executeAndGetProcessedEvent(testDoubleValue); assertThat(event.get(TEST_KEY, Object.class), equalTo(123.789)); - assertThat(event.getMetadata().getTags().size(), equalTo(1)); - assertThat(event.getMetadata().getTags(), containsInAnyOrder(tags.toArray())); + assertThat(event.getEventMetadata().getTags().size(), equalTo(1)); + assertThat(event.getEventMetadata().getTags(), containsInAnyOrder(tags.toArray())); } @Test diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorTest.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorTest.java index a32bd8f8d5..57d748dbdb 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorTest.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorTest.java @@ -322,7 +322,7 @@ public void testTagsAreAddedOnFailure() { final Event resultEvent = resultRecord.get(0).getData(); assertThat(resultEvent.get("mymap", Object.class), is(nullValue())); - assertThat(resultEvent.getMetadata().getTags(), is(new HashSet<>(testTags))); + assertThat(resultEvent.getEventMetadata().getTags(), is(new HashSet<>(testTags))); } @Test diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java index 83d736ba21..e983a2085b 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java @@ -341,7 +341,7 @@ void testFailureTagsAreAddedWhenException() { final Event resultEvent = resultRecord.get(0).getData(); assertThat(resultEvent.containsKey("my-list"), is(false)); assertSourceMapUnchanged(resultEvent); - assertThat(resultEvent.getMetadata().getTags(), is(new HashSet<>(testTags))); + assertThat(resultEvent.getEventMetadata().getTags(), is(new HashSet<>(testTags))); } private MapToListProcessor createObjectUnderTest() { diff --git a/data-prepper-plugins/newline-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/newline/NewlineDelimitedCodecTest.java b/data-prepper-plugins/newline-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/newline/NewlineDelimitedCodecTest.java index f202e3289e..690c293fee 100644 --- a/data-prepper-plugins/newline-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/newline/NewlineDelimitedCodecTest.java +++ b/data-prepper-plugins/newline-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/newline/NewlineDelimitedCodecTest.java @@ -142,8 +142,8 @@ void parse_calls_Consumer_for_each_line_after_skipping(final int numberOfLines) assertThat(record, notNullValue()); assertThat(record.getData(), notNullValue()); assertThat(record.getData().get("message", String.class), equalTo(linesList.get(i + skipLines))); - assertThat(record.getData().getMetadata(), notNullValue()); - assertThat(record.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(record.getData().getEventMetadata(), notNullValue()); + assertThat(record.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); } } @@ -178,8 +178,8 @@ void parse_with_header_calls_Consumer_with_header_fields_no_skip(final int numbe assertThat(record.getData(), notNullValue()); assertThat(record.getData().get("event_header", String.class), equalTo(headerMessage)); assertThat(record.getData().get("message", String.class), equalTo(linesList.get(i + headerOffset))); - assertThat(record.getData().getMetadata(), notNullValue()); - assertThat(record.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(record.getData().getEventMetadata(), notNullValue()); + assertThat(record.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); } } @@ -205,8 +205,8 @@ void parse_with_header_calls_Consumer_with_header_fields_after_skip(final int nu assertThat(record.getData(), notNullValue()); assertThat(record.getData().get("event_header", String.class), equalTo(headerMessage)); assertThat(record.getData().get("message", String.class), equalTo(linesList.get(i + skipLines + headerOffset))); - assertThat(record.getData().getMetadata(), notNullValue()); - assertThat(record.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(record.getData().getEventMetadata(), notNullValue()); + assertThat(record.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); } } @@ -232,8 +232,8 @@ void parse_with_header_calls_Consumer_with_header_fields_after_multiple_skips(fi assertThat(record.getData(), notNullValue()); assertThat(record.getData().get("event_header", String.class), equalTo(headerMessage)); assertThat(record.getData().get("message", String.class), equalTo(linesList.get(i + skipLines + headerOffset))); - assertThat(record.getData().getMetadata(), notNullValue()); - assertThat(record.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(record.getData().getEventMetadata(), notNullValue()); + assertThat(record.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); } } diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java index 21167bc747..dbe0e189d3 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java @@ -125,7 +125,7 @@ public Collection> doExecute(Collection> records) { // No changes means it does not match any patterns if (rawValue.equals(newValue)) { - recordEvent.getMetadata().addTags(obfuscationProcessorConfig.getTagsOnMatchFailure()); + recordEvent.getEventMetadata().addTags(obfuscationProcessorConfig.getTagsOnMatchFailure()); } // Update the event record. diff --git a/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java b/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java index b29ad3b0f4..c913d7077b 100644 --- a/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java +++ b/data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java @@ -118,9 +118,9 @@ void event_is_tagged_with_match_failure_tags_when_it_does_not_match_any_patterns assertThat(editedRecords.size(), equalTo(1)); assertThat(editedRecords.get(0).getData().toMap(), equalTo(expectedEventMap)); - assertThat(editedRecords.get(0).getData().getMetadata().getTags(), notNullValue()); - assertThat(editedRecords.get(0).getData().getMetadata().getTags().size(), equalTo(1)); - assertThat(editedRecords.get(0).getData().getMetadata().getTags().contains(mockConfig.getTagsOnMatchFailure().get(0)), equalTo(true)); + assertThat(editedRecords.get(0).getData().getEventMetadata().getTags(), notNullValue()); + assertThat(editedRecords.get(0).getData().getEventMetadata().getTags().size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().getEventMetadata().getTags().contains(mockConfig.getTagsOnMatchFailure().get(0)), equalTo(true)); } diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 38bb5d1caa..e700777569 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -767,11 +767,11 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "create"); + event.getEventMetadata().setAttribute("action", "create"); final String actionFormatExpression = "${getMetadata(\"action\")}"; when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); + when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getEventMetadata().getAttribute("action")); pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); sink.output(testRecords); @@ -800,11 +800,11 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final PluginSetting pluginSetting = generatePluginSetting(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); Event event = (Event) testRecords.get(0).getData(); - event.getMetadata().setAttribute("action", "unknown"); + event.getEventMetadata().setAttribute("action", "unknown"); final String actionFormatExpression = "${getMetadata(\"action\")}"; when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true); when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true); - when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action")); + when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getEventMetadata().getAttribute("action")); pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression); final OpenSearchSink sink = createObjectUnderTest(pluginSetting, true); sink.output(testRecords); @@ -1076,7 +1076,7 @@ public void testEventOutputWithTags() throws IOException, InterruptedException { .withEventType("event") .build(); List tagsList = List.of("tag1", "tag2"); - testEvent.getMetadata().addTags(tagsList); + testEvent.getEventMetadata().addTags(tagsList); final List> testRecords = Collections.singletonList(new Record<>(testEvent)); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java index 54736c0885..2fe853b07f 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java @@ -402,6 +402,7 @@ public void doOutput(final Collection> records) { try { bulkOperationWrapper = createBulkOperationWrapper(event, bulkResponseItemConsumer, indexName, null); } catch (final Exception e) { + LOG.error("Caught exception creating bulk wrapper", e); continue; } @@ -421,11 +422,15 @@ public void doOutput(final Collection> records) { } // Here down is findings shipping + if (findings.size() > 0) { + LOG.info("Shipping {} findings", findings.size()); + } for (final Record record : findings) { final Event event = record.getData(); final String ruleEngineId = event.get("RULE_ENGINE_DOC_MATCH_ID", String.class); final List replacementFields = event.getList("RULE_ENGINE_DOC_ID_REPLACEMENT_FIELDS", String.class); final String indexName = event.get("FINDINGS_INDEX_NAME", String.class); + LOG.info("Findings index: {}", indexName); final List docInfo = ruleEngineIdToDocId.get(ruleEngineId); if (docInfo != null) { @@ -445,6 +450,7 @@ public void doOutput(final Collection> records) { try { bulkOperationWrapper = createBulkOperationWrapper(event, null, indexName, event.get("id", String.class)); } catch (final Exception e) { + LOG.error("Caught exception creating bulk wrapper", e); continue; } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java index 52b0aa91d0..5334676b94 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/NoSearchContextWorker.java @@ -169,8 +169,8 @@ private void processIndex(final SourcePartition op } catch (Exception e) { openSearchSourcePluginMetrics.getProcessingErrorsCounter().increment(); LOG.error("Failed writing OpenSearch documents to buffer. The last document created has document id '{}' from index '{}' : {}", - record.getData().getMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), - record.getData().getMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); + record.getData().getEventMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), + record.getData().getEventMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); } }); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java index b6a4256d7d..93b7af0296 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/PitWorker.java @@ -206,8 +206,8 @@ private void processIndex(final SourcePartition op } catch (Exception e) { openSearchSourcePluginMetrics.getProcessingErrorsCounter().increment(); LOG.error("Failed writing OpenSearch documents to buffer. The last document created has document id '{}' from index '{}' : {}", - record.getData().getMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), - record.getData().getMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); + record.getData().getEventMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), + record.getData().getEventMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); } }); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java index b063219239..d1f984cbea 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/source/opensearch/worker/ScrollWorker.java @@ -213,8 +213,8 @@ private void writeDocumentsToBuffer(final List documents, } catch (Exception e) { openSearchSourcePluginMetrics.getProcessingErrorsCounter().increment(); LOG.error("Failed writing OpenSearch documents to buffer. The last document created has document id '{}' from index '{}' : {}", - record.getData().getMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), - record.getData().getMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); + record.getData().getEventMetadata().getAttribute(DOCUMENT_ID_METADATA_ATTRIBUTE_NAME), + record.getData().getEventMetadata().getAttribute(INDEX_METADATA_ATTRIBUTE_NAME), e.getMessage()); } }); } diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DocumentBuilderTest.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DocumentBuilderTest.java index 1277303bd0..71af45f777 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DocumentBuilderTest.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/DocumentBuilderTest.java @@ -40,7 +40,7 @@ public void setup() throws JsonProcessingException { .withData(data) .withEventType("TestEvent") .build(); - event.getMetadata().addTags(List.of("tag1")); + event.getEventMetadata().addTags(List.of("tag1")); expectedOutput = objectMapper.writeValueAsString(data); expectedOutputWithTags = event.jsonBuilder().includeTags(tagsKey).toJsonString(); } diff --git a/data-prepper-plugins/parquet-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetInputCodecTest.java b/data-prepper-plugins/parquet-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetInputCodecTest.java index b13e36b206..c516dc5f10 100644 --- a/data-prepper-plugins/parquet-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetInputCodecTest.java +++ b/data-prepper-plugins/parquet-codecs/src/test/java/org/opensearch/dataprepper/plugins/codec/parquet/ParquetInputCodecTest.java @@ -227,8 +227,8 @@ private void assertRecordsCorrect(final List> records) { assertThat(metadata.get("key"), equalTo("value")); assertThat(lastUpdated, equalTo(1684509331977L)); - assertThat(record.getData().getMetadata(), notNullValue()); - assertThat(record.getData().getMetadata().getEventType(), equalTo("event")); + assertThat(record.getData().getEventMetadata(), notNullValue()); + assertThat(record.getData().getEventMetadata().getEventType(), equalTo("event")); } } } diff --git a/data-prepper-plugins/parse-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/parse/AbstractParseProcessor.java b/data-prepper-plugins/parse-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/parse/AbstractParseProcessor.java index b113534983..14ba544bbb 100644 --- a/data-prepper-plugins/parse-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/parse/AbstractParseProcessor.java +++ b/data-prepper-plugins/parse-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/parse/AbstractParseProcessor.java @@ -78,7 +78,7 @@ public Collection> doExecute(final Collection> recor final Optional> parsedValueOptional = readValue(message, event); if (parsedValueOptional.isEmpty()) { - event.getMetadata().addTags(tagsOnFailure); + event.getEventMetadata().addTags(tagsOnFailure); continue; } diff --git a/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/json/JsonInputCodecTest.java b/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/json/JsonInputCodecTest.java index 544f486252..29b1792282 100644 --- a/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/json/JsonInputCodecTest.java +++ b/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/codec/json/JsonInputCodecTest.java @@ -181,8 +181,8 @@ void parse_with_InputFile_calls_Consumer_with_Event(final int numberOfObjects) t final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = jsonObjects.get(i); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -207,8 +207,8 @@ void parse_with_InputStream_calls_Consumer_with_Event(final int numberOfObjects) final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = jsonObjects.get(i); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -234,8 +234,8 @@ void parse_with_InputStream_calls_Consumer_for_arrays_in_Json_permutations(final final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = jsonObjects.get(i); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); @@ -267,8 +267,8 @@ void parse_with_InputStream_calls_Consumer_with_Event_for_two_parallel_arrays(fi final Record actualRecord = actualRecords.get(i); assertThat(actualRecord, notNullValue()); assertThat(actualRecord.getData(), notNullValue()); - assertThat(actualRecord.getData().getMetadata(), notNullValue()); - assertThat(actualRecord.getData().getMetadata().getEventType(), equalTo(EventType.LOG.toString())); + assertThat(actualRecord.getData().getEventMetadata(), notNullValue()); + assertThat(actualRecord.getData().getEventMetadata().getEventType(), equalTo(EventType.LOG.toString())); final Map expectedMap = expectedJsonObjects.get(i); assertThat(actualRecord.getData().toMap(), equalTo(expectedMap)); diff --git a/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorTest.java b/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorTest.java index 4594cbe2f5..deb33e1430 100644 --- a/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorTest.java +++ b/data-prepper-plugins/parse-json-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorTest.java @@ -341,7 +341,7 @@ void test_tags_when_json_parse_fails() { parseJsonProcessor = createObjectUnderTest(); final Event parsedEvent = createAndParseMessageEvent(testEvent); - assertTrue(parsedEvent.getMetadata().hasTags(testTags)); + assertTrue(parsedEvent.getEventMetadata().hasTags(testTags)); } @Test diff --git a/data-prepper-plugins/rule-engine/build.gradle b/data-prepper-plugins/rule-engine/build.gradle index 184515f6ae..18ed86a16b 100644 --- a/data-prepper-plugins/rule-engine/build.gradle +++ b/data-prepper-plugins/rule-engine/build.gradle @@ -37,6 +37,7 @@ dependencies { implementation libs.opensearch.java implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation libs.commons.lang3 implementation 'org.projectlombok:lombok:1.18.26' compileOnly 'org.projectlombok:lombok:1.18.26' diff --git a/data-prepper-plugins/rule-engine/src/main/antlr/Aggregation.g4 b/data-prepper-plugins/rule-engine/src/main/antlr/Aggregation.g4 new file mode 100644 index 0000000000..be395c5ae4 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/antlr/Aggregation.g4 @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +grammar Aggregation; + +GT : '>' ; +GE : '>=' ; +LT : '<' ; +LE : '<=' ; +EQ : '==' ; + +COUNT : 'count' ; +SUM : 'sum' ; +MIN : 'min' ; +MAX : 'max' ; +AVG : 'avg' ; +BY : 'by' ; +LPAREN : '(' ; +RPAREN : ')' ; + +DECIMAL : '-'?[0-9]+('.'[0-9]+)? ; + +IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ; +WS : [ \r\t\u000C\n]+ -> skip ; + +comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator + ; + +comparison_operand : agg_expr + ; + +comp_operator : GT + | GE + | LT + | LE + | EQ + ; + +agg_operator : COUNT + | SUM + | MIN + | MAX + | AVG + ; + +groupby_expr : IDENTIFIER ; + +agg_expr + : agg_operator LPAREN agg_expr RPAREN BY? groupby_expr? # AggExpressionParens + | numeric_entity # AggExpressionNumericEntity + ; + +numeric_entity : DECIMAL # NumericConst + | IDENTIFIER # NumericVariable + ; \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java index e3c3acbe0c..653caa2b3e 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngine.java @@ -1,5 +1,8 @@ package org.opensearch.dataprepper.plugins.processor; +import org.opensearch.dataprepper.plugins.processor.evaluator.CompositeRuleEvaluator; +import org.opensearch.dataprepper.plugins.processor.evaluator.CorrelationEvaluator; +import org.opensearch.dataprepper.plugins.processor.evaluator.DefaultCorrelationEvaluator; import org.opensearch.dataprepper.plugins.processor.evaluator.DefaultRuleEvaluator; import org.opensearch.dataprepper.plugins.processor.evaluator.RuleEvaluator; import org.opensearch.dataprepper.plugins.processor.model.mappings.Mapping; @@ -7,6 +10,8 @@ import org.opensearch.dataprepper.plugins.processor.provider.rules.RuleProvider; import org.opensearch.dataprepper.plugins.processor.registrar.MappingRegistrar; import org.opensearch.dataprepper.plugins.processor.registrar.RuleProviderRegistrar; +import org.opensearch.dataprepper.plugins.processor.registrar.SubMatchAccessorRegistrar; +import org.opensearch.dataprepper.plugins.processor.retrievers.SubMatchAccessor; import org.opensearch.dataprepper.plugins.processor.rules.RuleRefresher; import org.opensearch.dataprepper.plugins.processor.rules.RuleStore; import org.slf4j.Logger; @@ -25,11 +30,14 @@ public class RuleEngine { private final MappingRegistrar mappingRegistrar; private final RuleProviderRegistrar ruleProviderRegistrar; + private final SubMatchAccessorRegistrar subMatchAccessorRegistrar; private RuleEvaluator ruleEvaluator; + private CorrelationEvaluator correlationEvaluator; public RuleEngine() { mappingRegistrar = new MappingRegistrar(); ruleProviderRegistrar = new RuleProviderRegistrar(); + subMatchAccessorRegistrar = new SubMatchAccessorRegistrar(); } public void registerMapping(final String logType, final Supplier mappingSupplier) { @@ -40,6 +48,10 @@ public void registerRuleProvider(final String ruleLocation, final Supplier subMatchAccessorSupplier) { + subMatchAccessorRegistrar.registerSubMatchAccessor(accessorName, subMatchAccessorSupplier); + } + public RuleEvaluator start(final RuleEngineConfig config) { final RuleStore ruleStore = new RuleStore(); setupRuleFetching(config, ruleStore); @@ -47,6 +59,13 @@ public RuleEvaluator start(final RuleEngineConfig config) { if (ruleEvaluator == null) { ruleEvaluator = new DefaultRuleEvaluator(ruleStore); } + if (correlationEvaluator == null && config.getSubMatchAccessor() != null) { + correlationEvaluator = new DefaultCorrelationEvaluator(ruleStore, subMatchAccessorRegistrar.getSubMatchAccessor(config.getSubMatchAccessor())); + return CompositeRuleEvaluator.builder() + .ruleEvaluator(ruleEvaluator) + .correlationEvaluator(correlationEvaluator) + .build(); + } return ruleEvaluator; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java index 94c02d7a53..5767df7c93 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineConfig.java @@ -1,42 +1,18 @@ package org.opensearch.dataprepper.plugins.processor; +import lombok.Builder; +import lombok.Getter; import org.opensearch.dataprepper.plugins.processor.model.rule.RuleSchema; import java.time.Duration; +@Builder +@Getter public class RuleEngineConfig { private final Duration ruleRefreshInterval; private final String logFormat; private final String logType; private final RuleSchema ruleSchema; private final String ruleLocation; - - public RuleEngineConfig(final Duration ruleRefreshInterval, final String logFormat, final String logType, - final RuleSchema ruleSchema, final String ruleLocation) { - this.ruleRefreshInterval = ruleRefreshInterval; - this.logFormat = logFormat; - this.logType = logType; - this.ruleSchema = ruleSchema; - this.ruleLocation = ruleLocation; - } - - public Duration getRuleRefreshInterval() { - return ruleRefreshInterval; - } - - public String getLogFormat() { - return logFormat; - } - - public String getLogType() { - return logType; - } - - public RuleSchema getRuleSchema() { - return ruleSchema; - } - - public String getRuleLocation() { - return ruleLocation; - } + private final String subMatchAccessor; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java index d8f7a1fa71..07c62a9973 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessor.java @@ -21,6 +21,7 @@ import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; import org.opensearch.dataprepper.plugins.processor.model.matches.Match; import org.opensearch.dataprepper.plugins.processor.provider.rules.opensearch.OpenSearchRuleProvider; +import org.opensearch.dataprepper.plugins.processor.retrievers.OpenSearchSubMatchAccessor; import org.opensearch.dataprepper.plugins.processor.util.OpenSearchDocMetadata; import org.opensearch.dataprepper.plugins.sink.opensearch.OpenSearchSinkConfiguration; import org.opensearch.dataprepper.plugins.sink.opensearch.index.ClusterSettingsParser; @@ -78,9 +79,16 @@ public RuleEngineProcessor(final PluginMetrics pluginMetrics, final RuleEngine ruleEngine = new RuleEngine(); ruleEngine.registerRuleProvider("opensearch", () -> new OpenSearchRuleProvider(openSearchClient)); - - final RuleEngineConfig ruleEngineConfig = new RuleEngineConfig(config.getRuleRefreshInterval(), config.getLogFormat(), config.getLogType(), - config.getRuleSchema(), config.getRuleLocation()); + ruleEngine.registerSubMatchAccessor("opensearch", () -> new OpenSearchSubMatchAccessor(openSearchClient)); + + final RuleEngineConfig ruleEngineConfig = RuleEngineConfig.builder() + .ruleRefreshInterval(config.getRuleRefreshInterval()) + .logFormat(config.getLogFormat()) + .logType(config.getLogType()) + .ruleLocation(config.getRuleLocation()) + .ruleSchema(config.getRuleSchema()) + .subMatchAccessor(config.getSubMatchAccessor()) + .build(); ruleEvaluator = ruleEngine.start(ruleEngineConfig); findingConverter = new FindingConverter(); acknowledgementSet = null; @@ -98,7 +106,9 @@ public Collection> doExecute(final Collection> recor final Map idToData = addTrackingData(records); final Collection dataWithMatches = ruleEvaluator.evaluate(idToData.values()); + LOG.info("Matches: {}", dataWithMatches.size()); final Collection> matches = convertMatchesToEvents(dataWithMatches); + LOG.info("Matches as events: {}", matches.size()); if (config.isDropData()) { return matches; @@ -113,7 +123,7 @@ private Map addTrackingData(final Collection> re .map(record -> { final String id = UUID.randomUUID().toString(); final DataType dataType = (DataType) record.getData(); - dataType.putMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record)); + dataType.putDataTypeMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName(), getIndexName(record)); final Map.Entry mapEntry = Map.entry(id, dataType); record.getData().put(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName(), id); diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessorConfig.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessorConfig.java index 7a5245b6e1..6770dd3e5e 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessorConfig.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/RuleEngineProcessorConfig.java @@ -39,6 +39,9 @@ public class RuleEngineProcessorConfig { @JsonProperty("drop_data") private boolean dropData = false; + @JsonProperty("sub_match_accessor") + private String subMatchAccessor; + public Duration getRuleRefreshInterval() { return ruleRefreshInterval; } @@ -66,4 +69,8 @@ public Map getOpenSearchConfiguration() { public boolean isDropData() { return dropData; } + + public String getSubMatchAccessor() { + return subMatchAccessor; + } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java index 570e85f4bb..fd3be4e83e 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/converters/FindingConverter.java @@ -1,9 +1,10 @@ package org.opensearch.dataprepper.plugins.processor.converters; import org.opensearch.dataprepper.plugins.processor.model.matches.Match; -import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1Rule; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchRuleMetadata; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1StatefulRule; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1StatelessRule; import org.opensearch.dataprepper.plugins.processor.rules.Rule; -import org.opensearch.dataprepper.plugins.processor.rules.SigmaV1Rule; import org.opensearch.dataprepper.plugins.processor.util.OpenSearchDocMetadata; import java.time.Instant; @@ -26,8 +27,10 @@ public List> convert(final Match match) { private Map> groupMatchByMonitors(final Match match) { final Map> monitorToRules = new HashMap<>(); - match.getRuleMatches().forEach(rule -> { - final String monitorId = ((OpenSearchSigmaV1Rule) rule).getMonitorId(); + match.getRules().forEach(rule -> { + final String monitorId = rule instanceof OpenSearchSigmaV1StatelessRule ? + ((OpenSearchSigmaV1StatelessRule) rule).getOpenSearchRuleMetadata().getMonitorId() : + ((OpenSearchSigmaV1StatefulRule) rule).getOpenSearchRuleMetadata().getMonitorId(); monitorToRules.putIfAbsent(monitorId, new ArrayList<>()); monitorToRules.get(monitorId).add(rule); @@ -37,30 +40,37 @@ private Map> groupMatchByMonitors(final Match match) { } private Map generateEventForMonitor(final Match match, final List rules) { - final OpenSearchSigmaV1Rule openSearchSigmaV1Rule = (OpenSearchSigmaV1Rule) (rules.get(0)); + final OpenSearchRuleMetadata openSearchRuleMetadata = rules.get(0) instanceof OpenSearchSigmaV1StatelessRule ? + ((OpenSearchSigmaV1StatelessRule) rules.get(0)).getOpenSearchRuleMetadata() : + ((OpenSearchSigmaV1StatefulRule) rules.get(0)).getOpenSearchRuleMetadata(); final Map eventMap = new HashMap<>(); eventMap.put("id", UUID.randomUUID().toString()); - eventMap.put("monitor_id", openSearchSigmaV1Rule.getMonitorId()); - eventMap.put("monitor_name", openSearchSigmaV1Rule.getDetectorName()); - eventMap.put("index", match.getDataType().getMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName())); + eventMap.put("monitor_id", openSearchRuleMetadata.getMonitorId()); + eventMap.put("monitor_name", openSearchRuleMetadata.getDetectorName()); + eventMap.put("index", match.getDataType().getDataTypeMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName())); eventMap.put("queries", rules.stream().map(this::getQuery).collect(Collectors.toList())); eventMap.put("timestamp", Instant.now().toEpochMilli()); eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_ID_REPLACEMENT_FIELDS.getFieldName(), List.of("related_doc_ids", "correlated_doc_ids")); - eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getMetadataValue(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName())); - eventMap.put(OpenSearchDocMetadata.FINDINGS_INDEX_NAME.getFieldName(), openSearchSigmaV1Rule.getFindingsIndex()); + eventMap.put(OpenSearchDocMetadata.RULE_ENGINE_DOC_MATCH_ID.getFieldName(), match.getDataType().getDataTypeMetadataValue(OpenSearchDocMetadata.RULE_ENGINE_ID.getFieldName())); + eventMap.put(OpenSearchDocMetadata.FINDINGS_INDEX_NAME.getFieldName(), openSearchRuleMetadata.getFindingsIndex()); return eventMap; } private Map getQuery(final Rule rule) { - final SigmaV1Rule sigmaV1Rule = (SigmaV1Rule) rule; + List tags = null; + if (rule instanceof OpenSearchSigmaV1StatelessRule) { + tags = ((OpenSearchSigmaV1StatelessRule) rule).getSigmaV1RuleMetadata().getTags(); + } else if (rule instanceof OpenSearchSigmaV1StatefulRule) { + tags = ((OpenSearchSigmaV1StatefulRule) rule).getSigmaV1RuleMetadata().getTags(); + } final Map queryMap = new HashMap<>(); - queryMap.put("id", sigmaV1Rule.getId()); - queryMap.put("name", sigmaV1Rule.getId()); + queryMap.put("id", rule.getId()); + queryMap.put("name", rule.getId()); queryMap.put("query", "PLACEHOLDER"); - queryMap.put("tags", ((SigmaV1Rule) rule).getTags()); + queryMap.put("tags", tags); return queryMap; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CompositeRuleEvaluator.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CompositeRuleEvaluator.java new file mode 100644 index 0000000000..5af2677443 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CompositeRuleEvaluator.java @@ -0,0 +1,19 @@ +package org.opensearch.dataprepper.plugins.processor.evaluator; + +import lombok.Builder; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; + +import java.util.Collection; + +@Builder +public class CompositeRuleEvaluator implements RuleEvaluator { + private final RuleEvaluator ruleEvaluator; + private final CorrelationEvaluator correlationEvaluator; + + @Override + public Collection evaluate(final Collection data) { + final Collection matches = ruleEvaluator.evaluate(data); + return correlationEvaluator.evaluate(matches); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CorrelationEvaluator.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CorrelationEvaluator.java new file mode 100644 index 0000000000..e4824f3d7b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/CorrelationEvaluator.java @@ -0,0 +1,9 @@ +package org.opensearch.dataprepper.plugins.processor.evaluator; + +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; + +import java.util.Collection; + +public interface CorrelationEvaluator { + Collection evaluate(Collection matches); +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultCorrelationEvaluator.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultCorrelationEvaluator.java new file mode 100644 index 0000000000..06594be69a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultCorrelationEvaluator.java @@ -0,0 +1,147 @@ +package org.opensearch.dataprepper.plugins.processor.evaluator; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; +import org.opensearch.dataprepper.plugins.processor.retrievers.SubMatchAccessor; +import org.opensearch.dataprepper.plugins.processor.rules.StatelessRule; +import org.opensearch.dataprepper.plugins.processor.rules.RuleStore; +import org.opensearch.dataprepper.plugins.processor.rules.StatefulRule; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Log4j2 +public class DefaultCorrelationEvaluator implements CorrelationEvaluator { + private final RuleStore ruleStore; + private final SubMatchAccessor subMatchAccessor; + + public DefaultCorrelationEvaluator(final RuleStore ruleStore, final SubMatchAccessor subMatchAccessor) { + this.ruleStore = ruleStore; + this.subMatchAccessor = subMatchAccessor; + } + + @Override + public Collection evaluate(final Collection matches) { + if (matches.isEmpty()) { + return matches; + } + + log.info("Processing {} matches", matches.size()); + storeSubMatches(matches); + + final List rules = ruleStore.getStatefulRules(); + + matches.forEach(match -> { + log.info("Evaluating match"); + final List ruleMatches = rules.stream() + .filter(rule -> rule.testEvaluationCondition(match)) + .filter(rule -> evaluateRule(rule, match)) + .collect(Collectors.toList()); + log.info("Got stateful rule matches {}", ruleMatches); + match.setStatefulRuleMatches(ruleMatches); + + // Remove the stateless condition matches to avoid them generating findings + final List statefulConditionMatch = match.getStatelessRuleMatches().stream() + .filter(StatelessRule::isStatefulCondition) + .collect(Collectors.toList()); + match.getStatelessRuleMatches().removeAll(statefulConditionMatch); + }); + + final List filteredMatches = matches.stream() + .filter(match -> !match.getStatefulRuleMatches().isEmpty() || !match.getStatelessRuleMatches().isEmpty()) + .collect(Collectors.toList()); + log.info("Outputting match count {}", filteredMatches.size()); + + return filteredMatches; + } + + private void storeSubMatches(final Collection matches) { + final List subMatches = new ArrayList<>(); + matches.forEach(match -> { + final List statefulConditionMatch = match.getStatelessRuleMatches().stream() + .filter(StatelessRule::isStatefulCondition) + .collect(Collectors.toList()); + + final Match submatch = Match.builder() + .statelessRuleMatches(statefulConditionMatch) + .dataType(match.getDataType()) + .build(); + subMatches.add(submatch); + }); + + + log.info("Storing {} matches", subMatches.size()); + subMatchAccessor.storeSubMatches(subMatches); + } + + private boolean evaluateRule(final StatefulRule statefulRule, final Match match) { + final ArrayList matches = (ArrayList) subMatchAccessor.getSubMatches(statefulRule, match.getDataType()); + log.info("Evaluating rule with id {} against {} matches", statefulRule.getId(), matches.size()); + matches.add(match); + matches.sort(new MatchTimeComparator()); + + final int currentMatchIndex = matches.indexOf(match); + + int startIndex = 0; + while (startIndex <= currentMatchIndex) { + final List subList = getSubList(statefulRule.getTimeframe(), matches, startIndex, currentMatchIndex); + log.info("Start index {}, currentMatchIndex {}, subList size {}", startIndex, currentMatchIndex, subList.size()); + if (subList != null && statefulRule.testRuleCondition(subList)) { + log.info("Condition matched!"); + return true; + } + + startIndex++; + } + + return false; + } + + private List getSubList(final Duration window, final List matches, final int startIndex, final int currentMatchIndex) { + final String timeFieldName = matches.get(startIndex).getDataType().getTimeFieldName(); + final long windowStart = (long) matches.get(startIndex).getDataType().getValue(timeFieldName); + final long windowEnd = Instant.ofEpochMilli(windowStart).plus(window).toEpochMilli(); + + final int endIndex = findIndexOfTime(matches, startIndex, windowEnd); + + if (currentMatchIndex > endIndex) { + return null; + } + + return matches.subList(startIndex, endIndex); + } + + private int findIndexOfTime(final List matches, final int startIndex, final long epochMillis) { + if (startIndex >= matches.size()) { + return -1; + } + + // TODO binary search + for (int i = startIndex; i < matches.size(); i++) { + final Match match = matches.get(i); + final String timeFieldName = match.getDataType().getTimeFieldName(); + if ((long) match.getDataType().getValue(timeFieldName) > epochMillis) { + return i - 1; + } + } + + return matches.size() - 1; + } + + private static class MatchTimeComparator implements Comparator { + public int compare(final Match m1, final Match m2) { + final String timeField1 = m1.getDataType().getTimeFieldName(); + final Long epochMillis1 = (long) m1.getDataType().getValue(timeField1); + + final String timeField2 = m2.getDataType().getTimeFieldName(); + final Long epochMillis2 = (long) m2.getDataType().getValue(timeField2); + + return epochMillis1.compareTo(epochMillis2); + } + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultRuleEvaluator.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultRuleEvaluator.java index 9cf7f37026..b892729e04 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultRuleEvaluator.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/evaluator/DefaultRuleEvaluator.java @@ -1,8 +1,10 @@ package org.opensearch.dataprepper.plugins.processor.evaluator; +import lombok.extern.log4j.Log4j2; import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; import org.opensearch.dataprepper.plugins.processor.model.matches.Match; import org.opensearch.dataprepper.plugins.processor.rules.Rule; +import org.opensearch.dataprepper.plugins.processor.rules.StatelessRule; import org.opensearch.dataprepper.plugins.processor.rules.RuleStore; import java.util.ArrayList; @@ -11,6 +13,7 @@ import java.util.List; import java.util.stream.Collectors; +@Log4j2 public class DefaultRuleEvaluator implements RuleEvaluator { private final RuleStore ruleStore; @@ -24,20 +27,24 @@ public Collection evaluate(final Collection data) { return Collections.emptyList(); } - final List rules = ruleStore.getRules(); + final List statelessRules = ruleStore.getRules(); final List matches = new ArrayList<>(); - data.stream().forEach(item -> { - final List ruleMatches = rules.stream() + data.forEach(item -> { + final List statelessRuleMatches = statelessRules.stream() // Skip rules that don't apply - .filter(rule -> rule.getEvaluationCondition().test(item)) - .filter(rule -> rule.getRuleCondition().test(item)) + .filter(rule -> rule.testEvaluationCondition(item)) + .filter(rule -> rule.testRuleCondition(item)) .collect(Collectors.toList()); - if (ruleMatches.size() > 0) { - matches.add(new Match(item, ruleMatches)); + if (statelessRuleMatches.size() > 0) { + matches.add(Match.builder() + .dataType(item) + .rules(statelessRuleMatches) + .build()); } }); + log.info("Found {} matches from {} docs", matches.size(), data.size()); return matches; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java index f8deb2c412..8d045699d4 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/formats/accessors/FieldAccessor.java @@ -32,6 +32,10 @@ public Float getFloatValue(final DataType event, final String fieldName) { return getValue(event, convertFieldName(fieldName), Float.class); } + public Double getDoubleValue(final DataType event, final String fieldName) { + return getValue(event, convertFieldName(fieldName), Double.class); + } + public Object getObjectValue(final DataType event, final String fieldName) { return getValue(event, convertFieldName(fieldName), Object.class); } @@ -44,7 +48,7 @@ private T getValue(final DataType event, final String fieldName, final Class } } - private String convertFieldName(final String fieldName) { + public String convertFieldName(final String fieldName) { final String mappedFieldName = mapping.get(fieldName); return mappedFieldName == null ? fieldName : mappedFieldName; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/CloudTrail.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/CloudTrail.java index 5fdfb874a0..64bda1d72a 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/CloudTrail.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/CloudTrail.java @@ -26,4 +26,9 @@ public Object getValue(final String fieldName) { default: throw new IllegalArgumentException("Field " + fieldName + " does not exist in class " + getClass().getName()); } } + + @Override + public String getTimeFieldName() { + return "eventTime"; + } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java index 879e1949fb..ca0d3d60a8 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/DataType.java @@ -1,120 +1,33 @@ package org.opensearch.dataprepper.plugins.processor.model.datatypes; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import org.opensearch.dataprepper.expression.ExpressionEvaluator; -import org.opensearch.dataprepper.model.event.DefaultEventHandle; -import org.opensearch.dataprepper.model.event.DefaultEventMetadata; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.EventHandle; -import org.opensearch.dataprepper.model.event.EventMetadata; -import org.opensearch.dataprepper.model.event.EventType; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf.OCSF; import java.util.HashMap; -import java.util.List; import java.util.Map; -public abstract class DataType implements Event { +@JsonDeserialize(as = OCSF.class) +public abstract class DataType { @JsonIgnore - private final EventMetadata eventMetadata; - @JsonIgnore - private final HashMap metadata; - @JsonIgnore - private final transient EventHandle eventHandle; + private final HashMap dataTypeMetadata; public DataType() { - eventMetadata = DefaultEventMetadata.builder().withEventType(EventType.LOG.toString()).build(); - metadata = new HashMap<>(); - eventHandle = new DefaultEventHandle(eventMetadata.getTimeReceived()); + dataTypeMetadata = new HashMap<>(); } public abstract Object getValue(final String fieldName); + public abstract String getTimeFieldName(); - public void putMetadataValue(final String metadataFieldName, final String metadataFieldValue) { - metadata.put(metadataFieldName, metadataFieldValue); - } - - public void putAllMetadata(final Map metadataEntries) { - metadata.putAll(metadataEntries); - } - - public String getMetadataValue(final String metadataFieldName) { - return metadata.get(metadataFieldName); - } - - - @Override - public void put(String key, Object value) { - - } - - @Override - public T get(String key, Class clazz) { - return null; - } - - @Override - public List getList(String key, Class clazz) { - return null; - } - - @Override - public void delete(String key) { - - } - - @Override - public String toJsonString() { - return null; - } - - @Override - public JsonNode getJsonNode() { - return null; - } - - @Override - public String getAsJsonString(String key) { - return null; - } - - @Override - public EventMetadata getMetadata() { - return eventMetadata; - } - - @Override - public boolean containsKey(String key) { - return false; - } - - @Override - public boolean isValueAList(String key) { - return false; - } - - @Override - public Map toMap() { - return null; - } - - @Override - public String formatString(String format) { - return null; - } - - @Override - public String formatString(String format, ExpressionEvaluator expressionEvaluator) { - return format; + public void putDataTypeMetadataValue(final String metadataFieldName, final String metadataFieldValue) { + dataTypeMetadata.put(metadataFieldName, metadataFieldValue); } - @Override - public EventHandle getEventHandle() { - return eventHandle; + public void putAllDataTypeMetadata(final Map metadataEntries) { + dataTypeMetadata.putAll(metadataEntries); } - @Override - public JsonStringBuilder jsonBuilder() { - return null; + public String getDataTypeMetadataValue(final String metadataFieldName) { + return dataTypeMetadata.get(metadataFieldName); } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java index d7b6baae55..f038e5f4d5 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/datatypes/ocsf/OCSF.java @@ -1,14 +1,23 @@ package org.opensearch.dataprepper.plugins.processor.model.datatypes.ocsf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Setter; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.model.event.DefaultEventHandle; +import org.opensearch.dataprepper.model.event.DefaultEventMetadata; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventHandle; +import org.opensearch.dataprepper.model.event.EventMetadata; +import org.opensearch.dataprepper.model.event.EventType; import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; import java.util.List; import java.util.Map; -@Setter -public class OCSF extends DataType { +@Data +public class OCSF extends DataType implements Event { private Metadata metadata; private Long time; private Cloud cloud; @@ -47,6 +56,13 @@ public class OCSF extends DataType { private Actor actor; private Map unmapped; + @JsonIgnore + private EventMetadata eventMetadata = DefaultEventMetadata.builder().withEventType(EventType.LOG.toString()).build(); + @JsonIgnore + private EventHandle eventHandle = new DefaultEventHandle(eventMetadata.getTimeReceived()); + @JsonIgnore + private String timeFieldName = "time"; + public OCSF() { super(); } @@ -118,4 +134,84 @@ private Object handleOtherFields(final String fieldName) { return unmapped.get(parts[1]); } + + @Override + public String getTimeFieldName() { + return timeFieldName; + } + + @Override + public void put(String key, Object value) { + + } + + @Override + public T get(String key, Class clazz) { + return null; + } + + @Override + public List getList(String key, Class clazz) { + return null; + } + + @Override + public void delete(String key) { + + } + + @Override + public String toJsonString() { + return null; + } + + @Override + public JsonNode getJsonNode() { + return null; + } + + @Override + public String getAsJsonString(String key) { + return null; + } + + @Override + public EventMetadata getEventMetadata() { + return eventMetadata; + } + + @Override + public boolean containsKey(String key) { + return false; + } + + @Override + public boolean isValueAList(String key) { + return false; + } + + @Override + public Map toMap() { + return null; + } + + @Override + public String formatString(String format) { + return null; + } + + @Override + public String formatString(String format, ExpressionEvaluator expressionEvaluator) { + return format; + } + + @Override + public EventHandle getEventHandle() { + return eventHandle; + } + + @Override + public JsonStringBuilder jsonBuilder() { + return null; + } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/Match.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/Match.java index 2b47976b38..bce3baa6e7 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/Match.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/Match.java @@ -1,24 +1,22 @@ package org.opensearch.dataprepper.plugins.processor.model.matches; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; import org.opensearch.dataprepper.plugins.processor.rules.Rule; +import java.io.Serializable; import java.util.List; -public class Match { - private final DataType dataType; - private final List ruleMatches; - - public Match(final DataType dataType, final List ruleMatches) { - this.dataType = dataType; - this.ruleMatches = ruleMatches; - } - - public DataType getDataType() { - return dataType; - } - - public List getRuleMatches() { - return ruleMatches; - } +@Builder +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Match implements Serializable { + private DataType dataType; + @JsonProperty(access = JsonProperty.Access.READ_ONLY) + private List rules; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/SubMatch.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/SubMatch.java new file mode 100644 index 0000000000..a817a765cb --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/matches/SubMatch.java @@ -0,0 +1,14 @@ +package org.opensearch.dataprepper.plugins.processor.model.matches; + +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Builder +@Getter +public class SubMatch { + private String id; + private Map groupByFields; + private long time; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleSchema.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleSchema.java index 26fd19d969..1ee406addd 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleSchema.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/model/rule/RuleSchema.java @@ -2,13 +2,11 @@ import org.opensearch.dataprepper.plugins.processor.parser.OpenSearchSigmaV1RuleParser; import org.opensearch.dataprepper.plugins.processor.parser.RuleParser; -import org.opensearch.dataprepper.plugins.processor.parser.SigmaV1RuleParser; import java.util.Map; import java.util.function.Function; public enum RuleSchema { - SIGMA_V1(SigmaV1RuleParser::new), OPENSEARCH_SIGMA_V1(OpenSearchSigmaV1RuleParser::new); private final Function, RuleParser> parserConstructor; diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/OpenSearchSigmaV1RuleParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/OpenSearchSigmaV1RuleParser.java index 2df68546b2..6e43c1df52 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/OpenSearchSigmaV1RuleParser.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/OpenSearchSigmaV1RuleParser.java @@ -1,39 +1,169 @@ package org.opensearch.dataprepper.plugins.processor.parser; +import lombok.extern.log4j.Log4j2; +import org.opensearch.dataprepper.plugins.processor.formats.accessors.FieldAccessor; import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; +import org.opensearch.dataprepper.plugins.processor.parser.aggregation.AggregationItem; +import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; +import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaCondition; import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRuleTag; import org.opensearch.dataprepper.plugins.processor.provider.rules.model.RuleData; -import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1Rule; -import org.opensearch.dataprepper.plugins.processor.rules.Rule; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchRuleMetadata; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1StatefulRule; +import org.opensearch.dataprepper.plugins.processor.rules.OpenSearchSigmaV1StatelessRule; +import org.opensearch.dataprepper.plugins.processor.rules.SigmaV1RuleMetadata; +import org.opensearch.dataprepper.plugins.processor.rules.StatelessRule; +import org.opensearch.dataprepper.plugins.processor.rules.StatefulRule; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +@Log4j2 public class OpenSearchSigmaV1RuleParser implements RuleParser { - private final SigmaV1RuleConditionParser conditionParser; + private static final String TIMEFRAME_REGEX = "^(\\d+)(s|m|h|d)$"; + private static final Pattern TIMEFRAME_PATTERN = Pattern.compile(TIMEFRAME_REGEX); + + private final FieldAccessor fieldAccessor; + private final SigmaV1RuleStatelessConverter conditionParser; + private final SigmaV1RuleStatefulConverter aggregationParser; public OpenSearchSigmaV1RuleParser(final Map mapping) { - this.conditionParser = new SigmaV1RuleConditionParser(mapping); + this.fieldAccessor = new FieldAccessor(mapping); + this.conditionParser = new SigmaV1RuleStatelessConverter(fieldAccessor); + this.aggregationParser = new SigmaV1RuleStatefulConverter(fieldAccessor); } @Override - public Rule parseRule(final RuleData ruleData) { + public void parseRule(final RuleData ruleData, final Consumer ruleConsumer, final Consumer statefulRuleConsumer) { final SigmaRule sigmaRule = SigmaRule.fromYaml(ruleData.getRuleAsString(), true); - final Predicate ruleCondition = conditionParser.parseRuleCondition(sigmaRule); + final List conditionItems = getConditionItems(sigmaRule); + final AggregationItem aggregationItem = getAggregationItem(sigmaRule); + + final Predicate ruleCondition = conditionParser.parseRuleCondition(conditionItems); + final List tags = getTags(sigmaRule); + + final OpenSearchRuleMetadata openSearchRuleMetadata = OpenSearchRuleMetadata.builder() + .findingsIndex(ruleData.getMetadata().get("findingsIndex")) + .detectorName(ruleData.getMetadata().get("detectorName")) + .monitorId(ruleData.getMetadata().get("monitorId")) + .build(); + final SigmaV1RuleMetadata sigmaV1RuleMetadata = SigmaV1RuleMetadata.builder() + .title(sigmaRule.getTitle()) + .tags(tags) + .build(); + + final OpenSearchSigmaV1StatelessRule.OpenSearchSigmaV1StatelessRuleBuilder sigmaV1RuleBuilder = OpenSearchSigmaV1StatelessRule.builder() + .openSearchRuleMetadata(openSearchRuleMetadata) + .sigmaV1RuleMetadata(sigmaV1RuleMetadata) + .id(sigmaRule.getId().toString()) + .ruleCondition(ruleCondition) + .evaluationCondition(ruleData.getEvaluationCondition()); + + if (aggregationItem == null) { + ruleConsumer.accept(sigmaV1RuleBuilder.isStatefulCondition(false).build()); + } else { + ruleConsumer.accept(sigmaV1RuleBuilder.isStatefulCondition(true).build()); + + final List filterFields = getFilterFields(aggregationItem); + final Duration timeframe = parseTimeframe(sigmaRule.getDetection().getTimeframe()); + final Predicate evaluationCondition = getAggregationEvaluationCondition(sigmaRule.getId().toString()); + + final OpenSearchSigmaV1StatefulRule statefulRule = OpenSearchSigmaV1StatefulRule.builder() + .openSearchRuleMetadata(openSearchRuleMetadata) + .sigmaV1RuleMetadata(sigmaV1RuleMetadata) + .filterFields(filterFields) + .timeframe(timeframe) + .evaluationCondition(evaluationCondition) + .ruleCondition(aggregationParser.parseRuleAggregation(aggregationItem)) + .id(sigmaRule.getId().toString()) + .build(); + + statefulRuleConsumer.accept(statefulRule); + } + } + + private List getConditionItems(final SigmaRule sigmaRule) { + return sigmaRule.getDetection().getParsedConditions().stream() + .map(SigmaCondition::parseConditionItem) + .collect(Collectors.toList()); + } + + private AggregationItem getAggregationItem(final SigmaRule sigmaRule) { + final List aggregationItems = sigmaRule.getDetection().getParsedConditions().stream() + .map(SigmaCondition::parseAggregationItem) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (aggregationItems.size() > 1) { + throw new UnsupportedOperationException("Expected 0 or 1 aggregations. Found " + aggregationItems.size() + " for rule with id " + sigmaRule.getId()); + } + + return aggregationItems.isEmpty() ? null : aggregationItems.get(0); + } + + private List getTags(final SigmaRule sigmaRule) { final List tags = new ArrayList<>(); tags.add(sigmaRule.getLevel().toString()); tags.add(sigmaRule.getLogSource().getService()); sigmaRule.getTags().stream() .map(SigmaRuleTag::toString) .forEach(tags::add); - final String monitorId = ruleData.getMetadata().get("monitorId"); - final String detectorName = ruleData.getMetadata().get("detectorName"); - final String findingsIndex = ruleData.getMetadata().get("findingsIndex"); - return new OpenSearchSigmaV1Rule(monitorId, detectorName, findingsIndex, sigmaRule.getTitle(), - sigmaRule.getId().toString(), tags, ruleCondition, ruleData.getEvaluationCondition()); + return tags; + } + + private List getFilterFields(final AggregationItem aggregationItem) { + final List filterFields = new ArrayList<>(); + filterFields.add(fieldAccessor.convertFieldName(aggregationItem.getGroupByField())); + if ("count".equals(aggregationItem.getAggFunction()) && aggregationItem.getAggField() != null) { + filterFields.add(fieldAccessor.convertFieldName(aggregationItem.getAggField())); + } + + return filterFields; + } + + private Predicate getAggregationEvaluationCondition(final String id) { + return match -> { + final List ruleIds = match.getStatelessRuleMatches().stream() + .filter(StatelessRule::isStatefulCondition) + .map(StatelessRule::getId) + .collect(Collectors.toList()); + + return ruleIds.contains(id); + }; + } + + private Duration parseTimeframe(final String timeframeString) { + final String durationStringNoSpaces = timeframeString.replaceAll("\\s", ""); + final Matcher matcher = TIMEFRAME_PATTERN.matcher(durationStringNoSpaces); + if (!matcher.find()) { + return null; + } + + final long durationNumber = Long.parseLong(matcher.group(1)); + final String durationUnit = matcher.group(2); + + return getDurationCreatorFromUnit(durationUnit).apply(durationNumber); + } + + private Function getDurationCreatorFromUnit(final String durationUnit) { + switch (durationUnit) { + case "s": return Duration::ofSeconds; + case "m": return Duration::ofMinutes; + case "h": return Duration::ofHours; + case "d": return Duration::ofDays; + default: throw new UnsupportedOperationException("Unsupported timeframe unit \"" + durationUnit + "\""); + } } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/RuleParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/RuleParser.java index fca8468c8b..29b051415c 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/RuleParser.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/RuleParser.java @@ -1,8 +1,11 @@ package org.opensearch.dataprepper.plugins.processor.parser; import org.opensearch.dataprepper.plugins.processor.provider.rules.model.RuleData; -import org.opensearch.dataprepper.plugins.processor.rules.Rule; +import org.opensearch.dataprepper.plugins.processor.rules.StatelessRule; +import org.opensearch.dataprepper.plugins.processor.rules.StatefulRule; + +import java.util.function.Consumer; public interface RuleParser { - Rule parseRule(RuleData ruleData); + void parseRule(RuleData ruleData, Consumer ruleConsumer, Consumer statefulRuleConsumer); } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleParser.java deleted file mode 100644 index a0525dab48..0000000000 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleParser.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.opensearch.dataprepper.plugins.processor.parser; - -import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; -import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; -import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRuleTag; -import org.opensearch.dataprepper.plugins.processor.provider.rules.model.RuleData; -import org.opensearch.dataprepper.plugins.processor.rules.Rule; -import org.opensearch.dataprepper.plugins.processor.rules.SigmaV1Rule; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; - -public class SigmaV1RuleParser implements RuleParser { - private final SigmaV1RuleConditionParser conditionParser; - - public SigmaV1RuleParser(final Map mapping) { - this.conditionParser = new SigmaV1RuleConditionParser(mapping); - } - - @Override - public Rule parseRule(final RuleData ruleData) { - final SigmaRule sigmaRule = SigmaRule.fromYaml(ruleData.getRuleAsString(), true); - final Predicate ruleCondition = conditionParser.parseRuleCondition(sigmaRule); - final List tags = new ArrayList<>(); - tags.add(sigmaRule.getLevel().toString()); - tags.add(sigmaRule.getLogSource().getService()); - sigmaRule.getTags().stream() - .map(SigmaRuleTag::toString) - .forEach(tags::add); - - return new SigmaV1Rule(sigmaRule.getTitle(), sigmaRule.getId().toString(), tags, ruleCondition, ruleData.getEvaluationCondition()); - } -} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatefulConverter.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatefulConverter.java new file mode 100644 index 0000000000..0aa55ce106 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatefulConverter.java @@ -0,0 +1,67 @@ +package org.opensearch.dataprepper.plugins.processor.parser; + +import org.opensearch.dataprepper.plugins.processor.formats.accessors.FieldAccessor; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; +import org.opensearch.dataprepper.plugins.processor.parser.aggregation.AggregationItem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.OptionalDouble; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.DoubleStream; + +public class SigmaV1RuleStatefulConverter { + private static final Logger LOG = LoggerFactory.getLogger(SigmaV1RuleStatefulConverter.class); + + private final FieldAccessor fieldAccessor; + + public SigmaV1RuleStatefulConverter(final FieldAccessor fieldAccessor) { + this.fieldAccessor = fieldAccessor; + } + + public Predicate> parseRuleAggregation(final AggregationItem aggregationItem) { + switch (aggregationItem.getAggFunction()) { + case "count": return matches -> applyOperator((double) matches.size(), aggregationItem); + case "min": return matches -> applyOperator(getOptionalValue(matches, aggregationItem.getAggField(), DoubleStream::min), aggregationItem); + case "max": return matches -> applyOperator(getOptionalValue(matches, aggregationItem.getAggField(), DoubleStream::max), aggregationItem); + case "avg": return matches -> applyOperator(getOptionalValue(matches, aggregationItem.getAggField(), DoubleStream::average), aggregationItem); + case "sum": return matches -> applyOperator(getValue(matches, aggregationItem.getAggField(), DoubleStream::sum), aggregationItem); + default: throw new UnsupportedOperationException("Unexpected aggregation function \"" + aggregationItem.getAggFunction() + "\""); + } + + } + + private boolean applyOperator(final Double value, final AggregationItem aggregationItem) { + switch (aggregationItem.getCompOperator()) { + case ">": return value > aggregationItem.getThreshold(); + case "<": return value < aggregationItem.getThreshold(); + case ">=": return value >= aggregationItem.getThreshold(); + case "<=": return value <= aggregationItem.getThreshold(); + case "==": return aggregationItem.getThreshold().equals(value); + default: throw new UnsupportedOperationException("Unexpected operator \"" + aggregationItem.getCompOperator() + "\""); + } + } + + private double getOptionalValue(final List matches, final String fieldName, final Function comparator) { + final DoubleStream doubleStream = getDoubleStream(matches, fieldName); + final OptionalDouble optionalDouble = comparator.apply(doubleStream); + if (optionalDouble.isEmpty()) { + throw new IllegalArgumentException("No double value found for field name \"" + fieldName + "\""); + } + + return optionalDouble.getAsDouble(); + } + + private double getValue(final List matches, final String fieldName, final Function comparator) { + final DoubleStream doubleStream = getDoubleStream(matches, fieldName); + return comparator.apply(doubleStream); + } + + private DoubleStream getDoubleStream(final List matches, final String fieldName) { + return matches.stream() + .map(match -> fieldAccessor.getDoubleValue(match.getDataType(), fieldName)) + .mapToDouble(Double::doubleValue); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleConditionParser.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatelessConverter.java similarity index 84% rename from data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleConditionParser.java rename to data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatelessConverter.java index cfe03cf649..33209940b5 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleConditionParser.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleStatelessConverter.java @@ -8,8 +8,6 @@ import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionNOT; import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionOR; import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; -import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaCondition; -import org.opensearch.dataprepper.plugins.processor.parser.objects.SigmaRule; import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaBool; import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaFloat; import org.opensearch.dataprepper.plugins.processor.parser.types.SigmaInteger; @@ -20,22 +18,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; +import java.util.List; import java.util.function.Predicate; import java.util.stream.Stream; -public class SigmaV1RuleConditionParser { - private static final Logger LOG = LoggerFactory.getLogger(SigmaV1RuleConditionParser.class); +public class SigmaV1RuleStatelessConverter { + private static final Logger LOG = LoggerFactory.getLogger(SigmaV1RuleStatelessConverter.class); private final FieldAccessor fieldAccessor; - public SigmaV1RuleConditionParser(final Map mapping) { - this.fieldAccessor = new FieldAccessor(mapping); + public SigmaV1RuleStatelessConverter(final FieldAccessor fieldAccessor) { + this.fieldAccessor = fieldAccessor; } - public Predicate parseRuleCondition(final SigmaRule sigmaRule) { - return sigmaRule.getDetection().getParsedConditions().stream() - .map(SigmaCondition::parsed) + public Predicate parseRuleCondition(final List conditionItems) { + return conditionItems.stream() .map(this::parsePredicateFromConditionItem) // TODO - Not sure on this, need to figure out how there could be multiple conditions for same rule .reduce(Predicate::and) @@ -109,8 +106,8 @@ private Predicate convertFieldEquals(final ConditionFieldEqualsValueEx private Predicate convertStringEquals(final ConditionFieldEqualsValueExpression condition) { final SigmaString sigmaString = (SigmaString) condition.getValue(); - return event -> { - final String value = fieldAccessor.getStringValue(event, condition.getField()); + return dataType -> { + final String value = fieldAccessor.getStringValue(dataType, condition.getField()); return sigmaString.getOriginal().equals(value); }; @@ -118,20 +115,20 @@ private Predicate convertStringEquals(final ConditionFieldEqualsValueE private Predicate convertBooleanEquals(final ConditionFieldEqualsValueExpression condition) { final SigmaBool sigmaBool = (SigmaBool) condition.getValue(); - return event -> sigmaBool.getBoolean().equals(fieldAccessor.getBooleanValue(event, condition.getField())); + return dataType -> sigmaBool.getBoolean().equals(fieldAccessor.getBooleanValue(dataType, condition.getField())); } private Predicate convertIntegerEquals(final ConditionFieldEqualsValueExpression condition) { final SigmaInteger sigmaInteger = (SigmaInteger) condition.getValue(); - return event -> sigmaInteger.getInteger().equals(fieldAccessor.getIntegerValue(event, condition.getField())); + return dataType -> sigmaInteger.getInteger().equals(fieldAccessor.getIntegerValue(dataType, condition.getField())); } private Predicate convertFloatEquals(final ConditionFieldEqualsValueExpression condition) { final SigmaFloat sigmaFloat = (SigmaFloat) condition.getValue(); - return event -> sigmaFloat.getFloat().equals(fieldAccessor.getFloatValue(event, condition.getField())); + return dataType -> sigmaFloat.getFloat().equals(fieldAccessor.getFloatValue(dataType, condition.getField())); } private Predicate convertNullEquals(final ConditionFieldEqualsValueExpression condition) { - return event -> fieldAccessor.getObjectValue(event, condition.getField()) == null; + return dataType -> fieldAccessor.getObjectValue(dataType, condition.getField()) == null; } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationItem.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationItem.java new file mode 100644 index 0000000000..1fe1c97aac --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationItem.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.aggregation; + +import java.io.Serializable; + +public class AggregationItem implements Serializable { + + private static final long serialVersionUID = 1L; + + private String aggFunction; + + private String aggField; + + private String groupByField; + + private String compOperator; + + private Double threshold; + + public void setAggFunction(String aggFunction) { + this.aggFunction = aggFunction; + } + + public String getAggFunction() { + return aggFunction; + } + + public void setAggField(String aggField) { + this.aggField = aggField; + } + + public String getAggField() { + return aggField; + } + + public void setGroupByField(String groupByField) { + this.groupByField = groupByField; + } + + public String getGroupByField() { + return groupByField; + } + + public void setCompOperator(String compOperator) { + this.compOperator = compOperator; + } + + public String getCompOperator() { + return compOperator; + } + + public void setThreshold(Double threshold) { + this.threshold = threshold; + } + + public Double getThreshold() { + return threshold; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationTraverseVisitor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationTraverseVisitor.java new file mode 100644 index 0000000000..628c0031fd --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/aggregation/AggregationTraverseVisitor.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.processor.parser.aggregation; + +import org.opensearch.dataprepper.plugins.processor.rules.antlr.AggregationBaseVisitor; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.AggregationParser; + +public class AggregationTraverseVisitor extends AggregationBaseVisitor { + + private final AggregationItem aggregationItem; + + public AggregationTraverseVisitor() { + this.aggregationItem = new AggregationItem(); + } + + @Override + public AggregationItem visitComparisonExpressionWithOperator(AggregationParser.ComparisonExpressionWithOperatorContext ctx) { + if (ctx.comp_operator() != null) { + this.aggregationItem.setCompOperator(ctx.comp_operator().getText()); + } + return super.visitComparisonExpressionWithOperator(ctx); + } + + @Override + public AggregationItem visitAggExpressionParens(AggregationParser.AggExpressionParensContext ctx) { + if (ctx.agg_operator() != null) { + this.aggregationItem.setAggFunction(ctx.agg_operator().getText()); + } + return super.visitAggExpressionParens(ctx); + } + + @Override + public AggregationItem visitNumericConst(AggregationParser.NumericConstContext ctx) { + if (ctx.DECIMAL() != null) { + this.aggregationItem.setThreshold(Double.valueOf(ctx.DECIMAL().getText())); + } + return super.visitNumericConst(ctx); + } + + @Override + public AggregationItem visitNumericVariable(AggregationParser.NumericVariableContext ctx) { + if (ctx.IDENTIFIER() != null) { + this.aggregationItem.setAggField(ctx.IDENTIFIER().getText()); + } + return super.visitNumericVariable(ctx); + } + + @Override + public AggregationItem visitGroupby_expr(AggregationParser.Groupby_exprContext ctx) { + if(ctx.IDENTIFIER() != null) { + this.aggregationItem.setGroupByField(ctx.IDENTIFIER().getText()); + } + return super.visitGroupby_expr(ctx); + } + + public AggregationItem getAggregationItem() { + return aggregationItem; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java index 7dec12a160..50823fbc77 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/parser/objects/SigmaCondition.java @@ -6,6 +6,8 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; +import org.opensearch.dataprepper.plugins.processor.parser.aggregation.AggregationItem; +import org.opensearch.dataprepper.plugins.processor.parser.aggregation.AggregationTraverseVisitor; import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionFieldEqualsValueExpression; import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionIdentifier; import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionItem; @@ -14,6 +16,8 @@ import org.opensearch.dataprepper.plugins.processor.parser.condition.ConditionValueExpression; import org.opensearch.dataprepper.plugins.processor.parser.exceptions.SigmaConditionError; import org.opensearch.dataprepper.plugins.processor.parser.utils.Either; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.AggregationLexer; +import org.opensearch.dataprepper.plugins.processor.rules.antlr.AggregationParser; import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionLexer; import org.opensearch.dataprepper.plugins.processor.rules.antlr.ConditionParser; @@ -41,9 +45,11 @@ public class SigmaCondition { private SigmaDetections detections; private ConditionParser parser; - private ConditionTraverseVisitor conditionVisitor; + private AggregationParser aggParser; + private AggregationTraverseVisitor aggregationVisitor; + public SigmaCondition(String condition, SigmaDetections detections) { if (condition.contains(" | ")) { this.condition = condition.split(" \\| ")[0]; @@ -58,9 +64,13 @@ public SigmaCondition(String condition, SigmaDetections detections) { ConditionLexer lexer = new ConditionLexer(CharStreams.fromString(this.condition)); this.parser = new ConditionParser(new CommonTokenStream(lexer)); this.conditionVisitor = new ConditionTraverseVisitor(this); + + AggregationLexer aggLexer = new AggregationLexer(CharStreams.fromString(this.aggregation)); + this.aggParser = new AggregationParser(new CommonTokenStream(aggLexer)); + this.aggregationVisitor = new AggregationTraverseVisitor(); } - public ConditionItem parsed() throws SigmaConditionError { + public ConditionItem parseConditionItem() throws SigmaConditionError { ConditionItem parsedConditionItem; Either itemOrCondition = conditionVisitor.visit(parser.start()); if (itemOrCondition.isLeft()) { @@ -72,6 +82,15 @@ public ConditionItem parsed() throws SigmaConditionError { return parsedConditionItem; } + public AggregationItem parseAggregationItem() { + if (aggregation == null) { + return null; + } + + aggregationVisitor.visit(aggParser.comparison_expr()); + return aggregationVisitor.getAggregationItem(); + } + public List> convertArgs(List> parsedArgs) throws SigmaConditionError { List> newArgs = new ArrayList<>(); diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/provider/rules/opensearch/OpenSearchRuleProvider.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/provider/rules/opensearch/OpenSearchRuleProvider.java index a125bef559..45d04272de 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/provider/rules/opensearch/OpenSearchRuleProvider.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/provider/rules/opensearch/OpenSearchRuleProvider.java @@ -197,7 +197,7 @@ private Predicate getDetectorEvaluationCondition(final Detector detect .collect(Collectors.toSet()); return dataType -> { - final String index = dataType.getMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName()); + final String index = dataType.getDataTypeMetadataValue(OpenSearchDocMetadata.INDEX.getFieldName()); if (index == null) { return false; } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/registrar/SubMatchAccessorRegistrar.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/registrar/SubMatchAccessorRegistrar.java new file mode 100644 index 0000000000..8ba13d097e --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/registrar/SubMatchAccessorRegistrar.java @@ -0,0 +1,28 @@ +package org.opensearch.dataprepper.plugins.processor.registrar; + +import org.opensearch.dataprepper.plugins.processor.retrievers.SubMatchAccessor; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +public class SubMatchAccessorRegistrar { + private final Map> subMatchAccessorSuppliers; + + public SubMatchAccessorRegistrar() { + subMatchAccessorSuppliers = new HashMap<>(); + } + + public void registerSubMatchAccessor(final String accessorName, final Supplier subMatchAccessorSupplier) { + subMatchAccessorSuppliers.put(accessorName, subMatchAccessorSupplier); + } + + public SubMatchAccessor getSubMatchAccessor(final String accessorName) { + final Supplier subMatchAccessorSupplier = subMatchAccessorSuppliers.get(accessorName); + if (subMatchAccessorSupplier == null) { + throw new IllegalArgumentException("No SubMatchAccessor registered for name: " + accessorName); + } + + return subMatchAccessorSupplier.get(); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/OpenSearchSubMatchAccessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/OpenSearchSubMatchAccessor.java new file mode 100644 index 0000000000..e40d575456 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/OpenSearchSubMatchAccessor.java @@ -0,0 +1,119 @@ +package org.opensearch.dataprepper.plugins.processor.retrievers; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.RangeQuery; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; +import org.opensearch.dataprepper.plugins.processor.rules.StatefulRule; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; + +@Log4j2 +public class OpenSearchSubMatchAccessor implements SubMatchAccessor { + private static final String SUB_MATCHES_INDEX_NAME = "sub-matches"; + private final OpenSearchClient openSearchClient; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); + + public OpenSearchSubMatchAccessor(final OpenSearchClient openSearchClient) { + this.openSearchClient = openSearchClient; + } + + @Override + public List getSubMatches(final StatefulRule statefulRule, final DataType dataType) { + // TODO - add size and sort as well. Need to return <-max(N), +max(N)> window + final SearchResponse subMatches = getSubMatchesFromStore(statefulRule, dataType); + log.info("Retrieved {} hits from query", subMatches.hits().total().value()); + return subMatches.hits().hits().stream() + .map(Hit::source) + .collect(Collectors.toList()); + } + + @Override + public void storeSubMatches(final List subMatches) { + final List bulkOperations = subMatches.stream() + .map(this::getBulkOperation) + .collect(Collectors.toList()); + final BulkRequest bulkRequest = new BulkRequest.Builder() + .index(SUB_MATCHES_INDEX_NAME) + .operations(bulkOperations) + .build(); + + try { + final BulkResponse bulkResponse = openSearchClient.bulk(bulkRequest); + if (bulkResponse.errors()) { + log.error("Error writing submatches to store"); + bulkResponse.items().stream() + .filter(item -> item.error() != null) + .forEach(item -> log.error("BulkItem error: {}", item.error().reason())); + } + } catch (IOException e) { + throw new RuntimeException("Failed storing sub matches", e); + } + } + + private BulkOperation getBulkOperation(final Match match) { + + final IndexOperation.Builder indexOperationBuilder = + new IndexOperation.Builder<>() + .index(SUB_MATCHES_INDEX_NAME) + .document(match); + return new BulkOperation.Builder() + .index(indexOperationBuilder.build()) + .build(); + } + + private SearchResponse getSubMatchesFromStore(final StatefulRule statefulRule, final DataType dataType) { + try { + final SearchRequest searchRequest = getSubMatchesSearchRequest(statefulRule, dataType); + return openSearchClient.search(searchRequest, Match.class); + } catch (final Exception e) { + throw new RuntimeException("Failed to get submatches for rule with ID " + statefulRule.getId(), e); + } + } + + private SearchRequest getSubMatchesSearchRequest(final StatefulRule statefulRule, final DataType dataType) { + return new SearchRequest.Builder() + .index(SUB_MATCHES_INDEX_NAME) + .query(getSubMatchesQuery(statefulRule, dataType)._toQuery()) + .build(); + } + + private BoolQuery getSubMatchesQuery(final StatefulRule statefulRule, final DataType dataType) { + final List filterQueries = statefulRule.getFilterFields().stream() + .map(fieldName -> getFilterFieldMatchQuery(fieldName, (String) dataType.getValue(fieldName))) + .collect(Collectors.toList()); + filterQueries.add(MatchQuery.of(m -> m.field("statelessRuleMatches.id").query(FieldValue.of(statefulRule.getId())))._toQuery()); + + final String timeFieldName = dataType.getTimeFieldName(); + final Instant timestamp = Instant.ofEpochMilli((long) dataType.getValue(timeFieldName)); + final RangeQuery rangeQuery = RangeQuery.of(r -> r.field("dataType." + timeFieldName) + .gte(JsonData.of(timestamp.minus(statefulRule.getTimeframe()).toEpochMilli())) + .lte(JsonData.of(timestamp.plus(statefulRule.getTimeframe()).toEpochMilli()))); + filterQueries.add(rangeQuery._toQuery()); + + return BoolQuery.of(q -> q.must(filterQueries)); + } + + private Query getFilterFieldMatchQuery(final String filterFieldName, final String filterFieldValue) { + return MatchQuery.of(m -> m.field("dataType." + filterFieldName).query(FieldValue.of(filterFieldValue)))._toQuery(); + } +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/SubMatchAccessor.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/SubMatchAccessor.java new file mode 100644 index 0000000000..8ef498317c --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/retrievers/SubMatchAccessor.java @@ -0,0 +1,12 @@ +package org.opensearch.dataprepper.plugins.processor.retrievers; + +import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; +import org.opensearch.dataprepper.plugins.processor.rules.StatefulRule; + +import java.util.List; + +public interface SubMatchAccessor { + List getSubMatches(StatefulRule statefulRule, DataType dataType); + void storeSubMatches(List subMatches); +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchRuleMetadata.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchRuleMetadata.java new file mode 100644 index 0000000000..e2690d5668 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchRuleMetadata.java @@ -0,0 +1,12 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class OpenSearchRuleMetadata { + private final String monitorId; + private final String detectorName; + private final String findingsIndex; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1Rule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1Rule.java deleted file mode 100644 index ee90627cf0..0000000000 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1Rule.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.opensearch.dataprepper.plugins.processor.rules; - -import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; - -import java.util.List; -import java.util.function.Predicate; - -public class OpenSearchSigmaV1Rule extends SigmaV1Rule { - private final String monitorId; - private final String detectorName; - private final String findingsIndex; - - public OpenSearchSigmaV1Rule(final String monitorId, final String detectorName, final String findingsIndex, - final String title, final String id, final List tags, - final Predicate ruleCondition, final Predicate evaluationCondition) { - super(title, id, tags, ruleCondition, evaluationCondition); - this.monitorId = monitorId; - this.detectorName = detectorName; - this.findingsIndex = findingsIndex; - } - - public String getMonitorId() { - return monitorId; - } - - public String getDetectorName() { - return detectorName; - } - - public String getFindingsIndex() { - return findingsIndex; - } -} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatefulRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatefulRule.java new file mode 100644 index 0000000000..631926656b --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatefulRule.java @@ -0,0 +1,11 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Getter; +import lombok.experimental.SuperBuilder; + +@SuperBuilder +@Getter +public class OpenSearchSigmaV1StatefulRule extends StatefulRule { + private OpenSearchRuleMetadata openSearchRuleMetadata; + private SigmaV1RuleMetadata sigmaV1RuleMetadata; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatelessRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatelessRule.java new file mode 100644 index 0000000000..d0efcd29ba --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/OpenSearchSigmaV1StatelessRule.java @@ -0,0 +1,11 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Getter; +import lombok.experimental.SuperBuilder; + +@SuperBuilder +@Getter +public class OpenSearchSigmaV1StatelessRule extends StatelessRule { + private OpenSearchRuleMetadata openSearchRuleMetadata; + private SigmaV1RuleMetadata sigmaV1RuleMetadata; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java index 37bd18187b..8376082df1 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/Rule.java @@ -1,29 +1,24 @@ package org.opensearch.dataprepper.plugins.processor.rules; -import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; +import lombok.experimental.SuperBuilder; import java.util.function.Predicate; -public abstract class Rule { - private final Predicate ruleCondition; - private final Predicate evaluationCondition; +@SuperBuilder +public abstract class Rule { + private final String id; + private final Predicate evaluationCondition; + private final Predicate ruleCondition; - public Rule(final Predicate ruleCondition, final Predicate evaluationCondition) { - this.ruleCondition = ruleCondition; - this.evaluationCondition = evaluationCondition; + public boolean testEvaluationCondition(final T input) { + return evaluationCondition.test(input); } - // Helper for always evaluate rules - public Rule(final Predicate ruleCondition) { - this.ruleCondition = ruleCondition; - this.evaluationCondition = i -> true; + public boolean testRuleCondition(final U input) { + return ruleCondition.test(input); } - public Predicate getRuleCondition() { - return ruleCondition; - } - - public Predicate getEvaluationCondition() { - return evaluationCondition; + public String getId() { + return id; } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleRefresher.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleRefresher.java index 97df850add..95fd539644 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleRefresher.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleRefresher.java @@ -6,8 +6,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class RuleRefresher implements Runnable { private static final Logger LOG = LoggerFactory.getLogger(RuleRefresher.class); @@ -28,10 +28,11 @@ public RuleRefresher(final RuleProvider ruleProvider, public void run() { try { final List ruleData = ruleProvider.getRules(); - final List rules = ruleData.stream() - .map(ruleParser::parseRule) - .collect(Collectors.toList()); - sigmaRuleStore.updateRuleStore(rules); + final List statelessRules = new ArrayList<>(); + final List statefulRules = new ArrayList<>(); + + ruleData.forEach(ruleDatum -> ruleParser.parseRule(ruleDatum, statelessRules::add, statefulRules::add)); + sigmaRuleStore.updateRuleStore(statelessRules, statefulRules); } catch (final Exception e) { LOG.error("Caught exception refreshing rules", e); } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java index fe44f00480..f643a3d20f 100644 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/RuleStore.java @@ -5,17 +5,24 @@ // TODO - does this need locking? public class RuleStore { - private List rules; + private List statelessRules; + private List statefulRules; public RuleStore() { - this.rules = new ArrayList<>(); + this.statelessRules = new ArrayList<>(); + this.statefulRules = new ArrayList<>(); } - public void updateRuleStore(final List updatedRules) { - rules = updatedRules; + public void updateRuleStore(final List updatedStatelessRules, final List updatedStatefulRules) { + statelessRules = updatedStatelessRules; + statefulRules = updatedStatefulRules; } - public List getRules() { - return rules; + public List getRules() { + return statelessRules; + } + + public List getStatefulRules() { + return statefulRules; } } diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1Rule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1Rule.java deleted file mode 100644 index 9255d94371..0000000000 --- a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1Rule.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.opensearch.dataprepper.plugins.processor.rules; - -import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; - -import java.util.List; -import java.util.function.Predicate; - -public class SigmaV1Rule extends Rule { - private final String title; - private final String id; - private final List tags; - - public SigmaV1Rule(final String title, final String id, final List tags, final Predicate ruleCondition, - final Predicate evaluationCondition) { - super(ruleCondition, evaluationCondition); - this.title = title; - this.id = id; - this.tags = tags; - } - - public String getTitle() { - return title; - } - - public String getId() { - return id; - } - - public List getTags() { - return tags; - } -} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1RuleMetadata.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1RuleMetadata.java new file mode 100644 index 0000000000..e55ae7eeae --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/SigmaV1RuleMetadata.java @@ -0,0 +1,13 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Builder; +import lombok.Getter; + +import java.util.List; + +@Builder +@Getter +public class SigmaV1RuleMetadata { + private final String title; + private final List tags; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatefulRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatefulRule.java new file mode 100644 index 0000000000..29b096c29a --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatefulRule.java @@ -0,0 +1,15 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import org.opensearch.dataprepper.plugins.processor.model.matches.Match; + +import java.time.Duration; +import java.util.List; + +@SuperBuilder +@Getter +public class StatefulRule extends Rule> { + private List filterFields; + private Duration timeframe; +} diff --git a/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatelessRule.java b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatelessRule.java new file mode 100644 index 0000000000..dd1f3768f1 --- /dev/null +++ b/data-prepper-plugins/rule-engine/src/main/java/org/opensearch/dataprepper/plugins/processor/rules/StatelessRule.java @@ -0,0 +1,11 @@ +package org.opensearch.dataprepper.plugins.processor.rules; + +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import org.opensearch.dataprepper.plugins.processor.model.datatypes.DataType; + +@SuperBuilder +@Getter +public class StatelessRule extends Rule { + private final boolean isStatefulCondition; +} diff --git a/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java b/data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleConditionParserTest.java similarity index 100% rename from data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1SigmaV1RuleConditionParserTest.java rename to data-prepper-plugins/rule-engine/src/test/java/org/opensearch/dataprepper/plugins/processor/parser/SigmaV1RuleConditionParserTest.java diff --git a/data-prepper-plugins/s3-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/s3/S3SinkServiceIT.java b/data-prepper-plugins/s3-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/s3/S3SinkServiceIT.java index a5a80aa710..a583042d35 100644 --- a/data-prepper-plugins/s3-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/s3/S3SinkServiceIT.java +++ b/data-prepper-plugins/s3-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/s3/S3SinkServiceIT.java @@ -194,7 +194,7 @@ void verify_flushed_records_into_s3_bucketNewLine() throws JsonProcessingExcepti final Map actualDeserializedJson = objectMapperForDeserialization.readValue(actualLine, Map.class); final Map expectedMap = new HashMap<>(recordData.getData().toMap()); - expectedMap.put("Tag", new ArrayList<>(recordData.getData().getMetadata().getTags())); + expectedMap.put("Tag", new ArrayList<>(recordData.getData().getEventMetadata().getTags())); assertThat(actualDeserializedJson, equalTo(expectedMap)); final String expectedJsonString = recordData.getData().jsonBuilder().includeTags("Tag").toJsonString(); @@ -229,7 +229,7 @@ void verify_flushed_records_into_s3_bucketNewLine_with_compression() throws IOEx final Map actualDeserializedJson = objectMapperForDeserialization.readValue(actualLine, Map.class); final Map expectedMap = new HashMap<>(recordData.getData().toMap()); - expectedMap.put("Tag", new ArrayList<>(recordData.getData().getMetadata().getTags())); + expectedMap.put("Tag", new ArrayList<>(recordData.getData().getEventMetadata().getTags())); assertThat(actualDeserializedJson, equalTo(expectedMap)); final String expectedJsonString = recordData.getData().jsonBuilder().includeTags("Tag").toJsonString(); diff --git a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java index 220bb88287..d2a853fb64 100644 --- a/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java +++ b/data-prepper-plugins/user-agent-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessor.java @@ -61,7 +61,7 @@ public Collection> doExecute(final Collection> recor final List tagsOnParseFailure = config.getTagsOnParseFailure(); if (Objects.nonNull(tagsOnParseFailure) && tagsOnParseFailure.size() > 0) { - event.getMetadata().addTags(tagsOnParseFailure); + event.getEventMetadata().addTags(tagsOnParseFailure); } } } diff --git a/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java b/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java index da0923f509..b6995d0fec 100644 --- a/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java +++ b/data-prepper-plugins/user-agent-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/useragent/UserAgentProcessorTest.java @@ -133,8 +133,8 @@ public void testTagsAddedOnParseFailure() { final Event resultEvent = resultRecord.get(0).getData(); assertThat(resultEvent.containsKey("user_agent"), is(false)); - assertThat(resultEvent.getMetadata().getTags().contains(tagOnFailure1), is(true)); - assertThat(resultEvent.getMetadata().getTags().contains(tagOnFailure2), is(true)); + assertThat(resultEvent.getEventMetadata().getTags().contains(tagOnFailure1), is(true)); + assertThat(resultEvent.getEventMetadata().getTags().contains(tagOnFailure2), is(true)); } private UserAgentProcessor createObjectUnderTest() {