From 8bda4d1e7c17f900c26958957afd961a96679a1c Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Tue, 16 Jan 2024 14:42:14 -0800 Subject: [PATCH] support object fields in aggregation based sigma rules (#789) Signed-off-by: Subhobrata Dey --- .../aggregation/AggregationLexer.java | 88 ++--- src/main/grammars/Aggregation.g4 | 2 +- .../rules/backend/AggregationBuilders.java | 2 +- .../rules/backend/OSQueryBackend.java | 27 +- .../securityanalytics/TestHelpers.java | 373 ++++++++++++++++-- .../resthandler/DetectorMonitorRestApiIT.java | 239 ++++++++++- .../aggregation/AggregationBackendTests.java | 86 +++- 7 files changed, 714 insertions(+), 103 deletions(-) diff --git a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java index fcf7ae82a..115766bb1 100644 --- a/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java +++ b/src/main/generated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) { "\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+ "\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+ "\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+ - "\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+ - "\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+ - "\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+ - "\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+ - "\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+ - "\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+ - "\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+ - "\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+ - "\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+ - "\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+ - "&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+ - "\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+ - "\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+ - "\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+ - "\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+ - "\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+ - ">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+ - "=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+ - "\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+ - "\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+ - "\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+ - "u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+ - "\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+ - "\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+ - "\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+ - "\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+ - "\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+ - "C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+ - "F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+ - "\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+ - "-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+ - "\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+ - "PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+ - "\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+ - "\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+ - "\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+ - "\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+ - "\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+ - "\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+ - "\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+ - "\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+ - "\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+ - "\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+ - "\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; + "\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+ + "z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+ + "\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+ + "\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+ + "\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+ + "\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+ + "\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+ + "\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+ + "\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+ + "\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+ + "\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+ + "\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+ + "\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+ + "@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+ + "\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+ + "\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+ + "\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+ + "$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+ + "\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+ + "*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+ + "-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+ + "01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+ + "\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+ + "\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+ + "i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+ + "m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+ + "\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+ + "\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+ + "\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+ + "\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+ + "KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+ + "MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+ + "\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+ + "\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+ + "\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+ + "\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+ + "YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+ + "\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+ + "\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+ + "_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+ + "\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+ + "\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+ + "fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+ + "\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/src/main/grammars/Aggregation.g4 b/src/main/grammars/Aggregation.g4 index 00303b68a..be395c5ae 100644 --- a/src/main/grammars/Aggregation.g4 +++ b/src/main/grammars/Aggregation.g4 @@ -21,7 +21,7 @@ RPAREN : ')' ; DECIMAL : '-'?[0-9]+('.'[0-9]+)? ; -IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ; +IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ; WS : [ \r\t\u000C\n]+ -> skip ; comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java index 3927186fb..c0f6bbb7a 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat aggregationBuilder = new TermsAggregationBuilder(name).field(name); break; case "count": - aggregationBuilder = new ValueCountAggregationBuilder(name).field(name); + aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name); break; default: throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction)); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index 560bd47bf..814f32652 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -361,14 +361,15 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { BucketSelectorExtAggregationBuilder condition; String bucketTriggerSelectorId = UUIDs.base64UUID(); - if (aggregation.getAggFunction().equals("count")) { + if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) { String fieldName; - if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) { + if (aggregation.getGroupByField() == null) { fieldName = "_index"; fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index"); } else { - fieldName = aggregation.getGroupByField(); - fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField()); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fieldName = mappedGroupByField; + fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField); } aggBuilder.field(fieldName); fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_count", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()); @@ -376,17 +377,23 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) { Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold())); condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null); } else { - fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField()); - fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()); + /** + * removing dots to eliminate dots in aggregation names + */ + String mappedAggField = getFinalField(aggregation.getAggField()); + String mappedAggFieldUpdated = mappedAggField.replace(".", "_"); + String mappedGroupByField = getMappedField(aggregation.getGroupByField()); + fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField); + fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()); // Add subaggregation - AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField()); + AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField); if (subAgg != null) { - aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg); + aggBuilder.field(mappedGroupByField).subAggregation(subAgg); } - Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold())); - condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null); + Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold())); + condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null); } AggregationQueries aggregationQueries = new AggregationQueries(); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 7be4febff..843abbf3c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -86,7 +86,7 @@ public static Detector randomDetectorWithTriggers(List rules, List rules, List triggers, List inputIndices) { DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null); + return randomDetector(null, null, null, List.of(input), triggers, null, true, null, null); } public static Detector randomDetectorWithTriggersAndScheduleAndEnabled(List rules, List triggers, Schedule schedule, boolean enabled) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), @@ -168,44 +168,6 @@ public static CustomLogType randomCustomLogType(String name, String description, return new CustomLogType(null, null, name, description, category, source, null); } - public static String randomDocWithNullField() { - return "{\n" + - "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + - "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + - "\"Keywords\":\"9223372036854775808\",\n" + - "\"SeverityValue\":2,\n" + - "\"Severity\":\"INFO\",\n" + - "\"EventID\":22,\n" + - "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + - "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + - "\"Version\":5,\n" + - "\"TaskValue\":22,\n" + - "\"OpcodeValue\":0,\n" + - "\"RecordNumber\":null,\n" + - "\"ExecutionProcessID\":1996,\n" + - "\"ExecutionThreadID\":2616,\n" + - "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + - "\"Domain\":\"NTAUTHORITY\",\n" + - "\"AccountName\":\"SYSTEM\",\n" + - "\"UserID\":\"S-1-5-18\",\n" + - "\"AccountType\":\"User\",\n" + - "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + - "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + - "\"Opcode\":\"Info\",\n" + - "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + - "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + - "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + - "\"QueryResults\":\"172.31.46.38;\",\n" + - "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + - "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + - "\"SourceModuleName\":\"in\",\n" + - "\"SourceModuleType\":\"im_msvistalog\",\n" + - "\"CommandLine\": \"eachtest\",\n" + - "\"Initiated\": \"true\"\n" + - "}"; - } - public static Detector randomDetectorWithNoUser() { String name = OpenSearchRestTestCase.randomAlphaOfLength(10); String detectorType = randomDetectorType(); @@ -336,6 +298,29 @@ public static String randomRuleForMappingView(String field) { "level: high"; } + public static String randomCloudtrailRuleForCorrelations(String value) { + return "id: 5f92fff9-82e2-48ab-8fc1-8b133556a551\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: AWS User Created\n" + + "description: AWS User Created\n" + + "tags:\n" + + " - attack.test1\n" + + "falsepositives:\n" + + " - Legit User Account Administration\n" + + "level: high\n" + + "date: 2022/01/01\n" + + "status: experimental\n" + + "references:\n" + + " - 'https://github.com/RhinoSecurityLabs/AWS-IAM-Privilege-Escalation'\n" + + "author: toffeebr33k\n" + + "detection:\n" + + " condition: selection_source\n" + + " selection_source:\n" + + " EventName:\n" + + " - " + value; + } + public static String randomRuleForCustomLogType() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -879,6 +864,109 @@ public static String randomAggregationRule(String aggFunction, String signAndVa return String.format(Locale.ROOT, rule, opCode, aggFunction, signAndValue); } + public static String randomCloudtrailAggrRule() { + return "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " EventName:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by AccountName >= 2"; + } + + public static String randomCloudtrailAggrRuleWithDotFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String randomCloudtrailAggrRuleWithEcsFields() { + return "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(eventName) by awsRegion > 1\n" + + " selection1:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " eventSource:\n" + + " - lambda.amazonaws.com\n" + + " eventName: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078"; + } + + public static String cloudtrailOcsfMappings() { + return "\"properties\": {\n" + + " \"time\": {\n" + + " \"type\": \"date\"\n" + + " },\n" + + " \"cloud.region\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"api\": {\n" + + " \"properties\": {\n" + + " \"operation\": {\"type\": \"keyword\"},\n" + + " \"service\": {\n" + + " \"properties\": {\n" + + " \"name\": {\"type\": \"text\"}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } + public static String windowsIndexMapping() { return "\"properties\": {\n" + " \"@timestamp\": {\"type\":\"date\"},\n" + @@ -892,7 +980,10 @@ public static String windowsIndexMapping() { " \"type\": \"text\"\n" + " },\n" + " \"AccountName\": {\n" + - " \"type\": \"text\"\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"EventName\": {\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"AccountType\": {\n" + " \"type\": \"text\",\n" + @@ -1033,7 +1124,7 @@ public static String windowsIndexMapping() { " \"type\": \"date\"\n" + " },\n" + " \"EventType\": {\n" + - " \"type\": \"integer\"\n" + + " \"type\": \"keyword\"\n" + " },\n" + " \"ExecutionProcessID\": {\n" + " \"type\": \"long\"\n" + @@ -1544,6 +1635,43 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String randomDocWithNullField() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":null,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + @@ -1616,6 +1744,7 @@ public static String randomDoc() { "\"Severity\":\"INFO\",\n" + "\"EventID\":22,\n" + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"SourceIp\":\"1.2.3.4\",\n" + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + "\"Version\":5,\n" + "\"TaskValue\":22,\n" + @@ -1666,6 +1795,160 @@ public static String randomAdLdapDoc() { "}"; } + public static String randomCloudtrailOcsfDoc() { + return "{\n" + + " \"activity_id\": 8,\n" + + " \"activity_name\": \"Detach Policy\",\n" + + " \"actor\": {\n" + + " \"idp\": {\n" + + " \"name\": null\n" + + " },\n" + + " \"invoked_by\": null,\n" + + " \"session\": {\n" + + " \"created_time\": 1702510696000,\n" + + " \"issuer\": \"arn\",\n" + + " \"mfa\": false\n" + + " },\n" + + " \"user\": {\n" + + " \"account_uid\": \"\",\n" + + " \"credential_uid\": \"\",\n" + + " \"name\": null,\n" + + " \"type\": \"AssumedRole\",\n" + + " \"uid\": \"\",\n" + + " \"uuid\": \"\"\n" + + " }\n" + + " },\n" + + " \"api\": {\n" + + " \"operation\": \"CreateFunction\",\n" + + " \"request\": {\n" + + " \"uid\": \"0966237c-6279-43f4-a9d7-1eb416fca17d\"\n" + + " },\n" + + " \"response\": {\n" + + " \"error\": null,\n" + + " \"message\": null\n" + + " },\n" + + " \"service\": {\n" + + " \"name\": \"lambda.amazonaws.com\"\n" + + " },\n" + + " \"version\": null\n" + + " },\n" + + " \"category_name\": \"Audit Activity\",\n" + + " \"category_uid\": 3,\n" + + " \"class_name\": \"account_change\",\n" + + " \"class_uid\": 3001,\n" + + " \"cloud\": {\n" + + " \"provider\": \"AWS\",\n" + + " \"region\": \"us-east-1\"\n" + + " },\n" + + " \"dst_endpoint\": null,\n" + + " \"http_request\": {\n" + + " \"user_agent\": \"Boto3/1.26.90 Python/3.7.17 Linux/test.amzn2.x86_64 exec-env/AWS_Lambda_python3.7 Botocore/1.29.90\"\n" + + " },\n" + + " \"metadata\": {\n" + + " \"product\": {\n" + + " \"feature\": {\n" + + " \"name\": \"Management\"\n" + + " },\n" + + " \"name\": \"cloudtrail\",\n" + + " \"vendor_name\": \"AWS\",\n" + + " \"version\": \"1.08\"\n" + + " },\n" + + " \"profiles\": [\n" + + " \"cloud\"\n" + + " ],\n" + + " \"uid\": \"\",\n" + + " \"version\": \"1.0.0-rc.2\"\n" + + " },\n" + + " \"mfa\": null,\n" + + " \"resources\": null,\n" + + " \"severity\": \"Informational\",\n" + + " \"severity_id\": 1,\n" + + " \"src_endpoint\": {\n" + + " \"domain\": null,\n" + + " \"ip\": \"\",\n" + + " \"uid\": null\n" + + " },\n" + + " \"status\": \"Success\",\n" + + " \"status_id\": 1,\n" + + " \"time\": 1702952105000,\n" + + " \"type_name\": \"Account Change: Detach Policy\",\n" + + " \"type_uid\": 300108,\n" + + " \"unmapped\": {\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": \"true\",\n" + + " \"readOnly\": \"false\",\n" + + " \"recipientAccountId\": \"\",\n" + + " \"requestParameters.instanceProfileName\": \"\",\n" + + " \"tlsDetails.cipherSuite\": \"\",\n" + + " \"tlsDetails.clientProvidedHostHeader\": \"iam.amazonaws.com\",\n" + + " \"tlsDetails.tlsVersion\": \"TLSv1.2\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.accountId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.principalId\": \"\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.type\": \"Role\",\n" + + " \"userIdentity.sessionContext.sessionIssuer.userName\": \"\"\n" + + " },\n" + + " \"user\": {\n" + + " \"name\": \"\",\n" + + " \"uid\": null,\n" + + " \"uuid\": null\n" + + " }\n" + + "}"; + } + + public static String randomCloudtrailDoc(String user, String event) { + return "{\n" + + " \"eventVersion\": \"1.08\",\n" + + " \"userIdentity\": {\n" + + " \"type\": \"IAMUser\",\n" + + " \"principalId\": \"AIDA6ON6E4XEGITEXAMPLE\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Mary\",\n" + + " \"accountId\": \"888888888888\",\n" + + " \"accessKeyId\": \"AKIAIOSFODNN7EXAMPLE\",\n" + + " \"userName\": \"Mary\",\n" + + " \"sessionContext\": {\n" + + " \"sessionIssuer\": {},\n" + + " \"webIdFederationData\": {},\n" + + " \"attributes\": {\n" + + " \"creationDate\": \"2023-07-19T21:11:57Z\",\n" + + " \"mfaAuthenticated\": \"false\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"eventTime\": \"2023-07-19T21:25:09Z\",\n" + + " \"eventSource\": \"iam.amazonaws.com\",\n" + + " \"EventName\": \"" + event + "\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"sourceIPAddress\": \"192.0.2.0\",\n" + + " \"AccountName\": \"" + user + "\",\n" + + " \"userAgent\": \"aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user\",\n" + + " \"requestParameters\": {\n" + + " \"userName\": \"" + user + "\"\n" + + " },\n" + + " \"responseElements\": {\n" + + " \"user\": {\n" + + " \"path\": \"/\",\n" + + " \"arn\": \"arn:aws:iam::888888888888:user/Richard\",\n" + + " \"userId\": \"AIDA6ON6E4XEP7EXAMPLE\",\n" + + " \"createDate\": \"Jul 19, 2023 9:25:09 PM\",\n" + + " \"userName\": \"Richard\"\n" + + " }\n" + + " },\n" + + " \"requestID\": \"2d528c76-329e-410b-9516-EXAMPLE565dc\",\n" + + " \"eventID\": \"ba0801a1-87ec-4d26-be87-EXAMPLE75bbb\",\n" + + " \"readOnly\": false,\n" + + " \"eventType\": \"AwsApiCall\",\n" + + " \"managementEvent\": true,\n" + + " \"recipientAccountId\": \"888888888888\",\n" + + " \"eventCategory\": \"Management\",\n" + + " \"tlsDetails\": {\n" + + " \"tlsVersion\": \"TLSv1.2\",\n" + + " \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\",\n" + + " \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n" + + " },\n" + + " \"sessionCredentialFromConsole\": \"true\"\n" + + "}"; + } + public static String randomAppLogDoc() { return "{\n" + " \"endpoint\": \"/customer_records.txt\",\n" + @@ -1756,6 +2039,14 @@ public static String vpcFlowMappings() { " }"; } + private static String randomString() { + return OpenSearchTestCase.randomAlphaOfLengthBetween(2, 16); + } + + public static String randomLowerCaseString() { + return randomString().toLowerCase(Locale.ROOT); + } + public static XContentParser parser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 56ff61983..392909cfb 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.resthandler; import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -28,12 +30,20 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static java.util.Collections.emptyList; + +import static org.opensearch.securityanalytics.TestHelpers.cloudtrailOcsfMappings; import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRule; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithDotFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailAggrRuleWithEcsFields; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomCloudtrailOcsfDoc; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; @@ -51,7 +61,6 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMappingOnlyNumericAndText; import static org.opensearch.securityanalytics.TestHelpers.randomRuleWithDateKeywords; import static org.opensearch.securityanalytics.TestHelpers.randomDocOnlyNumericAndText; - import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase { @@ -1989,6 +1998,234 @@ public void testCreateDetectorWithKeywordsRule_ensureNoFindingsWithoutDateMappin assertEquals(0, noOfSigmaRuleMatches); } + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRule() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + indexDoc(index, "0", randomCloudtrailDoc("A12346", "CREATED")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomCloudtrailAggrRule(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailDoc("A12345", "CREATED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + indexDoc(index, "2", randomCloudtrailDoc("A12345", "DELETED")); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithDotFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithDotFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + + @SuppressWarnings("unchecked") + public void testCreateDetectorWithCloudtrailAggrRuleWithEcsFields() throws IOException { + String index = createTestIndex("cloudtrail", cloudtrailOcsfMappings()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{\n" + + " \"index_name\": \"" + index + "\",\n" + + " \"rule_topic\": \"cloudtrail\",\n" + + " \"partial\": true,\n" + + " \"alias_mappings\": {\n" + + " \"properties\": {\n" + + " \"aws.cloudtrail.event_name\": {\n" + + " \"path\": \"api.operation\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.event_source\": {\n" + + " \"path\": \"api.service.name\",\n" + + " \"type\": \"alias\"\n" + + " },\n" + + " \"aws.cloudtrail.aws_region\": {\n" + + " \"path\": \"cloud.region\",\n" + + " \"type\": \"alias\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + indexDoc(index, "0", randomCloudtrailOcsfDoc()); + + String rule = randomCloudtrailAggrRuleWithEcsFields(); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("cloudtrail detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + createdId = responseBody.get("_id").toString(); + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("rule_topic_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); + Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); + + String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); + Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String workflowId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("workflow_ids")).get(0); + + indexDoc(index, "1", randomCloudtrailOcsfDoc()); + indexDoc(index, "2", randomCloudtrailOcsfDoc()); + executeAlertingWorkflow(workflowId, Collections.emptyMap()); + + Map params = new HashMap<>(); + params.put("detector_id", createdId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + assertEquals(1, getFindingsBody.get("total_findings")); + } + private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); diff --git a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java index 395f15a79..71b855711 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/aggregation/AggregationBackendTests.java @@ -82,7 +82,7 @@ public void testCountAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -116,7 +116,7 @@ public void testSumAggregationWithGroupBy() throws IOException, SigmaError { // inputs.query.aggregations -> Query - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"sum\":{\"field\":\"fieldA\"}}}}}", aggQuery); // triggers.bucket_level_trigger.condition -> Condition Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -149,7 +149,7 @@ public void testMinAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"min\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -181,7 +181,7 @@ public void testMaxAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"max\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } @@ -213,7 +213,83 @@ public void testAvgAggregationWithGroupBy() throws IOException, SigmaError { String aggQuery = aggQueries.getAggQuery(); String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); - Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"fieldB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"mappedB\"},\"aggs\":{\"fieldA\":{\"avg\":{\"field\":\"fieldA\"}}}}}", aggQuery); Assert.assertEquals("{\"buckets_path\":{\"fieldA\":\"fieldA\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.fieldA > 110.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); } + + public void testCloudtrailAggregationRule() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: c64c5175-5189-431b-a55e-6d9882158250\n" + + "logsource:\n" + + " product: cloudtrail\n" + + "title: Accounts created and deleted within 24h\n" + + "description: Flag suspicious activity of accounts created and deleted within 24h\n" + + "date: 2021/09/23\n" + + "tags:\n" + + " - attack.exfiltration\n" + + "falsepositives: [ ]\n" + + "level: high\n" + + "status: test\n" + + "references: [ ]\n" + + "author: Sashank\n" + + "detection:\n" + + " selection:\n" + + " event:\n" + + " - CREATED\n" + + " - DELETED\n" + + " timeframe: 24h\n" + + " condition: selection | count(*) by accountid > 2", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("(event: \"CREATED\") OR (event: \"DELETED\")", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"accountid\"}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"_cnt\":\"_count\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params._cnt > 2.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } + + public void testCloudtrailAggregationRuleWithDotFields() throws IOException, SigmaError { + OSQueryBackend queryBackend = new OSQueryBackend(Map.of(), true, true); + List queries = queryBackend.convertRule(SigmaRule.fromYaml( + "id: 25b9c01c-350d-4c96-bed1-836d04a4f324\n" + + "title: test\n" + + "description: Detects when an user creates or invokes a lambda function.\n" + + "status: experimental\n" + + "author: deysubho\n" + + "date: 2023/12/07\n" + + "modified: 2023/12/07\n" + + "logsource:\n" + + " category: cloudtrail\n" + + "level: low\n" + + "detection:\n" + + " condition: selection1 or selection2 | count(api.operation) by cloud.region > 1\n" + + " selection1:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation:\n" + + " - CreateFunction\n" + + " selection2:\n" + + " api.service.name:\n" + + " - lambda.amazonaws.com\n" + + " api.operation: \n" + + " - Invoke\n" + + " timeframe: 20m\n" + + " tags:\n" + + " - attack.privilege_escalation\n" + + " - attack.t1078", true)); + + String query = queries.get(0).toString(); + Assert.assertEquals("((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"CreateFunction\")) OR ((api.service.name: \"lambda.amazonaws.com\") AND (api.operation: \"Invoke\"))", query); + + OSQueryBackend.AggregationQueries aggQueries = (OSQueryBackend.AggregationQueries) queries.get(1); + String aggQuery = aggQueries.getAggQuery(); + String bucketTriggerQuery = aggQueries.getBucketTriggerQuery(); + + Assert.assertEquals("{\"result_agg\":{\"terms\":{\"field\":\"cloud.region\"},\"aggs\":{\"api_operation\":{\"value_count\":{\"field\":\"api.operation\"}}}}}", aggQuery); + Assert.assertEquals("{\"buckets_path\":{\"api_operation\":\"api.operation\"},\"parent_bucket_path\":\"result_agg\",\"script\":{\"source\":\"params.api_operation > 1.0\",\"lang\":\"painless\"}}", bucketTriggerQuery); + } } \ No newline at end of file