diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java index b0e368151..2f4d2eb93 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java @@ -30,7 +30,7 @@ public class IndexCustomLogTypeRequest extends ActionRequest { private CustomLogType customLogType; - private static final Pattern IS_VALID_CUSTOM_LOG_NAME = Pattern.compile("[a-zA-Z0-9 _,-.]{5,50}"); + private static final Pattern IS_VALID_CUSTOM_LOG_NAME = Pattern.compile("[a-z0-9_-]{2,50}"); public IndexCustomLogTypeRequest( String logTypeId, diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index 0c49abc9d..b54e4eb82 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -343,17 +343,19 @@ private List mergeFieldMappings(List existingF // Insert new fieldMappings List newFieldMappings = new ArrayList<>(); fieldMappingDocs.forEach( newFieldMapping -> { - Optional foundFieldMappingDoc = existingFieldMappings - .stream() - .filter( - e -> e.getRawField().equals(newFieldMapping.getRawField()) && ( + Optional foundFieldMappingDoc = Optional.empty(); + for (FieldMappingDoc e: existingFieldMappings) { + if (e.getRawField().equals(newFieldMapping.getRawField())) { + if (( e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null && - e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) - ) || ( + e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField)) + ) || ( e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null - ) - ) - .findFirst(); + )) { + foundFieldMappingDoc = Optional.of(e); + } + } + } if (foundFieldMappingDoc.isEmpty()) { newFieldMapping.setIsDirty(true); newFieldMappings.add(newFieldMapping); diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index b21f34f64..059150e6a 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -485,15 +485,23 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { String rawPath = requiredField.getRawField(); String ocsfPath = requiredField.getOcsf(); if (allFieldsFromIndex.contains(rawPath)) { - // Maintain list of found paths in index - applyableAliases.add(alias); + if (alias != null) { + // Maintain list of found paths in index + applyableAliases.add(alias); + } else { + applyableAliases.add(rawPath); + } pathsOfApplyableAliases.add(rawPath); } else if (allFieldsFromIndex.contains(ocsfPath)) { applyableAliases.add(alias); pathsOfApplyableAliases.add(ocsfPath); - } else if (allFieldsFromIndex.contains(alias) == false) { - // we don't want to send back aliases which have same name as existing field in index - unmappedFieldAliases.add(alias); + } else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) { + if (alias != null) { + // we don't want to send back aliases which have same name as existing field in index + unmappedFieldAliases.add(alias); + } else { + unmappedFieldAliases.add(rawPath); + } } } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java index f375e84ba..fb190e880 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java @@ -39,6 +39,7 @@ import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; +import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -62,6 +63,8 @@ public class TransportDeleteCustomLogTypeAction extends HandledTransportAction() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); - return; - } + if (detectorIndices.detectorIndexExists()) { + searchDetectors(logType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active detectors exist", logType.getId()), RestStatus.BAD_REQUEST)); - return; - } + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active detectors exist", logType.getId()), RestStatus.BAD_REQUEST)); + return; + } - searchRules(logType.getName(), new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); - return; - } + searchRules(logType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST)); - return; - } + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST)); + return; + } - DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) - .setRefreshPolicy(request.getRefreshPolicy()) - .timeout(indexTimeout); + DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) + .setRefreshPolicy(request.getRefreshPolicy()) + .timeout(indexTimeout); + + client.delete(deleteRequest, new ActionListener<>() { + @Override + public void onResponse(DeleteResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + } + onOperation(response); + } - client.delete(deleteRequest, new ActionListener<>() { - @Override - public void onResponse(DeleteResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + @Override + public void onFailure(Exception e) { + onFailures(e); } - onOperation(response); - } + }); + } - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } - @Override - public void onFailure(Exception e) { - onFailures(e); + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) + .setRefreshPolicy(request.getRefreshPolicy()) + .timeout(indexTimeout); + + client.delete(deleteRequest, new ActionListener<>() { + @Override + public void onResponse(DeleteResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); } - }); - } + onOperation(response); + } - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } } private void searchDetectors(String logTypeName, ActionListener listener) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java index 70c576963..70e9b26f4 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java @@ -28,6 +28,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.authuser.User; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.rest.RestRequest; @@ -45,6 +46,7 @@ import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; +import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; @@ -68,6 +70,8 @@ public class TransportIndexCustomLogTypeAction extends HandledTransportAction() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); - return; - } - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active detectors exist", logTypeId), RestStatus.BAD_REQUEST)); - return; - } + if (detectorIndices.detectorIndexExists()) { + searchDetectors(existingLogType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } - searchRules(existingLogType.getName(), new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); - return; - } + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active detectors exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active rules exist", logTypeId), RestStatus.BAD_REQUEST)); - return; - } + searchRules(existingLogType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active rules exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } + + try { + request.getCustomLogType().setTags(existingLogType.getTags()); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(request.getLogTypeId()) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + onOperation(response, request.getCustomLogType()); + } - try { - request.getCustomLogType().setTags(existingLogType.getTags()); - IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) - .setRefreshPolicy(request.getRefreshPolicy()) - .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(request.getLogTypeId()) - .timeout(indexTimeout); - - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + @Override + public void onFailure(Exception e) { + onFailures(e); } - onOperation(response, request.getCustomLogType()); - } - - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } catch (IOException e) { + }); + } catch (IOException e) { + onFailures(e); + } + } + + @Override + public void onFailure(Exception e) { onFailures(e); } - } + }); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } else { + request.getCustomLogType().setTags(existingLogType.getTags()); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(request.getLogTypeId()) + .timeout(indexTimeout); - @Override - public void onFailure(Exception e) { - onFailures(e); + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); } - }); - } - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); + request.getCustomLogType().setId(response.getId()); + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } } else { request.getCustomLogType().setTags(existingLogType.getTags()); IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) @@ -332,12 +365,12 @@ public void onFailure(Exception e) { } }); } else { - logTypeService.ensureConfigIndexIsInitialized(new ActionListener() { + logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() { @Override public void onResponse(Void unused) { - MaxAggregationBuilder queryBuilder = AggregationBuilders.max("agg").field("tags.correlation_id"); + MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("name", request.getCustomLogType().getName()); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.aggregation(queryBuilder); + searchSourceBuilder.query(queryBuilder); SearchRequest searchRequest = new SearchRequest(); searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); searchRequest.source(searchSourceBuilder); @@ -350,33 +383,60 @@ public void onResponse(SearchResponse response) { return; } - try { - Max agg = response.getAggregations().get("agg"); - int value = Double.valueOf(agg.getValue()).intValue(); - request.getCustomLogType().setTags(Map.of("correlation_id", value+1)); - IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) - .setRefreshPolicy(request.getRefreshPolicy()) - .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .timeout(indexTimeout); - - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse response) { - if (response.status() != RestStatus.CREATED) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); - } - request.getCustomLogType().setId(response.getId()); - onOperation(response, request.getCustomLogType()); + long noOfHits = response.getHits().getTotalHits().value; + if (noOfHits > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.ROOT, "Log Type with name %s already exists", request.getCustomLogType().getName()), RestStatus.INTERNAL_SERVER_ERROR)); + return; + } + MaxAggregationBuilder queryBuilder = AggregationBuilders.max("agg").field("tags.correlation_id"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.aggregation(queryBuilder); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(LogTypeService.LOG_TYPE_INDEX); + searchRequest.source(searchSourceBuilder); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(response.toString(), RestStatus.INTERNAL_SERVER_ERROR)); + return; } - @Override - public void onFailure(Exception e) { - onFailures(e); + try { + Max agg = response.getAggregations().get("agg"); + int value = Double.valueOf(agg.getValue()).intValue(); + request.getCustomLogType().setTags(Map.of("correlation_id", value + 1)); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.CREATED) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + request.getCustomLogType().setId(response.getId()); + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } catch (IOException ex) { + onFailures(ex); } - }); - } catch (IOException ex) { - onFailures(ex); - } + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java index 09a488175..e42266103 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListCorrelationAction.java @@ -34,6 +34,7 @@ import org.opensearch.transport.TransportService; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -119,7 +120,7 @@ public void onResponse(SearchResponse response) { onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); } - List correlatedFindings = new ArrayList<>(); + Map correlatedFindings = new HashMap<>(); Iterator hits = response.getHits().iterator(); while (hits.hasNext()) { SearchHit hit = hits.next(); @@ -131,9 +132,9 @@ public void onResponse(SearchResponse response) { source.get("finding2").toString(), source.get("logType").toString().split("-")[1], (List) source.get("corrRules")); - correlatedFindings.add(correlatedFinding); + correlatedFindings.put(source.get("finding1").toString() + ":" + source.get("finding2").toString(), correlatedFinding); } - onOperation(new ListCorrelationsResponse(correlatedFindings)); + onOperation(new ListCorrelationsResponse(new ArrayList<>(correlatedFindings.values()))); } @Override diff --git a/src/main/resources/mappings/log_type_config_mapping.json b/src/main/resources/mappings/log_type_config_mapping.json index cc5910d19..16c84f810 100644 --- a/src/main/resources/mappings/log_type_config_mapping.json +++ b/src/main/resources/mappings/log_type_config_mapping.json @@ -33,6 +33,7 @@ }, "name": { "type": "text", + "analyzer": "whitespace", "fields": { "keyword": { "type": "keyword", diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 20605d8f8..db857eb9c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -209,6 +209,36 @@ public static String randomRule() { "level: high"; } + public static String randomRuleForCustomLogType() { + return "title: Remote Encrypting File System Abuse\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 22\n" + + " Author: 'Hello'\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomRuleWithAlias() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java index 1bc37485d..d4f412589 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java @@ -25,6 +25,7 @@ import org.opensearch.securityanalytics.model.DetectorRule; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -214,6 +215,24 @@ public void testEditACustomLogTypeDescription() throws IOException { Assert.assertEquals(correlationId, Integer.parseInt(((Map)(((Map) responseBody.get("logType")).get("tags"))).get("correlation_id").toString())); } + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeWithoutDetectors() throws IOException { + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated_name", null, "Custom"); + Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); + Assert.assertEquals("Update custom log type failed", RestStatus.OK, restStatus(updatedResponse)); + + responseBody = asMap(updatedResponse); + Assert.assertEquals(updatedCustomLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + } + @SuppressWarnings("unchecked") public void testEditACustomLogTypeNameFailsAsDetectorExist() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -368,7 +387,7 @@ public void testEditACustomLogTypeName() throws IOException, InterruptedExceptio Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); Thread.sleep(5000); - CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated name", null, "Custom"); + CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated_name", null, "Custom"); Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); responseBody = asMap(updatedResponse); Assert.assertEquals(updatedCustomLogType.getName(), ((Map) responseBody.get("logType")).get("name")); @@ -511,6 +530,20 @@ public void testDeleteCustomLogTypeFailsAsRulesExist() throws IOException { }); } + @SuppressWarnings("unchecked") + public void testDeleteCustomLogTypeWithoutDetectors() throws IOException { + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + Assert.assertEquals(200, deleteResponse.getStatusLine().getStatusCode()); + } + @SuppressWarnings("unchecked") public void testDeleteCustomLogType() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -621,7 +654,7 @@ public void testCreateMultipleLogTypes() throws IOException { String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - customLogType = TestHelpers.randomCustomLogType("custom-log-type-again", null, "Custom"); + customLogType = TestHelpers.randomCustomLogType("custom-again", null, "Custom"); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); @@ -688,4 +721,98 @@ public void testCreateMultipleLogTypes() throws IOException { noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); } + + public void testGetMappingsView() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + String rule = randomRuleForCustomLogType(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", customLogType.getName()); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + Assert.assertTrue(unmappedFieldAliases.contains("Author")); + Assert.assertFalse(unmappedIndexFields.contains("EventID")); + } + + public void testMultipleLogTypesUpdateFieldMappings() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + CustomLogType customLogType = TestHelpers.randomCustomLogType("logtype1", null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + String rule = randomRuleForCustomLogType(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + customLogType = TestHelpers.randomCustomLogType("logtype2", null, "Custom"); + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + rule = randomRuleForCustomLogType(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + // Execute GetMappingsViewAction to add alias mapping for index + Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "logtype1"); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + Map respMap = responseAsMap(response); + // Verify alias mappings + List unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + List unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + Assert.assertTrue(unmappedFieldAliases.contains("Author")); + Assert.assertFalse(unmappedIndexFields.contains("EventID")); + + // Execute GetMappingsViewAction to add alias mapping for index + request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI); + // both req params and req body are supported + request.addParameter("index_name", index); + request.addParameter("rule_topic", "logtype2"); + response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + respMap = responseAsMap(response); + // Verify alias mappings + unmappedFieldAliases = (List) respMap.get("unmapped_field_aliases"); + unmappedIndexFields = (List) respMap.get("unmapped_index_fields"); + Assert.assertTrue(unmappedFieldAliases.contains("Author")); + Assert.assertFalse(unmappedIndexFields.contains("EventID")); + } + + public void testLogTypeNamesAlwaysLowercase() throws IOException { + CustomLogType customLogType = TestHelpers.randomCustomLogType("Logtype1", null, "Custom"); + expectThrows(ResponseException.class, () -> { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + }); + } + + public void testMultipleLogTypesCannotBeCreatedWithSameName() throws IOException { + CustomLogType customLogType = TestHelpers.randomCustomLogType("logtype", null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + expectThrows(ResponseException.class, () -> { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + }); + } } \ No newline at end of file