Skip to content

Commit

Permalink
fix for mappings of custom log types & other bug fixes (opensearch-pr…
Browse files Browse the repository at this point in the history 
…oject#505)

Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 authored Aug 2, 2023
1 parent ef64f00 commit 745874d
Show file tree
Hide file tree
Showing 9 changed files with 405 additions and 150 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/securityanalytics/action/IndexCustomLogTypeRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class IndexCustomLogTypeRequest extends ActionRequest {

private CustomLogType customLogType;

private static final Pattern IS_VALID_CUSTOM_LOG_NAME = Pattern.compile("[a-zA-Z0-9 _,-.]{5,50}");
private static final Pattern IS_VALID_CUSTOM_LOG_NAME = Pattern.compile("[a-z0-9_-]{2,50}");

public IndexCustomLogTypeRequest(
String logTypeId,
Expand Down
20 changes: 11 additions & 9 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,17 +343,19 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
// Insert new fieldMappings
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( newFieldMapping -> {
Optional<FieldMappingDoc> foundFieldMappingDoc = existingFieldMappings
.stream()
.filter(
e -> e.getRawField().equals(newFieldMapping.getRawField()) && (
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
for (FieldMappingDoc e: existingFieldMappings) {
if (e.getRawField().equals(newFieldMapping.getRawField())) {
if ((
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
) || (
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
)
)
.findFirst();
)) {
foundFieldMappingDoc = Optional.of(e);
}
}
}
if (foundFieldMappingDoc.isEmpty()) {
newFieldMapping.setIsDirty(true);
newFieldMappings.add(newFieldMapping);
Expand Down
18 changes: 13 additions & 5 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,15 +485,23 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
String rawPath = requiredField.getRawField();
String ocsfPath = requiredField.getOcsf();
if (allFieldsFromIndex.contains(rawPath)) {
// Maintain list of found paths in index
applyableAliases.add(alias);
if (alias != null) {
// Maintain list of found paths in index
applyableAliases.add(alias);
} else {
applyableAliases.add(rawPath);
}
pathsOfApplyableAliases.add(rawPath);
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
} else if (allFieldsFromIndex.contains(alias) == false) {
// we don't want to send back aliases which have same name as existing field in index
unmappedFieldAliases.add(alias);
} else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) {
if (alias != null) {
// we don't want to send back aliases which have same name as existing field in index
unmappedFieldAliases.add(alias);
} else {
unmappedFieldAliases.add(rawPath);
}
}
}

Expand Down
122 changes: 74 additions & 48 deletions ...n/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
import org.opensearch.securityanalytics.util.DetectorIndices;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.tasks.Task;
import org.opensearch.threadpool.ThreadPool;
Expand All @@ -62,6 +63,8 @@ public class TransportDeleteCustomLogTypeAction extends HandledTransportAction<D

private final Settings settings;

private final DetectorIndices detectorIndices;

private final CustomLogTypeIndices customLogTypeIndices;

private volatile Boolean filterByEnabled;
Expand All @@ -73,6 +76,7 @@ public TransportDeleteCustomLogTypeAction(TransportService transportService,
Client client,
ActionFilters actionFilters,
ClusterService clusterService,
DetectorIndices detectorIndices,
CustomLogTypeIndices customLogTypeIndices,
Settings settings,
ThreadPool threadPool) {
Expand All @@ -81,6 +85,7 @@ public TransportDeleteCustomLogTypeAction(TransportService transportService,
this.clusterService = clusterService;
this.threadPool = threadPool;
this.settings = settings;
this.detectorIndices = detectorIndices;
this.customLogTypeIndices = customLogTypeIndices;
this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
Expand Down Expand Up @@ -164,64 +169,85 @@ private void onGetResponse(CustomLogType logType) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because source is sigma", logType.getId()), RestStatus.BAD_REQUEST));
}

searchDetectors(logType.getName(), new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
return;
}
if (detectorIndices.detectorIndexExists()) {
searchDetectors(logType.getName(), new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
return;
}

if (response.getHits().getTotalHits().value > 0) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active detectors exist", logType.getId()), RestStatus.BAD_REQUEST));
return;
}
if (response.getHits().getTotalHits().value > 0) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active detectors exist", logType.getId()), RestStatus.BAD_REQUEST));
return;
}

searchRules(logType.getName(), new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
return;
}
searchRules(logType.getName(), new ActionListener<>() {
@Override
public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
return;
}

if (response.getHits().getTotalHits().value > 0) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST));
return;
}
if (response.getHits().getTotalHits().value > 0) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST));
return;
}

DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
.setRefreshPolicy(request.getRefreshPolicy())
.timeout(indexTimeout);
DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
.setRefreshPolicy(request.getRefreshPolicy())
.timeout(indexTimeout);

client.delete(deleteRequest, new ActionListener<>() {
@Override
public void onResponse(DeleteResponse response) {
if (response.status() != RestStatus.OK) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
}
onOperation(response);
}

client.delete(deleteRequest, new ActionListener<>() {
@Override
public void onResponse(DeleteResponse response) {
if (response.status() != RestStatus.OK) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
@Override
public void onFailure(Exception e) {
onFailures(e);
}
onOperation(response);
}
});
}

@Override
public void onFailure(Exception e) {
onFailures(e);
}
});
}
@Override
public void onFailure(Exception e) {
onFailures(e);
}
});
}

@Override
public void onFailure(Exception e) {
onFailures(e);
@Override
public void onFailure(Exception e) {
onFailures(e);
}
});
} else {
DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
.setRefreshPolicy(request.getRefreshPolicy())
.timeout(indexTimeout);

client.delete(deleteRequest, new ActionListener<>() {
@Override
public void onResponse(DeleteResponse response) {
if (response.status() != RestStatus.OK) {
onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
}
});
}
onOperation(response);
}

@Override
public void onFailure(Exception e) {
onFailures(e);
}
});
@Override
public void onFailure(Exception e) {
onFailures(e);
}
});
}
}

private void searchDetectors(String logTypeName, ActionListener<SearchResponse> listener) {
Expand Down
Loading

0 comments on commit 745874d

Please sign in to comment.