From 2d92f6c2de702c9bcdb19b34baf6a936af809ce8 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Fri, 13 Oct 2023 08:47:56 +0200 Subject: [PATCH] IM rule use fields caps (#168447) ## IM rule use fields caps instead of field mappign API Serverless don't support `getFieldMapping` API, so we use `fieldCaps` API to check if the field are support term query. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...get_allowed_fields_for_terms_query.test.ts | 115 +++++++++--------- .../get_allowed_fields_for_terms_query.ts | 49 ++++---- 2 files changed, 80 insertions(+), 84 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.test.ts index deb182e6860dd..8ab76fe55bdf9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { IndicesGetFieldMappingResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; import { alertsMock } from '@kbn/alerting-plugin/server/mocks'; import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; @@ -14,52 +14,40 @@ import { getAllowedFieldsForTermQuery, } from './get_allowed_fields_for_terms_query'; -const indexMapping = { - 'source-index': { - mappings: { - 'host.name': { - full_name: 'host.name', - mapping: { - name: { - type: 'keyword', - }, - }, - }, - 'url.full': { - full_name: 'url.full', - mapping: { - full: { - type: 'keyword', - }, - }, +const fieldsCapsResponse: FieldCapsResponse = { + indices: ['source-index', 'other-source-index'], + fields: { + 'url.full': { + keyword: { + type: 'keyword', + metadata_field: false, + searchable: true, + aggregatable: true, }, - 'source.range': { - full_name: 'source.range', - mapping: { - range: { - type: 'ip_range', - }, - }, + }, + 'host.name': { + keyword: { + type: 'keyword', + metadata_field: false, + searchable: true, + aggregatable: true, }, }, - }, - 'other-source-index': { - mappings: { - 'host.name': { - full_name: 'host.name', - mapping: { - name: { - type: 'keyword', - }, - }, + 'host.ip': { + ip: { + type: 'ip', + metadata_field: false, + searchable: true, + aggregatable: true, }, - 'host.ip': { - full_name: 'host.ip', - mapping: { - name: { - type: 'ip', - }, - }, + }, + 'source.range': { + ip_range: { + type: 'ip_range', + metadata_field: false, + searchable: true, + aggregatable: true, + indices: ['source-index'], }, }, }, @@ -68,9 +56,12 @@ const indexMapping = { describe('get_allowed_fields_for_terms_query copy', () => { describe('getAllowedFieldForTermQueryFromMapping', () => { it('should return map of fields allowed for term query', () => { - const result = getAllowedFieldForTermQueryFromMapping( - indexMapping as IndicesGetFieldMappingResponse - ); + const result = getAllowedFieldForTermQueryFromMapping(fieldsCapsResponse, [ + 'host.ip', + 'url.full', + 'host.name', + 'source.range', + ]); expect(result).toEqual({ 'host.ip': true, 'url.full': true, @@ -78,21 +69,25 @@ describe('get_allowed_fields_for_terms_query copy', () => { }); }); it('should disable fields if in one index type not supported', () => { - const result = getAllowedFieldForTermQueryFromMapping({ - 'new-source-index': { - mappings: { + const result = getAllowedFieldForTermQueryFromMapping( + { + ...fieldsCapsResponse, + fields: { + ...fieldsCapsResponse.fields, 'host.name': { - full_name: 'host.name', - mapping: { - name: { - type: 'text', - }, + ...fieldsCapsResponse.fields['host.name'], + text: { + type: 'text', + metadata_field: false, + searchable: true, + aggregatable: true, + indices: ['new-source-index'], }, }, }, }, - ...indexMapping, - } as IndicesGetFieldMappingResponse); + ['host.ip', 'url.full', 'host.name', 'source.range'] + ); expect(result).toEqual({ 'host.ip': true, 'url.full': true, @@ -106,16 +101,16 @@ describe('get_allowed_fields_for_terms_query copy', () => { beforeEach(() => { alertServices = alertsMock.createRuleExecutorServices(); - alertServices.scopedClusterClient.asCurrentUser.indices.getFieldMapping.mockResolvedValue( - indexMapping as IndicesGetFieldMappingResponse + alertServices.scopedClusterClient.asCurrentUser.fieldCaps.mockResolvedValue( + fieldsCapsResponse ); ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); }); it('should return map of fields allowed for term query for source and threat indices', async () => { const threatMatchedFields = { - source: ['host.name', 'url.full'], - threat: ['host.name', 'url.full'], + source: ['host.name', 'url.full', 'host.ip'], + threat: ['host.name', 'url.full', 'host.ip'], }; const threatIndex = ['threat-index']; const inputIndex = ['source-index']; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts index bde234e2bdc20..4e1c50c72745c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_allowed_fields_for_terms_query.ts @@ -5,39 +5,34 @@ * 2.0. */ -import type { IndicesGetFieldMappingResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; import type { AllowedFieldsForTermsQuery, GetAllowedFieldsForTermQuery } from './types'; -const allowedFieldTypes = ['keyword', 'constant_keyword', 'wildcard', 'ip']; +const allowedFieldTypesSet = new Set(['keyword', 'constant_keyword', 'wildcard', 'ip']); /* * Return map of fields allowed for term query */ export const getAllowedFieldForTermQueryFromMapping = ( - indexMapping: IndicesGetFieldMappingResponse + fieldsCapsResponse: FieldCapsResponse, + fields: string[] ): Record => { - const result: Record = {}; - const notAllowedFields: string[] = []; + const fieldsCaps = fieldsCapsResponse.fields; - const indices = Object.values(indexMapping); - indices.forEach((index) => { - Object.entries(index.mappings).forEach(([field, fieldValue]) => { - Object.values(fieldValue.mapping).forEach((mapping) => { - const fieldType = mapping?.type; - if (!fieldType) return; + const availableFields = fields.filter((field) => { + const fieldCaps = fieldsCaps[field]; - if (allowedFieldTypes.includes(fieldType) && !notAllowedFields.includes(field)) { - result[field] = true; - } else { - notAllowedFields.push(field); - // if we the field allowed in one index, but not allowed in another, we should delete it from result - delete result[field]; - } - }); + const isAllVariationsAllowed = Object.values(fieldCaps).every((fieldCapByType) => { + return allowedFieldTypesSet.has(fieldCapByType.type); }); + + return isAllVariationsAllowed; }); - return result; + return availableFields.reduce>((acc, field) => { + acc[field] = true; + return acc; + }, {}); }; /** @@ -53,19 +48,25 @@ export const getAllowedFieldsForTermQuery = async ({ let allowedFieldsForTermsQuery = { source: {}, threat: {} }; try { const [sourceFieldsMapping, threatFieldsMapping] = await Promise.all([ - services.scopedClusterClient.asCurrentUser.indices.getFieldMapping({ + services.scopedClusterClient.asCurrentUser.fieldCaps({ index: inputIndex, fields: threatMatchedFields.source, }), - services.scopedClusterClient.asCurrentUser.indices.getFieldMapping({ + services.scopedClusterClient.asCurrentUser.fieldCaps({ index: threatIndex, fields: threatMatchedFields.threat, }), ]); allowedFieldsForTermsQuery = { - source: getAllowedFieldForTermQueryFromMapping(sourceFieldsMapping), - threat: getAllowedFieldForTermQueryFromMapping(threatFieldsMapping), + source: getAllowedFieldForTermQueryFromMapping( + sourceFieldsMapping, + threatMatchedFields.source + ), + threat: getAllowedFieldForTermQueryFromMapping( + threatFieldsMapping, + threatMatchedFields.threat + ), }; } catch (e) { ruleExecutionLogger.debug(`Can't get allowed fields for terms query: ${e}`);