diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts b/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts index d4ae7a0f52b26..83c55cc29178c 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts @@ -29,6 +29,30 @@ export const buildLargeDocument = ({ return doc; }; +export const buildLargeNestedDocument = ({ + fieldsPerObject, + levels, + fieldSize, +}: { + fieldsPerObject: number; + levels: number; + fieldSize: number; +}): Record => { + if (levels === 1) { + return buildLargeDocument({ numFields: fieldsPerObject, fieldSize }); + } else { + const doc: Record = {}; + range(fieldsPerObject).forEach((idx) => { + doc[`level_${levels}_field${idx}`] = buildLargeNestedDocument({ + fieldsPerObject, + levels: levels - 1, + fieldSize, + }); + }); + return doc; + } +}; + export const addTimestampToDoc = ({ timestamp = new Date(), doc, diff --git a/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts b/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts index 4448d0cee6897..d061d86fe37dd 100644 --- a/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts +++ b/x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts @@ -14,7 +14,7 @@ import type { import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; -export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap); +export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap, false); export interface GenerateLargeMappingPropertiesProps { size: number; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 2d3a4cd40b89e..1e5e70a37ae5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isEmpty } from 'lodash'; +import { isEmpty, partition } from 'lodash'; import agent from 'elastic-apm-node'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -375,8 +375,17 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = ); const legacySignalFields: string[] = Object.keys(aadFieldConversion); + const [ignoreFieldsRegexes, ignoreFieldsStandard] = partition( + [...ignoreFields, ...legacySignalFields], + (field: string) => field.startsWith('/') && field.endsWith('/') + ); + const ignoreFieldsObject: Record = {}; + ignoreFieldsStandard.forEach((field) => { + ignoreFieldsObject[field] = true; + }); const wrapHits = wrapHitsFactory({ - ignoreFields: [...ignoreFields, ...legacySignalFields], + ignoreFields: ignoreFieldsObject, + ignoreFieldsRegexes, mergeStrategy, completeRule, spaceId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 2675c3996e865..d865ae6232005 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -12,8 +12,8 @@ import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_pat import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants'; import type { ConfigType } from '../../../../config'; import type { Ancestor, SignalSource, SignalSourceHit } from '../types'; -import { buildAlert, buildAncestors, generateAlertId } from '../factories/utils/build_alert'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { buildAlertFields, buildAncestors, generateAlertId } from '../factories/utils/build_alert'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { EqlSequence } from '../../../../../common/detection_engine/types'; import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids'; import type { BuildReasonMessage } from '../utils/reason_formatters'; @@ -59,20 +59,21 @@ export const buildAlertGroupFromSequence = ( let baseAlerts: BaseFieldsLatest[] = []; try { baseAlerts = sequence.events.map((event) => - buildBulkBody( + transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - false, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: false, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - 'placeholder-alert-uuid', // This is overriden below - publicBaseUrl - ) + alertUuid: 'placeholder-alert-uuid', // This is overriden below + publicBaseUrl, + }) ); } catch (error) { ruleExecutionLogger.error(error); @@ -153,16 +154,16 @@ export const buildAlertRoot = ( severity: completeRule.ruleParams.severity, mergedDoc: mergedAlerts as SignalSourceHit, }); - const doc = buildAlert( - wrappedBuildingBlocks, + const doc = buildAlertFields({ + docs: wrappedBuildingBlocks, completeRule, spaceId, reason, indicesToQuery, - 'placeholder-uuid', // These will be overriden below + alertUuid: 'placeholder-uuid', // These will be overriden below publicBaseUrl, // Not necessary now, but when the ID is created ahead of time this can be passed - alertTimestampOverride - ); + alertTimestampOverride, + }); const alertId = generateAlertId(doc); const alertUrl = getAlertDetailsUrl({ alertId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts index b0fa2fd6638fa..762f3cff4ce45 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts @@ -16,7 +16,7 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { SignalSource } from '../types'; import { generateAlertId } from './utils'; @@ -55,20 +55,21 @@ export const wrapEsqlAlerts = ({ index: i, }); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, - [], + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, + indicesToQuery: [], alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts index 7b180d1adfa62..2a6e9a42acf28 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -19,7 +19,7 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { SignalSource } from '../types'; import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; import { generateAlertId } from './utils'; @@ -73,20 +73,21 @@ export const wrapSuppressedEsqlAlerts = ({ const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, - [], + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, + indicesToQuery: [], alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap new file mode 100644 index 0000000000000..17d8ceb0c50b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/__snapshots__/build_alert.test.ts.snap @@ -0,0 +1,223 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`buildAlertFields it creates the expected alert fields 1`] = ` +Object { + "@timestamp": "2020-01-01T00:00:00.000Z", + "event.kind": "signal", + "host.asset.criticality": undefined, + "host.risk.calculated_level": undefined, + "host.risk.calculated_score_norm": undefined, + "kibana.alert.ancestors": Array [ + Object { + "depth": 0, + "id": "d5e8eb51-a6a0-456d-8a15-4b79bfec3d71", + "index": "myFakeSignalIndex", + "rule": undefined, + "type": "event", + }, + ], + "kibana.alert.building_block_type": "default", + "kibana.alert.depth": 1, + "kibana.alert.host.criticality_level": undefined, + "kibana.alert.original_time": "2020-04-20T21:27:45.000Z", + "kibana.alert.reason": "test reason", + "kibana.alert.risk_score": 50, + "kibana.alert.rule.actions": Array [], + "kibana.alert.rule.author": Array [ + "Elastic", + ], + "kibana.alert.rule.building_block_type": "default", + "kibana.alert.rule.consumer": "siem", + "kibana.alert.rule.created_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.created_by": "sample user", + "kibana.alert.rule.description": "Detecting root and admin users", + "kibana.alert.rule.enabled": true, + "kibana.alert.rule.exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "kibana.alert.rule.false_positives": Array [], + "kibana.alert.rule.from": "now-6m", + "kibana.alert.rule.immutable": false, + "kibana.alert.rule.indices": Array [], + "kibana.alert.rule.interval": "5m", + "kibana.alert.rule.license": "Elastic License", + "kibana.alert.rule.max_signals": 10000, + "kibana.alert.rule.meta.someMeta": "someField", + "kibana.alert.rule.name": "rule-name", + "kibana.alert.rule.namespace": undefined, + "kibana.alert.rule.note": "# Investigative notes", + "kibana.alert.rule.parameters": Object { + "alert_suppression": undefined, + "author": Array [ + "Elastic", + ], + "building_block_type": "default", + "data_view_id": undefined, + "description": "Detecting root and admin users", + "exceptions_list": Array [ + Object { + "id": "some_uuid", + "list_id": "list_id_single", + "namespace_type": "single", + "type": "detection", + }, + Object { + "id": "endpoint_list", + "list_id": "endpoint_list", + "namespace_type": "agnostic", + "type": "endpoint", + }, + ], + "false_positives": Array [], + "filters": Array [ + Object { + "query": Object { + "match_phrase": Object { + "host.name": "some-host", + }, + }, + }, + ], + "from": "now-6m", + "immutable": false, + "index": Array [ + "auditbeat-*", + "filebeat-*", + "packetbeat-*", + "winlogbeat-*", + ], + "investigation_fields": undefined, + "language": "kuery", + "license": "Elastic License", + "max_signals": 10000, + "meta": Object { + "someMeta": "someField", + }, + "namespace": undefined, + "note": "# Investigative notes", + "query": "user.name: root or user.name: admin", + "references": Array [ + "http://example.com", + "https://example.com", + ], + "related_integrations": Array [], + "required_fields": Array [], + "response_actions": undefined, + "risk_score": 50, + "risk_score_mapping": Array [], + "rule_id": "rule-1", + "rule_name_override": undefined, + "rule_source": Object { + "type": "internal", + }, + "saved_id": undefined, + "setup": "", + "severity": "high", + "severity_mapping": Array [], + "threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "timeline_id": "some-timeline-id", + "timeline_title": "some-timeline-title", + "timestamp_override": undefined, + "timestamp_override_fallback_disabled": undefined, + "to": "now", + "type": "query", + "version": 1, + }, + "kibana.alert.rule.references": Array [ + "http://example.com", + "https://example.com", + ], + "kibana.alert.rule.risk_score": 50, + "kibana.alert.rule.risk_score_mapping": Array [], + "kibana.alert.rule.rule_id": "rule-1", + "kibana.alert.rule.rule_name_override": undefined, + "kibana.alert.rule.severity": "high", + "kibana.alert.rule.severity_mapping": Array [], + "kibana.alert.rule.tags": Array [ + "some fake tag 1", + "some fake tag 2", + ], + "kibana.alert.rule.threat": Array [ + Object { + "framework": "MITRE ATT&CK", + "tactic": Object { + "id": "TA0000", + "name": "test tactic", + "reference": "https://attack.mitre.org/tactics/TA0000/", + }, + "technique": Array [ + Object { + "id": "T0000", + "name": "test technique", + "reference": "https://attack.mitre.org/techniques/T0000/", + "subtechnique": Array [ + Object { + "id": "T0000.000", + "name": "test subtechnique", + "reference": "https://attack.mitre.org/techniques/T0000/000/", + }, + ], + }, + ], + }, + ], + "kibana.alert.rule.throttle": "no_actions", + "kibana.alert.rule.timeline_id": "some-timeline-id", + "kibana.alert.rule.timeline_title": "some-timeline-title", + "kibana.alert.rule.timestamp_override": undefined, + "kibana.alert.rule.to": "now", + "kibana.alert.rule.type": "query", + "kibana.alert.rule.updated_at": "2020-03-27T22:55:59.577Z", + "kibana.alert.rule.updated_by": "sample user", + "kibana.alert.rule.uuid": "04128c15-0d1b-4716-a4c5-46997ac7f3bd", + "kibana.alert.rule.version": 1, + "kibana.alert.severity": "high", + "kibana.alert.status": "active", + "kibana.alert.url": "test/url/app/security/alerts/redirect/test-uuid?index=.alerts-security.alerts-default×tamp=2020-01-01T00:00:00.000Z", + "kibana.alert.user.criticality_level": undefined, + "kibana.alert.uuid": "test-uuid", + "kibana.alert.workflow_assignee_ids": Array [], + "kibana.alert.workflow_status": "open", + "kibana.alert.workflow_tags": Array [], + "kibana.space_ids": Array [ + "default", + ], + "user.asset.criticality": undefined, + "user.risk.calculated_level": undefined, + "user.risk.calculated_score_norm": undefined, +} +`; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts index 4aaa0189eefc4..b7f83106ea0b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.test.ts @@ -8,42 +8,20 @@ import { ALERT_INSTANCE_ID, ALERT_NAMESPACE, - ALERT_REASON, - ALERT_RISK_SCORE, - ALERT_RULE_CONSUMER, - ALERT_RULE_NAMESPACE, - ALERT_RULE_PARAMETERS, ALERT_RULE_UUID, - ALERT_SEVERITY, - ALERT_STATUS, - ALERT_STATUS_ACTIVE, - ALERT_URL, ALERT_UUID, - ALERT_WORKFLOW_ASSIGNEE_IDS, - ALERT_WORKFLOW_STATUS, - ALERT_WORKFLOW_TAGS, EVENT_ACTION, EVENT_KIND, EVENT_MODULE, - SPACE_IDS, TIMESTAMP, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { sampleDocNoSortIdWithTimestamp } from '../../__mocks__/es_results'; -import { buildAlert, buildParent, buildAncestors, additionalAlertFields } from './build_alert'; +import { buildAlertFields, buildParent, buildAncestors } from './build_alert'; import type { Ancestor, SignalSourceHit } from '../../types'; -import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; -import { DEFAULT_ALERTS_INDEX, SERVER_APP_ID } from '../../../../../../common/constants'; import { EVENT_DATASET } from '../../../../../../common/cti/constants'; -import { - ALERT_ANCESTORS, - ALERT_ORIGINAL_TIME, - ALERT_DEPTH, - ALERT_ORIGINAL_EVENT, - ALERT_BUILDING_BLOCK_TYPE, - ALERT_RULE_INDICES, -} from '../../../../../../common/field_maps/field_names'; +import { ALERT_ANCESTORS, ALERT_DEPTH } from '../../../../../../common/field_maps/field_names'; import { getCompleteRuleMock, getQueryRuleParams } from '../../../rule_schema/mocks'; type SignalDoc = SignalSourceHit & { @@ -51,408 +29,30 @@ type SignalDoc = SignalSourceHit & { _source: Required['_source'] & { [TIMESTAMP]: string }; }; -const SPACE_ID = 'space'; -const reason = 'alert reasonable reason'; -const publicBaseUrl = 'testKibanaBasePath.com'; -const alertUuid = 'test-uuid'; - -describe('buildAlert', () => { +describe('buildAlertFields', () => { beforeEach(() => { jest.clearAllMocks(); }); - test('it builds an alert as expected without original_event if event does not exist', () => { - const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - delete doc._source.event; - const completeRule = getCompleteRuleMock(getQueryRuleParams()); - const alert = { - ...buildAlert( - [doc], - completeRule, - SPACE_ID, - reason, - completeRule.ruleParams.index as string[], - alertUuid, - publicBaseUrl, - undefined - ), - ...additionalAlertFields(doc), - }; - const timestamp = alert[TIMESTAMP]; - const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; - const expected = { - [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'signal', - [SPACE_IDS]: [SPACE_ID], - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_ANCESTORS]: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', - [ALERT_REASON]: 'alert reasonable reason', - [ALERT_STATUS]: ALERT_STATUS_ACTIVE, - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_BUILDING_BLOCK_TYPE]: 'default', - [ALERT_SEVERITY]: 'high', - [ALERT_RISK_SCORE]: 50, - [ALERT_RULE_PARAMETERS]: { - description: 'Detecting root and admin users', - risk_score: 50, - severity: 'high', - building_block_type: 'default', - note: '# Investigative notes', - license: 'Elastic License', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, - author: ['Elastic'], - false_positives: [], - from: 'now-6m', - rule_id: 'rule-1', - max_signals: 10000, - risk_score_mapping: [], - severity_mapping: [], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - to: 'now', - references: ['http://example.com', 'https://example.com'], - related_integrations: [], - required_fields: [], - setup: '', - version: 1, - exceptions_list: [ - { - id: 'some_uuid', - list_id: 'list_id_single', - namespace_type: 'single', - type: 'detection', - }, - { - id: 'endpoint_list', - list_id: 'endpoint_list', - namespace_type: 'agnostic', - type: 'endpoint', - }, - ], - immutable: false, - rule_source: { - type: 'internal', - }, - type: 'query', - language: 'kuery', - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - query: 'user.name: root or user.name: admin', - filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], - investigation_fields: undefined, - }, - [ALERT_RULE_INDICES]: completeRule.ruleParams.index, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { - actions: [], - author: ['Elastic'], - uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - building_block_type: 'default', - created_at: '2020-03-27T22:55:59.577Z', - updated_at: '2020-03-27T22:55:59.577Z', - created_by: 'sample user', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - immutable: false, - license: 'Elastic License', - meta: { - someMeta: 'someField', - }, - name: 'rule-name', - note: '# Investigative notes', - references: ['http://example.com', 'https://example.com'], - severity: 'high', - severity_mapping: [], - updated_by: 'sample user', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - version: 1, - max_signals: 10000, - risk_score: 50, - risk_score_mapping: [], - rule_id: 'rule-1', - interval: '5m', - exceptions_list: getListArrayMock(), - throttle: 'no_actions', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - }), - [ALERT_DEPTH]: 1, - [ALERT_URL]: expectedAlertUrl, - [ALERT_UUID]: alertUuid, - [ALERT_WORKFLOW_TAGS]: [], - [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], - }; - expect(alert).toEqual(expected); - }); - - test('it builds an alert as expected with original_event if present', () => { + test('it creates the expected alert fields', () => { const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - [EVENT_ACTION]: 'socket_opened', - [EVENT_DATASET]: 'socket', - [EVENT_KIND]: 'event', - [EVENT_MODULE]: 'system', - }, - }; const completeRule = getCompleteRuleMock(getQueryRuleParams()); - const alert = { - ...buildAlert( - [doc], - completeRule, - SPACE_ID, - reason, - completeRule.ruleParams.index as string[], - alertUuid, - publicBaseUrl, - undefined - ), - ...additionalAlertFields(doc), - }; - const timestamp = alert[TIMESTAMP]; - const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; - const expected = { - [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'signal', - [SPACE_IDS]: [SPACE_ID], - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_ANCESTORS]: [ - { - id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', - type: 'event', - index: 'myFakeSignalIndex', - depth: 0, - }, - ], - [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', - [ALERT_RULE_INDICES]: completeRule.ruleParams.index, - ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { - action: 'socket_opened', - dataset: 'socket', - kind: 'event', - module: 'system', - }), - [ALERT_REASON]: 'alert reasonable reason', - [ALERT_STATUS]: ALERT_STATUS_ACTIVE, - [ALERT_WORKFLOW_STATUS]: 'open', - [ALERT_BUILDING_BLOCK_TYPE]: 'default', - [ALERT_SEVERITY]: 'high', - [ALERT_RISK_SCORE]: 50, - [ALERT_RULE_PARAMETERS]: { - description: 'Detecting root and admin users', - risk_score: 50, - severity: 'high', - building_block_type: 'default', - note: '# Investigative notes', - license: 'Elastic License', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, - author: ['Elastic'], - false_positives: [], - from: 'now-6m', - rule_id: 'rule-1', - max_signals: 10000, - risk_score_mapping: [], - severity_mapping: [], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - to: 'now', - references: ['http://example.com', 'https://example.com'], - related_integrations: [], - required_fields: [], - setup: '', - version: 1, - exceptions_list: [ - { - id: 'some_uuid', - list_id: 'list_id_single', - namespace_type: 'single', - type: 'detection', - }, - { - id: 'endpoint_list', - list_id: 'endpoint_list', - namespace_type: 'agnostic', - type: 'endpoint', - }, - ], - immutable: false, - rule_source: { - type: 'internal', - }, - type: 'query', - language: 'kuery', - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - query: 'user.name: root or user.name: admin', - filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], - investigation_fields: undefined, - }, - ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { - actions: [], - author: ['Elastic'], - uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - building_block_type: 'default', - created_at: '2020-03-27T22:55:59.577Z', - updated_at: '2020-03-27T22:55:59.577Z', - created_by: 'sample user', - description: 'Detecting root and admin users', - enabled: true, - false_positives: [], - from: 'now-6m', - immutable: false, - license: 'Elastic License', - meta: { - someMeta: 'someField', - }, - name: 'rule-name', - note: '# Investigative notes', - references: ['http://example.com', 'https://example.com'], - severity: 'high', - severity_mapping: [], - updated_by: 'sample user', - tags: ['some fake tag 1', 'some fake tag 2'], - to: 'now', - type: 'query', - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0000', - name: 'test tactic', - reference: 'https://attack.mitre.org/tactics/TA0000/', - }, - technique: [ - { - id: 'T0000', - name: 'test technique', - reference: 'https://attack.mitre.org/techniques/T0000/', - subtechnique: [ - { - id: 'T0000.000', - name: 'test subtechnique', - reference: 'https://attack.mitre.org/techniques/T0000/000/', - }, - ], - }, - ], - }, - ], - version: 1, - max_signals: 10000, - risk_score: 50, - risk_score_mapping: [], - rule_id: 'rule-1', - interval: '5m', - exceptions_list: getListArrayMock(), - throttle: 'no_actions', - timeline_id: 'some-timeline-id', - timeline_title: 'some-timeline-title', - }), - [ALERT_DEPTH]: 1, - [ALERT_URL]: expectedAlertUrl, - [ALERT_UUID]: alertUuid, - [ALERT_WORKFLOW_TAGS]: [], - [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], - }; - expect(alert).toEqual(expected); + const alertFields = buildAlertFields({ + docs: [sampleDoc], + completeRule, + spaceId: 'default', + reason: 'test reason', + indicesToQuery: [], + alertUuid: 'test-uuid', + publicBaseUrl: 'test/url', + alertTimestampOverride: new Date('2020-01-01T00:00:00.000Z'), + }); + expect(alertFields).toMatchSnapshot(); }); test('it builds a parent correctly if the parent does not exist', () => { const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); - const doc = { - ...sampleDoc, - _source: { - ...sampleDoc._source, - [EVENT_ACTION]: 'socket_opened', - [EVENT_DATASET]: 'socket', - [EVENT_KIND]: 'event', - [EVENT_MODULE]: 'system', - }, - }; - const parent = buildParent(doc); + const parent = buildParent(sampleDoc); const expected: Ancestor = { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts index d81fe7d020282..036ba8c9a644a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts @@ -49,7 +49,7 @@ import { requiredOptional } from '@kbn/zod-helpers'; import { createHash } from 'crypto'; import { getAlertDetailsUrl } from '../../../../../../common/utils/alert_detail_path'; -import type { BaseSignalHit, SimpleHit } from '../../types'; +import type { SimpleHit } from '../../types'; import type { ThresholdResult } from '../../threshold/types'; import { getField, @@ -63,8 +63,6 @@ import { ALERT_ANCESTORS, ALERT_DEPTH, ALERT_ORIGINAL_TIME, - ALERT_THRESHOLD_RESULT, - ALERT_ORIGINAL_EVENT, ALERT_BUILDING_BLOCK_TYPE, ALERT_RULE_ACTIONS, ALERT_RULE_INDICES, @@ -97,6 +95,22 @@ import type { BaseFieldsLatest, } from '../../../../../../common/api/detection_engine/model/alerts'; +export interface BuildAlertFieldsProps { + docs: SimpleHit[]; + completeRule: CompleteRule; + spaceId: string | null | undefined; + reason: string; + indicesToQuery: string[]; + alertUuid: string; + publicBaseUrl: string | undefined; + alertTimestampOverride: Date | undefined; + overrides?: { + nameOverride: string; + severityOverride: string; + riskScoreOverride: number; + }; +} + export const generateAlertId = (alert: BaseFieldsLatest) => { return createHash('sha256') .update( @@ -145,21 +159,17 @@ export const buildAncestors = (doc: SimpleHit): AncestorLatest[] => { * @param reason Human readable string summarizing alert. * @param indicesToQuery Array of index patterns searched by the rule. */ -export const buildAlert = ( - docs: SimpleHit[], - completeRule: CompleteRule, - spaceId: string | null | undefined, - reason: string, - indicesToQuery: string[], - alertUuid: string, - publicBaseUrl: string | undefined, - alertTimestampOverride: Date | undefined, - overrides?: { - nameOverride: string; - severityOverride: string; - riskScoreOverride: number; - } -): BaseFieldsLatest => { +export const buildAlertFields = ({ + docs, + completeRule, + spaceId, + reason, + indicesToQuery, + alertUuid, + publicBaseUrl, + alertTimestampOverride, + overrides, +}: BuildAlertFieldsProps): BaseFieldsLatest => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce( @@ -276,28 +286,8 @@ export const buildAlert = ( }; }; -const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => { +export const isThresholdResult = ( + thresholdResult: SearchTypes +): thresholdResult is ThresholdResult => { return typeof thresholdResult === 'object'; }; - -/** - * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. - * We copy the original time from the document as "original_time" since we override the timestamp with the current date time. - * @param doc The parent signal/event of the new signal to be built. - */ -export const additionalAlertFields = (doc: BaseSignalHit) => { - const thresholdResult = doc._source?.threshold_result; - if (thresholdResult != null && !isThresholdResult(thresholdResult)) { - throw new Error(`threshold_result failed to validate: ${thresholdResult}`); - } - const additionalFields: Record = { - ...(thresholdResult != null ? { [ALERT_THRESHOLD_RESULT]: thresholdResult } : {}), - }; - - for (const [key, val] of Object.entries(doc._source ?? {})) { - if (key.startsWith('event.')) { - additionalFields[`${ALERT_ORIGINAL_EVENT}.${key.replace('event.', '')}`] = val; - } - } - return additionalFields; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts deleted file mode 100644 index b2426ceda9767..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { sampleDocWithNonEcsCompliantFields } from '../../__mocks__/es_results'; -import { buildBulkBody } from './build_bulk_body'; -import { getCompleteRuleMock, getEsqlRuleParams } from '../../../rule_schema/mocks'; -import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; - -const SPACE_ID = 'space'; -const publicBaseUrl = 'testKibanaBasePath.com'; -const alertUuid = 'test-uuid'; -const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; -const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); - -describe('buildBulkBody', () => { - test('should strip non-ECS compliant sub-fields of `event.action` field', () => { - const doc = sampleDocWithNonEcsCompliantFields(docId, { - 'event.action': 'process', - 'event.action.keyword': 'process', - }); - const completeRule = getCompleteRuleMock(getEsqlRuleParams()); - const buildReasonMessageStub = jest.fn(); - const alert = buildBulkBody( - SPACE_ID, - completeRule, - doc, - 'missingFields', - [], - true, - buildReasonMessageStub, - [], - undefined, - ruleExecutionLogger, - alertUuid, - publicBaseUrl - ); - - expect(alert['kibana.alert.original_event.action']).toEqual('process'); - expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined(); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts deleted file mode 100644 index 9294cc7159c12..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_bulk_body.ts +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { flattenWithPrefix } from '@kbn/securitysolution-rules'; -import type * as estypes from '@elastic/elasticsearch/lib/api/types'; -import { requiredOptional } from '@kbn/zod-helpers'; - -import type { BaseHit, SearchTypes } from '../../../../../../common/detection_engine/types'; -import type { ConfigType } from '../../../../../config'; -import type { BuildReasonMessage } from '../../utils/reason_formatters'; -import { getMergeStrategy } from '../../utils/source_fields_merging/strategies'; -import type { BaseSignalHit, SignalSource, SignalSourceHit } from '../../types'; -import { additionalAlertFields, buildAlert } from './build_alert'; -import { filterSource } from './filter_source'; -import type { CompleteRule, RuleParams } from '../../../rule_schema'; -import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; -import { buildRuleNameFromMapping } from '../../utils/mappings/build_rule_name_from_mapping'; -import { buildSeverityFromMapping } from '../../utils/mappings/build_severity_from_mapping'; -import { buildRiskScoreFromMapping } from '../../utils/mappings/build_risk_score_from_mapping'; -import type { BaseFieldsLatest } from '../../../../../../common/api/detection_engine/model/alerts'; -import { stripNonEcsFields } from './strip_non_ecs_fields'; - -const isSourceDoc = ( - hit: SignalSourceHit -): hit is BaseHit<{ '@timestamp': string; _source: SignalSource }> => { - return hit._source != null; -}; - -const buildEventTypeAlert = (doc: BaseSignalHit): Record => { - if (doc._source?.event != null && doc._source?.event instanceof Object) { - return flattenWithPrefix('event', doc._source?.event ?? {}); - } - return {}; -}; - -/** - * Formats the search_after result for insertion into the signals index. We first create a - * "best effort" merged "fields" with the "_source" object, then build the signal object, - * then the event object, and finally we strip away any additional temporary data that was added - * such as the "threshold_result". - * @param completeRule The rule saved object to build overrides - * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" - * @returns The body that can be added to a bulk call for inserting the signal. - */ -export const buildBulkBody = ( - spaceId: string | null | undefined, - completeRule: CompleteRule, - doc: estypes.SearchHit, - mergeStrategy: ConfigType['alertMergeStrategy'], - ignoreFields: ConfigType['alertIgnoreFields'], - applyOverrides: boolean, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, - ruleExecutionLogger: IRuleExecutionLogForExecutors, - alertUuid: string, - publicBaseUrl?: string -): BaseFieldsLatest => { - const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields }); - - const eventFields = buildEventTypeAlert(mergedDoc); - const { result: validatedEventFields, removed: removedEventFields } = - stripNonEcsFields(eventFields); - - const filteredSource = filterSource(mergedDoc); - const { result: validatedSource, removed: removedSourceFields } = - stripNonEcsFields(filteredSource); - - if (removedEventFields.length || removedSourceFields.length) { - ruleExecutionLogger?.debug( - 'Following fields were removed from alert source as ECS non-compliant:', - JSON.stringify(removedSourceFields), - JSON.stringify(removedEventFields) - ); - } - - const overrides = applyOverrides - ? { - nameOverride: buildRuleNameFromMapping({ - eventSource: mergedDoc._source ?? {}, - ruleName: completeRule.ruleConfig.name, - ruleNameMapping: completeRule.ruleParams.ruleNameOverride, - }).ruleName, - severityOverride: buildSeverityFromMapping({ - eventSource: mergedDoc._source ?? {}, - severity: completeRule.ruleParams.severity, - severityMapping: completeRule.ruleParams.severityMapping, - }).severity, - riskScoreOverride: buildRiskScoreFromMapping({ - eventSource: mergedDoc._source ?? {}, - riskScore: completeRule.ruleParams.riskScore, - riskScoreMapping: requiredOptional(completeRule.ruleParams.riskScoreMapping), - }).riskScore, - } - : undefined; - - const reason = buildReasonMessage({ - name: overrides?.nameOverride ?? completeRule.ruleConfig.name, - severity: overrides?.severityOverride ?? completeRule.ruleParams.severity, - mergedDoc, - }); - - const thresholdResult = mergedDoc._source?.threshold_result; - if (isSourceDoc(mergedDoc)) { - return { - ...validatedSource, - ...validatedEventFields, - ...buildAlert( - [mergedDoc], - completeRule, - spaceId, - reason, - indicesToQuery, - alertUuid, - publicBaseUrl, - alertTimestampOverride, - overrides - ), - ...additionalAlertFields({ - ...mergedDoc, - _source: { - ...validatedSource, - ...validatedEventFields, - threshold_result: thresholdResult, - }, - }), - }; - } - - throw Error('Error building alert from source document.'); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts deleted file mode 100644 index 73717a7c0a5ec..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { filterSource } from './filter_source'; - -describe('filterSource', () => { - test('should remove keys starting with kibana without modifying the original doc', () => { - const testDoc = { - _index: '', - _id: '', - _source: { - 'kibana.alert.suppression.docs_count': 5, - 'host.name': 'test-host', - }, - }; - const filtered = filterSource(testDoc); - expect(filtered).toEqual({ - 'host.name': 'test-host', - }); - expect(testDoc).toEqual({ - _index: '', - _id: '', - _source: { - 'kibana.alert.suppression.docs_count': 5, - 'host.name': 'test-host', - }, - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts deleted file mode 100644 index c0172207c7a60..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ALERT_THRESHOLD_RESULT } from '../../../../../../common/field_maps/field_names'; -import type { SignalSourceHit } from '../../types'; - -export const filterSource = (doc: SignalSourceHit) => { - const docSource = doc._source ?? {}; - const { - event, - kibana, - signal, - threshold_result: siemSignalsThresholdResult, - [ALERT_THRESHOLD_RESULT]: alertThresholdResult, - ...filteredSource - } = docSource || { - event: null, - kibana: null, - signal: null, - threshold_result: null, - [ALERT_THRESHOLD_RESULT]: null, - }; - - Object.keys(filteredSource).forEach((key) => { - if (key.startsWith('kibana')) { - delete filteredSource[key]; - } - }); - - return filteredSource; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts new file mode 100644 index 0000000000000..92d02e1b3ac4a --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.test.ts @@ -0,0 +1,438 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + ALERT_REASON, + ALERT_RISK_SCORE, + ALERT_RULE_CONSUMER, + ALERT_RULE_NAMESPACE, + ALERT_RULE_PARAMETERS, + ALERT_SEVERITY, + ALERT_STATUS, + ALERT_STATUS_ACTIVE, + ALERT_URL, + ALERT_UUID, + ALERT_WORKFLOW_ASSIGNEE_IDS, + ALERT_WORKFLOW_STATUS, + ALERT_WORKFLOW_TAGS, + EVENT_ACTION, + EVENT_KIND, + EVENT_MODULE, + SPACE_IDS, + TIMESTAMP, +} from '@kbn/rule-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; + +import { + sampleDocNoSortIdWithTimestamp, + sampleDocWithNonEcsCompliantFields, +} from '../../__mocks__/es_results'; +import { getListArrayMock } from '../../../../../../common/detection_engine/schemas/types/lists.mock'; +import { DEFAULT_ALERTS_INDEX, SERVER_APP_ID } from '../../../../../../common/constants'; +import { EVENT_DATASET } from '../../../../../../common/cti/constants'; +import { + ALERT_ANCESTORS, + ALERT_ORIGINAL_TIME, + ALERT_DEPTH, + ALERT_ORIGINAL_EVENT, + ALERT_BUILDING_BLOCK_TYPE, + ALERT_RULE_INDICES, +} from '../../../../../../common/field_maps/field_names'; + +import { transformHitToAlert } from './transform_hit_to_alert'; +import { + getCompleteRuleMock, + getEsqlRuleParams, + getQueryRuleParams, +} from '../../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../../rule_monitoring/mocks'; +import { get } from 'lodash'; + +const SPACE_ID = 'space'; +const publicBaseUrl = 'testKibanaBasePath.com'; +const alertUuid = 'test-uuid'; +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); +const buildReasonMessageStub = jest.fn(); + +describe('transformHitToAlert', () => { + it('should strip non-ECS compliant sub-fields of `event.action` field', () => { + const doc = sampleDocWithNonEcsCompliantFields(docId, { + 'event.action': 'process', + 'event.action.keyword': 'process', + }); + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert['kibana.alert.original_event.action']).toEqual('process'); + expect(alert['kibana.alert.original_event.action.keyword']).toBeUndefined(); + }); + + it('should unset an existing event.kind field in nested notation', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + event: { + kind: 'test-value', + }, + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(get(alert.event, 'kind')).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should replace an existing event.kind in dot notation', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + 'event.kind': 'test-value', + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(get(alert.event, 'kind')).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should not add an empty event object if event.kind does not exist', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + testField: 'testValue', + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert.event).toEqual(undefined); + expect(alert['event.kind']).toEqual('signal'); + }); + + it('should only copy ECS compatible array elements from event subfields to kibana.alert.original_event', () => { + const doc = { + _index: 'testindex', + _id: 'myId', + _source: { + 'event.action': ['process', { objectSubfield: 'test' }], + }, + }; + const completeRule = getCompleteRuleMock(getEsqlRuleParams()); + + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + + expect(alert['kibana.alert.original_event.action']).toEqual(['process']); + }); + + it('builds an alert as expected without original_event if event does not exist', () => { + const doc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + delete doc._source.event; + + const timestamp = alert[TIMESTAMP]; + const expectedAlertUrl = `${publicBaseUrl}/s/${SPACE_ID}/app/security/alerts/redirect/${alertUuid}?index=${DEFAULT_ALERTS_INDEX}-${SPACE_ID}×tamp=${timestamp}`; + const expected = { + [TIMESTAMP]: timestamp, + [EVENT_KIND]: 'signal', + [SPACE_IDS]: [SPACE_ID], + [ALERT_RULE_CONSUMER]: SERVER_APP_ID, + [ALERT_ANCESTORS]: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + [ALERT_ORIGINAL_TIME]: '2020-04-20T21:27:45.000Z', + [ALERT_REASON]: undefined, + [ALERT_STATUS]: ALERT_STATUS_ACTIVE, + [ALERT_WORKFLOW_STATUS]: 'open', + [ALERT_BUILDING_BLOCK_TYPE]: 'default', + [ALERT_SEVERITY]: 'high', + [ALERT_RISK_SCORE]: 50, + [ALERT_RULE_PARAMETERS]: { + description: 'Detecting root and admin users', + risk_score: 50, + severity: 'high', + building_block_type: 'default', + note: '# Investigative notes', + license: 'Elastic License', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + meta: { someMeta: 'someField' }, + author: ['Elastic'], + false_positives: [], + from: 'now-6m', + rule_id: 'rule-1', + max_signals: 10000, + risk_score_mapping: [], + severity_mapping: [], + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0000', + name: 'test tactic', + reference: 'https://attack.mitre.org/tactics/TA0000/', + }, + technique: [ + { + id: 'T0000', + name: 'test technique', + reference: 'https://attack.mitre.org/techniques/T0000/', + subtechnique: [ + { + id: 'T0000.000', + name: 'test subtechnique', + reference: 'https://attack.mitre.org/techniques/T0000/000/', + }, + ], + }, + ], + }, + ], + to: 'now', + references: ['http://example.com', 'https://example.com'], + related_integrations: [], + required_fields: [], + setup: '', + version: 1, + exceptions_list: [ + { + id: 'some_uuid', + list_id: 'list_id_single', + namespace_type: 'single', + type: 'detection', + }, + { + id: 'endpoint_list', + list_id: 'endpoint_list', + namespace_type: 'agnostic', + type: 'endpoint', + }, + ], + immutable: false, + rule_source: { + type: 'internal', + }, + type: 'query', + language: 'kuery', + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + query: 'user.name: root or user.name: admin', + filters: [{ query: { match_phrase: { 'host.name': 'some-host' } } }], + investigation_fields: undefined, + }, + [ALERT_RULE_INDICES]: completeRule.ruleParams.index, + ...flattenWithPrefix(ALERT_RULE_NAMESPACE, { + actions: [], + author: ['Elastic'], + uuid: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + building_block_type: 'default', + created_at: '2020-03-27T22:55:59.577Z', + updated_at: '2020-03-27T22:55:59.577Z', + created_by: 'sample user', + description: 'Detecting root and admin users', + enabled: true, + false_positives: [], + from: 'now-6m', + immutable: false, + license: 'Elastic License', + meta: { + someMeta: 'someField', + }, + name: 'rule-name', + note: '# Investigative notes', + references: ['http://example.com', 'https://example.com'], + severity: 'high', + severity_mapping: [], + updated_by: 'sample user', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0000', + name: 'test tactic', + reference: 'https://attack.mitre.org/tactics/TA0000/', + }, + technique: [ + { + id: 'T0000', + name: 'test technique', + reference: 'https://attack.mitre.org/techniques/T0000/', + subtechnique: [ + { + id: 'T0000.000', + name: 'test subtechnique', + reference: 'https://attack.mitre.org/techniques/T0000/000/', + }, + ], + }, + ], + }, + ], + version: 1, + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + rule_id: 'rule-1', + interval: '5m', + exceptions_list: getListArrayMock(), + throttle: 'no_actions', + timeline_id: 'some-timeline-id', + timeline_title: 'some-timeline-title', + }), + [ALERT_DEPTH]: 1, + [ALERT_URL]: expectedAlertUrl, + [ALERT_UUID]: alertUuid, + [ALERT_WORKFLOW_TAGS]: [], + [ALERT_WORKFLOW_ASSIGNEE_IDS]: [], + someKey: 'someValue', + source: { + ip: '127.0.0.1', + }, + }; + expect(alert).toEqual(expected); + }); + + it('builds an alert as expected with original_event if present', () => { + const sampleDoc = sampleDocNoSortIdWithTimestamp('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = { + ...sampleDoc, + _source: { + ...sampleDoc._source, + [EVENT_ACTION]: 'socket_opened', + [EVENT_DATASET]: 'socket', + [EVENT_KIND]: 'event', + [EVENT_MODULE]: 'system', + }, + }; + const completeRule = getCompleteRuleMock(getQueryRuleParams()); + const alert = transformHitToAlert({ + spaceId: SPACE_ID, + completeRule, + doc, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageStub, + indicesToQuery: [], + alertTimestampOverride: undefined, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, + }); + const expected = { + ...alert, + ...flattenWithPrefix(ALERT_ORIGINAL_EVENT, { + action: 'socket_opened', + dataset: 'socket', + kind: 'event', + module: 'system', + }), + }; + expect(alert).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts new file mode 100644 index 0000000000000..c5112006ae251 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/transform_hit_to_alert.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import { requiredOptional } from '@kbn/zod-helpers'; +import { EVENT_KIND } from '@kbn/rule-data-utils'; + +import type { BaseHit } from '../../../../../../common/detection_engine/types'; +import type { ConfigType } from '../../../../../config'; +import type { BuildReasonMessage } from '../../utils/reason_formatters'; +import { getMergeStrategy } from '../../utils/source_fields_merging/strategies'; +import type { SignalSource, SignalSourceHit } from '../../types'; +import { buildAlertFields, isThresholdResult } from './build_alert'; +import type { CompleteRule, RuleParams } from '../../../rule_schema'; +import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; +import { buildRuleNameFromMapping } from '../../utils/mappings/build_rule_name_from_mapping'; +import { buildSeverityFromMapping } from '../../utils/mappings/build_severity_from_mapping'; +import { buildRiskScoreFromMapping } from '../../utils/mappings/build_risk_score_from_mapping'; +import type { BaseFieldsLatest } from '../../../../../../common/api/detection_engine/model/alerts'; +import { traverseAndMutateDoc } from './traverse_and_mutate_doc'; +import { ALERT_THRESHOLD_RESULT } from '../../../../../../common/field_maps/field_names'; +import { robustGet, robustSet } from '../../utils/source_fields_merging/utils/robust_field_access'; + +const isSourceDoc = (hit: SignalSourceHit): hit is BaseHit => { + return hit._source != null && hit._id != null; +}; + +export interface TransformHitToAlertProps { + spaceId: string | null | undefined; + completeRule: CompleteRule; + doc: estypes.SearchHit; + mergeStrategy: ConfigType['alertMergeStrategy']; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; + applyOverrides: boolean; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + alertUuid: string; + publicBaseUrl?: string; +} + +/** + * Formats the search_after result for insertion into the signals index. We first create a + * "best effort" merged "fields" with the "_source" object, then build the signal object, + * then the event object, and finally we strip away any additional temporary data that was added + * such as the "threshold_result". + * @param completeRule The rule saved object to build overrides + * @param doc The SignalSourceHit with "_source", "fields", and additional data such as "threshold_result" + * @returns The body that can be added to a bulk call for inserting the signal. + */ +export const transformHitToAlert = ({ + spaceId, + completeRule, + doc, + mergeStrategy, + ignoreFields, + ignoreFieldsRegexes, + applyOverrides, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + ruleExecutionLogger, + alertUuid, + publicBaseUrl, +}: TransformHitToAlertProps): BaseFieldsLatest => { + const mergedDoc = getMergeStrategy(mergeStrategy)({ doc, ignoreFields, ignoreFieldsRegexes }); + const thresholdResult = mergedDoc._source?.threshold_result; + + if (isSourceDoc(mergedDoc)) { + const overrides = applyOverrides + ? { + nameOverride: buildRuleNameFromMapping({ + eventSource: mergedDoc._source ?? {}, + ruleName: completeRule.ruleConfig.name, + ruleNameMapping: completeRule.ruleParams.ruleNameOverride, + }).ruleName, + severityOverride: buildSeverityFromMapping({ + eventSource: mergedDoc._source ?? {}, + severity: completeRule.ruleParams.severity, + severityMapping: completeRule.ruleParams.severityMapping, + }).severity, + riskScoreOverride: buildRiskScoreFromMapping({ + eventSource: mergedDoc._source ?? {}, + riskScore: completeRule.ruleParams.riskScore, + riskScoreMapping: requiredOptional(completeRule.ruleParams.riskScoreMapping), + }).riskScore, + } + : undefined; + + const reason = buildReasonMessage({ + name: overrides?.nameOverride ?? completeRule.ruleConfig.name, + severity: overrides?.severityOverride ?? completeRule.ruleParams.severity, + mergedDoc, + }); + + const alertFields = buildAlertFields({ + docs: [mergedDoc], + completeRule, + spaceId, + reason, + indicesToQuery, + alertUuid, + publicBaseUrl, + alertTimestampOverride, + overrides, + }); + + const { result: validatedSource, removed: removedSourceFields } = traverseAndMutateDoc( + mergedDoc._source + ); + + // The `alertFields` we add to alerts contain `event.kind: 'signal'` in dot notation. To avoid duplicating `event.kind`, + // we remove any existing `event.kind` field here before we merge `alertFields` into `validatedSource` later on + if (robustGet({ key: EVENT_KIND, document: validatedSource }) != null) { + robustSet({ key: EVENT_KIND, document: validatedSource, valueToSet: undefined }); + } + + if (removedSourceFields.length) { + ruleExecutionLogger?.debug( + 'Following fields were removed from alert source as ECS non-compliant:', + JSON.stringify(removedSourceFields) + ); + } + + merge(validatedSource, alertFields); + if (thresholdResult != null && isThresholdResult(thresholdResult)) { + validatedSource[ALERT_THRESHOLD_RESULT] = thresholdResult; + } + return validatedSource as BaseFieldsLatest; + } + + throw Error('Error building alert from source document.'); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts similarity index 63% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts index 21f9adc96bd60..0ec57843c83da 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.test.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { stripNonEcsFields } from './strip_non_ecs_fields'; +import { traverseAndMutateDoc } from './traverse_and_mutate_doc'; -describe('stripNonEcsFields', () => { +describe('traverseAndMutateDoc', () => { it('should not strip ECS compliant fields', () => { const document = { client: { @@ -19,14 +19,14 @@ describe('stripNonEcsFields', () => { }, }; - const { result, removed } = stripNonEcsFields(document); + const { result, removed } = traverseAndMutateDoc(document); expect(result).toEqual(document); expect(removed).toEqual([]); }); it('should strip source object field if ECS mapping is not object', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: { name: { first: 'test-1', @@ -54,7 +54,7 @@ describe('stripNonEcsFields', () => { }); it('should strip source keyword field if ECS mapping is object', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: 'test', message: 'test message', }); @@ -74,7 +74,7 @@ describe('stripNonEcsFields', () => { // https://github.com/elastic/sdh-security-team/issues/736 describe('fields that exists in the alerts mapping but not in local ECS(ruleRegistry) definition', () => { it('should strip object type "device" field if it is supplied as a keyword', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ device: 'test', message: 'test message', }); @@ -94,7 +94,7 @@ describe('stripNonEcsFields', () => { describe('array fields', () => { it('should not strip arrays of objects when an object is expected', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: [{ name: 'agent-1' }, { name: 'agent-2' }], message: 'test message', }); @@ -107,7 +107,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting fields in array of objects', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: [ { name: 'agent-1', @@ -135,7 +135,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting array of keyword fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: ['agent-1', 'agent-2'], message: 'test message', }); @@ -156,7 +156,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting array of object fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ agent: { name: [{ conflict: 'agent-1' }, { conflict: 'agent-2' }], type: 'filebeat' }, message: 'test message', }); @@ -176,11 +176,29 @@ describe('stripNonEcsFields', () => { }, ]); }); + + it('should entirely strip objects that end up empty in arrays', () => { + const { result, removed } = traverseAndMutateDoc({ + agent: [{ name: { conflict: 'agent-1' } }, { name: 'test' }], + message: 'test message', + }); + + expect(result).toEqual({ + agent: [{ name: 'test' }], + message: 'test message', + }); + expect(removed).toEqual([ + { + key: 'agent.name', + value: { conflict: 'agent-1' }, + }, + ]); + }); }); describe('dot notation', () => { it('should strip conflicting fields that use dot notation', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name.conflict': 'some-value', message: 'test message', }); @@ -198,7 +216,7 @@ describe('stripNonEcsFields', () => { }); it('should strip conflicting fields that use dot notation and is an array', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name.text': ['1'], message: 'test message', }); @@ -215,8 +233,8 @@ describe('stripNonEcsFields', () => { ]); }); - it('should strip conflicting fields that use dot notation and is an empty array', () => { - const { result, removed } = stripNonEcsFields({ + it('should strip conflicting fields that use dot notation and is an empty array but not report the empty array as removed', () => { + const { result, removed } = traverseAndMutateDoc({ 'agent.name.text': [], message: 'test message', }); @@ -225,16 +243,11 @@ describe('stripNonEcsFields', () => { message: 'test message', }); - expect(removed).toEqual([ - { - key: 'agent.name.text', - value: [], - }, - ]); + expect(removed).toEqual([]); }); it('should not strip valid ECS fields that use dot notation', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'agent.name': 'some name', 'agent.build.original': 'v10', message: 'test message', @@ -253,7 +266,7 @@ describe('stripNonEcsFields', () => { describe('non-ECS fields', () => { it('should not strip non-ECS fields that don`t conflict', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ non_ecs_object: { field1: 'value1', }, @@ -272,7 +285,7 @@ describe('stripNonEcsFields', () => { it('should not strip non-ECS fields that don`t conflict even when nested inside ECS fieldsets', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ agent: { non_ecs_object: { field1: 'value1', @@ -298,7 +311,7 @@ describe('stripNonEcsFields', () => { describe('ip field', () => { it('should not strip valid CIDR', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ source: { ip: '192.168.0.0', name: 'test source', @@ -315,7 +328,7 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid ip', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ source: { ip: 'invalid-ip', name: 'test source', @@ -336,7 +349,7 @@ describe('stripNonEcsFields', () => { describe('nested field', () => { it('should strip invalid nested', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ threat: { enrichments: ['non-valid-threat-1', 'non-valid-threat-2'], 'indicator.port': 443, @@ -361,7 +374,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid values', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ threat: { enrichments: [ { @@ -386,7 +399,7 @@ describe('stripNonEcsFields', () => { describe('date field', () => { it('should strip invalid date', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ event: { created: true, category: 'start', @@ -397,6 +410,7 @@ describe('stripNonEcsFields', () => { event: { category: 'start', }, + 'kibana.alert.original_event.category': 'start', }); expect(removed).toEqual([ { @@ -407,7 +421,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string or number date field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ event: { created: '2020-12-12', end: [2345562, '2022-10-12'], @@ -419,6 +433,8 @@ describe('stripNonEcsFields', () => { created: '2020-12-12', end: [2345562, '2022-10-12'], }, + 'kibana.alert.original_event.created': '2020-12-12', + 'kibana.alert.original_event.end': [2345562, '2022-10-12'], }); expect(removed).toEqual([]); }); @@ -426,13 +442,13 @@ describe('stripNonEcsFields', () => { describe('long field', () => { it('should strip invalid long field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ client: { bytes: 'non-valid', }, }); - expect(result).toEqual({ client: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'client.bytes', @@ -442,13 +458,13 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid long field with space in it', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ client: { bytes: '24 ', }, }); - expect(result).toEqual({ client: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'client.bytes', @@ -459,7 +475,7 @@ describe('stripNonEcsFields', () => { }); describe('numeric field', () => { it('should strip invalid float field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': 'non-valid', }); @@ -473,13 +489,13 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid scaled_float field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ host: { 'cpu.usage': 'non-valid', }, }); - expect(result).toEqual({ host: {} }); + expect(result).toEqual({}); expect(removed).toEqual([ { key: 'host.cpu.usage', @@ -489,7 +505,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string float field with space', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': '24 ', }); @@ -500,7 +516,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip string scaled_float field with space', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'host.cpu.usage': '24 ', }); @@ -511,7 +527,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid number in string field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ host: { 'cpu.usage': '1234', }, @@ -526,7 +542,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip array of valid numeric fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'user.risk.calculated_score': [458.3333, '45.3', 10, 0, -667.23], }); @@ -539,7 +555,7 @@ describe('stripNonEcsFields', () => { describe('boolean field', () => { it('should strip invalid boolean fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': ['conflict', 'true', 5, 'False', 'ee', 'True'], }); @@ -571,7 +587,7 @@ describe('stripNonEcsFields', () => { }); it('should strip invalid boolean True', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': 'True', }); @@ -585,7 +601,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid boolean fields', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': ['true', 'false', true, false, ''], }); @@ -596,7 +612,7 @@ describe('stripNonEcsFields', () => { }); it('should not strip valid boolean fields nested in array', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'dll.code_signature.trusted': [[true, false], ''], }); @@ -610,7 +626,7 @@ describe('stripNonEcsFields', () => { // geo_point is too complex so we going to skip its validation describe('geo_point field', () => { it('should not strip invalid geo_point field', () => { - const { result, removed } = stripNonEcsFields({ + const { result, removed } = traverseAndMutateDoc({ 'client.location.geo': 'invalid geo_point', }); @@ -622,7 +638,7 @@ describe('stripNonEcsFields', () => { it('should not strip valid geo_point fields', () => { expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': [0, 90], }).result ).toEqual({ @@ -630,7 +646,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': { type: 'Point', coordinates: [-88.34, 20.12], @@ -644,7 +660,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ 'client.geo.location': 'POINT (-71.34 41.12)', }).result ).toEqual({ @@ -652,7 +668,7 @@ describe('stripNonEcsFields', () => { }); expect( - stripNonEcsFields({ + traverseAndMutateDoc({ client: { geo: { location: { @@ -674,4 +690,152 @@ describe('stripNonEcsFields', () => { }); }); }); + + describe('globally ignored fields', () => { + it('should strip out globally ignored top level fields', () => { + const { result, removed } = traverseAndMutateDoc({ + kibana: 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + non_ecs_field: 'value', + }); + expect(removed).toEqual([ + { + key: 'kibana', + value: 'test-value', + }, + ]); + }); + + it('should strip out globally ignored nested fields', () => { + const { result, removed } = traverseAndMutateDoc({ + 'kibana.test': 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + non_ecs_field: 'value', + }); + expect(removed).toEqual([ + { + key: 'kibana.test', + value: 'test-value', + }, + ]); + }); + + it('should not strip out fields that use ignored field names as a prefix', () => { + const { result, removed } = traverseAndMutateDoc({ + kibana_test_prefix: 'test-value', + non_ecs_field: 'value', + }); + + expect(result).toEqual({ + kibana_test_prefix: 'test-value', + non_ecs_field: 'value', + }); + expect(removed).toEqual([]); + }); + }); + + describe('fieldsToAdd', () => { + it('should extract a nested event field to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ event: { action: 'test' } }); + expect(result).toEqual({ + event: { action: 'test' }, + 'kibana.alert.original_event.action': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should extract multiple nested event fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { action: 'test', field2: 'test2' }, + }); + expect(result).toEqual({ + event: { action: 'test', field2: 'test2' }, + 'kibana.alert.original_event.action': 'test', + 'kibana.alert.original_event.field2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract a dot notation event field to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ 'event.action': 'test' }); + expect(result).toEqual({ + 'event.action': 'test', + 'kibana.alert.original_event.action': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should extract multiple dot notation event fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + 'event.action': 'test', + 'event.field2': 'test2', + }); + expect(result).toEqual({ + 'event.action': 'test', + 'event.field2': 'test2', + 'kibana.alert.original_event.action': 'test', + 'kibana.alert.original_event.field2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract mixed notation fields to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { 'field.subfield': 'test', 'field2.subfield': 'test2' }, + }); + expect(result).toEqual({ + event: { 'field.subfield': 'test', 'field2.subfield': 'test2' }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field2.subfield': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should extract mixed notation with dot notation first to kibana.alert.original_event', () => { + const { result, removed } = traverseAndMutateDoc({ + 'event.field': { subfield: 'test', subfield2: 'test2' }, + }); + expect(result).toEqual({ + 'event.field': { subfield: 'test', subfield2: 'test2' }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field.subfield2': 'test2', + }); + expect(removed).toEqual([]); + }); + + it('should not extract original event fields if they are not top level', () => { + const { result, removed } = traverseAndMutateDoc({ + 'top_field.event.action': 'test', + }); + expect(result).toEqual({ 'top_field.event.action': 'test' }); + expect(removed).toEqual([]); + }); + + it('should not duplicate added fields', () => { + const { result, removed } = traverseAndMutateDoc({ event: { event: 'test' } }); + expect(result).toEqual({ + event: { event: 'test' }, + 'kibana.alert.original_event.event': 'test', + }); + expect(removed).toEqual([]); + }); + + it('should work on multiple levels of nesting', () => { + const { result, removed } = traverseAndMutateDoc({ + event: { field: { subfield: 'test', subfield2: 'test2' } }, + }); + expect(result).toEqual({ + event: { field: { subfield: 'test', subfield2: 'test2' } }, + 'kibana.alert.original_event.field.subfield': 'test', + 'kibana.alert.original_event.field.subfield2': 'test2', + }); + expect(removed).toEqual([]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts similarity index 50% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts index 08e5fa5fd879d..c9720a139ae7d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/traverse_and_mutate_doc.ts @@ -6,8 +6,9 @@ */ import { ecsFieldMap } from '@kbn/alerts-as-data-utils'; +import { flattenWithPrefix } from '@kbn/securitysolution-rules'; -import { isPlainObject, cloneDeep, isArray } from 'lodash'; +import { isPlainObject, isArray, set } from 'lodash'; import type { SearchTypes } from '../../../../../../common/detection_engine/types'; import { isValidIpType } from './ecs_types_validators/is_valid_ip_type'; @@ -15,6 +16,10 @@ import { isValidDateType } from './ecs_types_validators/is_valid_date_type'; import { isValidNumericType } from './ecs_types_validators/is_valid_numeric_type'; import { isValidBooleanType } from './ecs_types_validators/is_valid_boolean_type'; import { isValidLongType } from './ecs_types_validators/is_valid_long_type'; +import { + ALERT_ORIGINAL_EVENT, + ALERT_THRESHOLD_RESULT, +} from '../../../../../../common/field_maps/field_names'; type SourceFieldRecord = Record; type SourceField = SearchTypes | SourceFieldRecord; @@ -153,67 +158,149 @@ const computeIsEcsCompliant = (value: SourceField, path: string) => { return isEcsFieldObject ? isPlainObject(value) : !isPlainObject(value); }; -interface StripNonEcsFieldsReturn { - result: SourceFieldRecord; - removed: Array<{ key: string; value: SearchTypes }>; -} +const bannedFields = ['kibana', 'signal', 'threshold_result', ALERT_THRESHOLD_RESULT]; /** - * strips alert source object from ECS non compliant fields + * Traverse an entire source document and mutate it to prepare for indexing into the alerts index. Traversing the document + * is computationally expensive so we only want to traverse it once, therefore a few distinct cases are handled in this function: + * 1. Fields that we must explicitly remove, like `kibana` and `signal`, fields, are removed from the document. + * 2. Fields that are incompatible with ECS are removed. + * 3. All `event.*` fields are collected and copied to `kibana.alert.original_event.*` using `fieldsToAdd` + * @param document The document to traverse + * @returns The mutated document, a list of removed fields */ -export const stripNonEcsFields = (doc: SourceFieldRecord): StripNonEcsFieldsReturn => { - const result = cloneDeep(doc); - const removed: Array<{ key: string; value: SearchTypes }> = []; - - /** - * traverses through object and deletes ECS non compliant fields - * @param document - document to traverse - * @param documentKey - document key in parent document, if exists - * @param parent - parent of traversing document - * @param parentPath - path of parent in initial source document - */ - const traverseAndDeleteInObj = ( - document: SourceField, - documentKey: string, - parent?: SourceFieldRecord, - parentPath?: string - ) => { - const fullPath = [parentPath, documentKey].filter(Boolean).join('.'); - // if document array, traverse through each item w/o changing documentKey, parent, parentPath - if (isArray(document) && document.length > 0) { - document.slice().forEach((value) => { - traverseAndDeleteInObj(value, documentKey, parent, parentPath); +export const traverseAndMutateDoc = (document: SourceFieldRecord) => { + const { result, removed, fieldsToAdd } = internalTraverseAndMutateDoc({ + document, + path: [], + topLevel: true, + removed: [], + fieldsToAdd: [], + }); + + fieldsToAdd.forEach(({ key, value }) => { + result[key] = value; + }); + + return { result, removed }; +}; + +const internalTraverseAndMutateDoc = ({ + document, + path, + topLevel, + removed, + fieldsToAdd, +}: { + document: T; + path: string[]; + topLevel: boolean; + removed: Array<{ key: string; value: SearchTypes }>; + fieldsToAdd: Array<{ key: string; value: SearchTypes }>; +}) => { + Object.keys(document).forEach((key) => { + // Using Object.keys and fetching the value for each key separately performs better in profiling than using Object.entries + const value = document[key]; + const fullPathArray = [...path, key]; + const fullPath = fullPathArray.join('.'); + // Insert checks that don't care about the value - only depend on the key - up here + let deleted = false; + if (topLevel) { + const firstKeyString = key.split('.')[0]; + bannedFields.forEach((bannedField) => { + if (firstKeyString === bannedField) { + delete document[key]; + deleted = true; + removed.push({ key: fullPath, value }); + } }); - return; } - if (parent && !computeIsEcsCompliant(document, fullPath)) { - const documentReference = parent[documentKey]; - // if document reference in parent is array, remove only this item from array - // e.g. a boolean mapped field with values ['not-boolean', 'true'] should strip 'not-boolean' and leave 'true' - if (isArray(documentReference)) { - const indexToDelete = documentReference.findIndex((item) => item === document); - documentReference.splice(indexToDelete, 1); - if (documentReference.length === 0) { - delete parent[documentKey]; + // If we passed the key check, additional checks based on key and value are done below. Items in arrays are treated independently from each other. + if (!deleted) { + if (isArray(value)) { + const newValue = traverseArray({ array: value, path: fullPathArray, removed, fieldsToAdd }); + if (newValue.length > 0) { + set(document, key, newValue); + } else { + delete document[key]; + deleted = true; + } + } else if (!computeIsEcsCompliant(value, fullPath)) { + delete document[key]; + deleted = true; + removed.push({ key: fullPath, value }); + } else if (isSearchTypesRecord(value)) { + internalTraverseAndMutateDoc({ + document: value, + path: fullPathArray, + topLevel: false, + removed, + fieldsToAdd, + }); + if (Object.keys(value).length === 0) { + delete document[key]; + deleted = true; + } + } + } + + // We're keeping the field, but maybe we want to copy it to a different field as well + if (!deleted && fullPath.split('.')[0] === 'event' && topLevel) { + // The value might have changed above when we `set` after traversing an array + const valueRefetch = document[key]; + const newKey = `${ALERT_ORIGINAL_EVENT}${fullPath.replace('event', '')}`; + if (isPlainObject(valueRefetch)) { + const flattenedObject = flattenWithPrefix(newKey, valueRefetch); + for (const [k, v] of Object.entries(flattenedObject)) { + fieldsToAdd.push({ key: k, value: v }); } } else { - delete parent[documentKey]; + fieldsToAdd.push({ + key: `${ALERT_ORIGINAL_EVENT}${fullPath.replace('event', '')}`, + value: valueRefetch, + }); } - removed.push({ key: fullPath, value: document }); - return; } + }); + return { result: document, removed, fieldsToAdd }; +}; - if (isSearchTypesRecord(document)) { - Object.entries(document).forEach(([key, value]) => { - traverseAndDeleteInObj(value, key, document, fullPath); +const traverseArray = ({ + array, + path, + removed, + fieldsToAdd, +}: { + array: SearchTypes[]; + path: string[]; + removed: Array<{ key: string; value: SearchTypes }>; + fieldsToAdd: Array<{ key: string; value: SearchTypes }>; +}): SearchTypes[] => { + const pathString = path.join('.'); + for (let i = 0; i < array.length; i++) { + const value = array[i]; + if (isArray(value)) { + array[i] = traverseArray({ array: value, path, removed, fieldsToAdd }); + } + } + return array.filter((value) => { + if (isArray(value)) { + return value.length > 0; + } else if (!computeIsEcsCompliant(value, pathString)) { + removed.push({ key: pathString, value }); + return false; + } else if (isSearchTypesRecord(value)) { + internalTraverseAndMutateDoc({ + document: value, + path, + topLevel: false, + removed, + fieldsToAdd, }); + return Object.keys(value).length > 0; + } else { + return true; } - }; - - traverseAndDeleteInObj(result, ''); - return { - result, - removed, - }; + }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts index 6b21ed226c165..2495e48c1cc81 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/wrap_hits_factory.ts @@ -10,7 +10,7 @@ import type { ConfigType } from '../../../../config'; import type { SignalSource, SimpleHit } from '../types'; import type { CompleteRule, RuleParams } from '../../rule_schema'; import { generateId } from '../utils/utils'; -import { buildBulkBody } from './utils/build_bulk_body'; +import { transformHitToAlert } from './utils/transform_hit_to_alert'; import type { BuildReasonMessage } from '../utils/reason_formatters'; import type { BaseFieldsLatest, @@ -22,6 +22,7 @@ export const wrapHitsFactory = ({ completeRule, ignoreFields, + ignoreFieldsRegexes, mergeStrategy, spaceId, indicesToQuery, @@ -30,7 +31,8 @@ export const wrapHitsFactory = ruleExecutionLogger, }: { completeRule: CompleteRule; - ignoreFields: ConfigType['alertIgnoreFields']; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; mergeStrategy: ConfigType['alertMergeStrategy']; spaceId: string | null | undefined; indicesToQuery: string[]; @@ -51,20 +53,21 @@ export const wrapHitsFactory = `${spaceId}:${completeRule.alertId}` ); - const baseAlert = buildBulkBody( + const baseAlert = transformHitToAlert({ spaceId, completeRule, - event as SimpleHit, + doc: event as SimpleHit, mergeStrategy, ignoreFields, - true, + ignoreFieldsRegexes, + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts index 5028c15c2c8d1..efd33c5442bf3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_new_terms_alerts.ts @@ -18,7 +18,7 @@ import type { CompleteRule, RuleParams } from '../../rule_schema'; import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { SignalSource } from '../types'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; export interface EventsAndTerms { event: estypes.SearchHit; @@ -52,20 +52,21 @@ export const wrapNewTermsAlerts = ({ `${spaceId}:${completeRule.alertId}`, eventAndTerms.newTerms, ]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - eventAndTerms.event, + doc: eventAndTerms.event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts index 274a11fc4ffcc..ad34feb81eab1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts @@ -21,7 +21,7 @@ import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; import type { SignalSource } from '../types'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; export interface EventsAndTerms { event: estypes.SearchHit; @@ -69,20 +69,21 @@ export const wrapSuppressedNewTermsAlerts = ({ eventAndTerms.newTerms, ]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, - buildReasonMessageForNewTermsAlert, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: buildReasonMessageForNewTermsAlert, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts index e7bfe3f7eaacd..abff900a5dcb4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/alert_suppression/wrap_suppressed_alerts.ts @@ -23,7 +23,7 @@ import type { ConfigType } from '../../../../../config'; import type { CompleteRule, RuleParams } from '../../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../../rule_monitoring'; import type { SignalSource } from '../../types'; -import { buildBulkBody } from '../../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../../factories/utils/transform_hit_to_alert'; import type { BuildReasonMessage } from '../../utils/reason_formatters'; export interface SuppressionBucket { @@ -82,20 +82,21 @@ export const wrapSuppressedAlerts = ({ ruleId: completeRule.alertId, spaceId, }); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - bucket.event, + doc: bucket.event, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts index 492d27fba091f..e9acb8284b740 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/wrap_suppressed_threshold_alerts.ts @@ -25,7 +25,7 @@ import type { import type { ConfigType } from '../../../../config'; import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import type { ThresholdBucket } from './types'; import type { BuildReasonMessage } from '../utils/reason_formatters'; @@ -86,20 +86,21 @@ export const wrapSuppressedThresholdALerts = ({ const instanceId = objectHash([suppressedValues, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - hit, + doc: hit, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts index 4d414d71cfadf..60b7e3edacedf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/search_after_bulk_create.test.ts @@ -10,7 +10,6 @@ import { repeatedSearchResultsWithSortId, repeatedSearchResultsWithNoSortId, sampleDocSearchResultsNoSortIdNoHits, - sampleDocWithSortId, } from '../__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import type { RuleExecutorServicesMock } from '@kbn/alerting-plugin/server/mocks'; @@ -114,7 +113,8 @@ describe('searchAfterAndBulkCreate', () => { wrapHits = wrapHitsFactory({ completeRule: queryCompleteRule, mergeStrategy: 'missingFields', - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], spaceId: 'default', indicesToQuery: inputIndexPattern, alertTimestampOverride: undefined, @@ -1025,8 +1025,22 @@ describe('searchAfterAndBulkCreate', () => { expect(mockEnrichment).toHaveBeenCalledWith( expect.objectContaining([ expect.objectContaining({ - ...sampleDocWithSortId(), _id: expect.any(String), + _index: 'myFakeSignalIndex', + _score: 100, + _source: expect.objectContaining({ + destination: { ip: '127.0.0.1' }, + someKey: 'someValue', + source: { ip: '127.0.0.1' }, + }), + _version: 1, + fields: { + '@timestamp': ['2020-04-20T21:27:45+0000'], + 'destination.ip': ['127.0.0.1'], + someKey: ['someValue'], + 'source.ip': ['127.0.0.1'], + }, + sort: ['1234567891111', '2233447556677'], }), ]) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts index 2b69f4fe980f2..6f4f15ad942be 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.test.ts @@ -46,7 +46,11 @@ describe('merge_all_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -55,7 +59,11 @@ describe('merge_all_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -64,7 +72,11 @@ describe('merge_all_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -73,7 +85,11 @@ describe('merge_all_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -82,7 +98,11 @@ describe('merge_all_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -91,7 +111,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -100,7 +124,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -109,7 +137,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -135,7 +167,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -144,7 +180,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -153,7 +193,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -162,7 +206,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -171,7 +219,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -180,7 +232,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -189,7 +245,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -219,7 +279,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -228,7 +292,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -237,7 +305,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -246,7 +318,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -255,7 +331,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -264,7 +344,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -273,7 +357,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -301,7 +389,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -310,7 +402,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -319,7 +415,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -328,7 +428,11 @@ describe('merge_all_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -337,7 +441,11 @@ describe('merge_all_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -346,7 +454,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -355,7 +467,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -378,7 +494,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -391,7 +511,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -404,7 +528,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -415,7 +543,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -442,7 +574,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -455,7 +591,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -466,7 +606,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -479,7 +623,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }); @@ -505,7 +653,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': 'other_value_1', }); @@ -516,7 +668,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -525,7 +681,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': { zed: 'other_value_1' }, }); @@ -536,7 +696,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -562,7 +726,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -573,7 +741,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -584,7 +756,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -595,7 +771,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -621,7 +801,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -630,7 +814,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -639,7 +827,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -648,7 +840,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -672,7 +868,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'] }, }); @@ -683,7 +883,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'] }, }); @@ -694,7 +898,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -705,7 +913,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -731,7 +943,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -740,7 +956,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -749,7 +969,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -758,7 +982,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -784,7 +1012,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1'], @@ -797,7 +1029,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -810,7 +1046,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }], @@ -823,7 +1063,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], @@ -851,7 +1095,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -860,7 +1108,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -869,7 +1121,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -878,7 +1134,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -904,7 +1164,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -913,7 +1177,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -922,7 +1190,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: { zed: 'other_value_1' } }, }); @@ -933,7 +1205,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -959,7 +1235,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -968,7 +1248,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -977,7 +1261,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': { mars: 'other_value_1' }, }); @@ -988,7 +1276,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }); @@ -1016,7 +1308,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1025,7 +1321,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1034,7 +1334,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1045,7 +1349,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1071,7 +1379,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1080,7 +1392,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1089,7 +1405,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }] }, }); @@ -1100,7 +1420,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: [{ zed: 'other_value_1' }, { zed: 'other_value_2' }] }, }); @@ -1126,7 +1450,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1135,7 +1463,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1144,7 +1476,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -1153,7 +1489,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -1177,8 +1517,12 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; - expect(merged).toEqual(_source); + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; + expect(merged).toEqual(cloneDeep(_source)); }); test('fields has primitive values (f_[p2, ...2]), merged doc is the same _source (f_[{}1, ...1])"', () => { @@ -1186,8 +1530,12 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; - expect(merged).toEqual(_source); + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; + expect(merged).toEqual(cloneDeep(_source)); }); test('fields has a single nested object (f_[{}2]), merged doc is array value (f_[{}2])"', () => { @@ -1195,7 +1543,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); @@ -1204,7 +1556,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(fields); }); }); @@ -1230,10 +1586,14 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - foo: { bar: 'value_1' }, - 'foo.bar': 'other_value_1', + foo: { bar: 'other_value_1' }, + 'foo.bar': 'value_2', }); }); @@ -1245,10 +1605,14 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - foo: { bar: 'value_1' }, // <--- We have duplicated value_1 twice which is a bug - 'foo.bar': ['value_1', 'value_2'], // <-- We have merged the array value because we do not understand if we should or not + foo: { bar: ['value_1', 'value_2'] }, // <-- We have merged the array value because we do not understand if we should or not + 'foo.bar': 'value_2', // <--- We have duplicated value_2 twice which is a bug }); }); }); @@ -1272,7 +1636,11 @@ describe('merge_all_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'foo_other_value_1', bar: 'bar_other_value_1', @@ -1293,7 +1661,11 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ host: { hostname: 'hostname_other_value_1', @@ -1318,7 +1690,11 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { host: { @@ -1339,7 +1715,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1353,7 +1733,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1365,7 +1749,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1385,7 +1773,11 @@ describe('merge_all_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'host.name': 'host_name_other_value_1', 'host.hostname': 'hostname_other_value_1', @@ -1404,7 +1796,11 @@ describe('merge_all_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'foo.host.name': 'host_name_other_value_1', 'foo.host.hostname': 'hostname_other_value_1', @@ -1419,7 +1815,11 @@ describe('merge_all_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1434,7 +1834,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1447,7 +1851,11 @@ describe('merge_all_fields_with_source', () => { 'process.command_line.text': ['string longer than 10 characters'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1472,7 +1880,11 @@ describe('merge_all_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1492,7 +1904,11 @@ describe('merge_all_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1507,7 +1923,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'single_value', zed: 'single_value' }, }); @@ -1526,7 +1946,11 @@ describe('merge_all_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: [{ bar: ['single_value'], zed: ['single_value'] }], }); @@ -1543,15 +1967,15 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - email: { - headers: { - 'x-test': 'from fields', - }, + 'email.headers': { + 'x-test': 'from fields', }, - // preserves conflicting keys if values contain empty objects - 'email.headers': {}, }); }); @@ -1564,7 +1988,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields', @@ -1581,7 +2009,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields' }, }); @@ -1596,7 +2028,11 @@ describe('merge_all_fields_with_source', () => { 'email.headers.x-test': ['b1', 'b2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': ['b1', 'b2'] }, }); @@ -1611,7 +2047,11 @@ describe('merge_all_fields_with_source', () => { 'a.b.c.d': ['5', '6'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1624,7 +2064,11 @@ describe('merge_all_fields_with_source', () => { 'a.b.c': [{ d: '3 ' }, { d: '4' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeAllFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeAllFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'a.b': { c: [{ d: '3 ' }, { d: '4' }] }, }); @@ -1645,7 +2089,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1664,7 +2109,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + ignoreFields: { 'other.string': true }, + ignoreFieldsRegexes: ['/[z]+/'], // Neither of these two should match anything })._source; expect(merged).toEqual({ foo: { @@ -1694,7 +2140,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1718,7 +2165,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: ['nothing.to.match', '/[z]+/'], // these match nothing + ignoreFields: { 'nothing.to.match': true }, + ignoreFieldsRegexes: ['/[z]+/'], // these match nothing })._source; expect(merged).toEqual({ foo: { @@ -1743,7 +2191,8 @@ describe('merge_all_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeAllFieldsWithSource({ doc, - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], })._source; expect(merged).toEqual({ foo: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts index 762542d2b93dc..7b75899fe60f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_all_fields_with_source.ts @@ -5,10 +5,9 @@ * 2.0. */ -import { get } from 'lodash/fp'; -import { set } from '@kbn/safer-lodash-set/fp'; +import { robustGet, robustSet } from '../utils/robust_field_access'; import type { SignalSource } from '../../../types'; -import { filterFieldEntries } from '../utils/filter_field_entries'; +import { filterFieldEntry } from '../utils/filter_field_entry'; import type { FieldsType, MergeStrategyFunction } from '../types'; import { isObjectLikeOrArrayOfObjectLikes } from '../utils/is_objectlike_or_array_of_objectlikes'; import { isNestedObject } from '../utils/is_nested_object'; @@ -16,8 +15,7 @@ import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isPrimitive } from '../utils/is_primitive'; import { isArrayOfPrimitives } from '../utils/is_array_of_primitives'; import { isTypeObject } from '../utils/is_type_object'; -import { isPathValid } from '../utils/is_path_valid'; -import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map'; +import { robustIsPathValid } from '../utils/is_path_valid'; /** * Merges all of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -29,63 +27,56 @@ import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { +export const mergeAllFieldsWithSource: MergeStrategyFunction = ({ + doc, + ignoreFields, + ignoreFieldsRegexes, +}) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; - const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); - const fieldsKeyMap = buildFieldsKeyAsArrayMap(source); - - const transformedSource = filteredEntries.reduce( - (merged, [fieldsKeyAsString, fieldsValue]: [string, FieldsType]) => { - const fieldsKey = fieldsKeyMap[fieldsKeyAsString] ?? fieldsKeyAsString; - - if ( - hasEarlyReturnConditions({ - fieldsValue, - fieldsKey, - merged, - }) - ) { - return merged; - } + const fieldsKeys = Object.keys(fields); - const valueInMergedDocument = get(fieldsKey, merged); + fieldsKeys.forEach((fieldsKey) => { + const valueInMergedDocument = robustGet({ key: fieldsKey, document: source }); + const fieldsValue = fields[fieldsKey]; + if ( + !hasEarlyReturnConditions({ + fieldsValue, + fieldsKey, + merged: source, + }) && + filterFieldEntry([fieldsKey, fieldsValue], fieldsKeys, ignoreFields, ignoreFieldsRegexes) + ) { if (valueInMergedDocument === undefined) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if (isPrimitive(valueInMergedDocument)) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if (isArrayOfPrimitives(valueInMergedDocument)) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if ( isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && isNestedObject(fieldsValue) && !Array.isArray(valueInMergedDocument) ) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } else if ( isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && isNestedObject(fieldsValue) && Array.isArray(valueInMergedDocument) ) { const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(fieldsKey, valueToMerge, merged); - } else { - // fail safe catch all condition for production, but we shouldn't try to reach here and - // instead write tests if we encounter this situation. - return merged; + robustSet({ key: fieldsKey, valueToSet: valueToMerge, document: source }); } - }, - { ...source } - ); + } + }); return { ...doc, - _source: transformedSource, + _source: source, }; }; @@ -105,13 +96,13 @@ const hasEarlyReturnConditions = ({ merged, }: { fieldsValue: FieldsType; - fieldsKey: string[] | string; + fieldsKey: string; merged: SignalSource; }) => { - const valueInMergedDocument = get(fieldsKey, merged); + const valueInMergedDocument = robustGet({ key: fieldsKey, document: merged }); return ( fieldsValue.length === 0 || - (valueInMergedDocument === undefined && !isPathValid(fieldsKey, merged)) || + (valueInMergedDocument === undefined && !robustIsPathValid(fieldsKey, merged)) || (isObjectLikeOrArrayOfObjectLikes(valueInMergedDocument) && !isNestedObject(fieldsValue) && !isTypeObject(fieldsValue)) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts index 80b9360541563..04089f449c368 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.test.ts @@ -45,7 +45,11 @@ describe('merge_missing_fields_with_source', () => { test('when source is "undefined", merged doc is "undefined"', () => { const _source: SignalSourceHit['_source'] = {}; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -54,7 +58,11 @@ describe('merge_missing_fields_with_source', () => { foo: [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -63,7 +71,11 @@ describe('merge_missing_fields_with_source', () => { foo: 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -72,7 +84,11 @@ describe('merge_missing_fields_with_source', () => { foo: ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -81,7 +97,11 @@ describe('merge_missing_fields_with_source', () => { foo: ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -90,7 +110,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -99,7 +123,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -108,7 +136,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -134,7 +166,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -143,7 +179,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -152,7 +192,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -161,7 +205,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -170,7 +218,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -179,7 +231,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -188,7 +244,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -218,7 +278,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -227,7 +291,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -236,7 +304,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -245,7 +317,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: ['value_1', 'value_2'] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -254,7 +330,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: { mars: 'some value' } }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -263,7 +343,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -272,7 +356,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: [{ mars: 'some value' }, { mars: 'some other value' }] }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -310,7 +398,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': [], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -319,7 +411,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': 'value', }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -328,7 +424,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -337,7 +437,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.foo': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -346,7 +450,11 @@ describe('merge_missing_fields_with_source', () => { foo: { bar: 'some value' }, }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -355,7 +463,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -364,7 +476,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'some value' }, { foo: 'some other value' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -390,7 +506,7 @@ describe('merge_missing_fields_with_source', () => { const start = performance.now(); // we don't care about the response just determining performance // eslint-disable-next-line @typescript-eslint/no-unused-expressions - mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + mergeMissingFieldsWithSource({ doc, ignoreFields: {}, ignoreFieldsRegexes: [] })._source; const end = performance.now(); expect(end - start).toBeLessThan(500); }); @@ -414,7 +530,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: 'other_value_1', @@ -427,7 +547,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: { bar: ['other_value_1', 'other_value_2'], @@ -440,7 +564,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); @@ -449,7 +577,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); }); @@ -474,7 +606,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -483,7 +619,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -492,7 +632,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -501,7 +645,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: 'other_value_1' }, { bar: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -525,7 +673,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -534,7 +686,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -543,7 +699,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -552,7 +712,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -578,7 +742,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -587,7 +755,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -596,7 +768,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -605,7 +781,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -629,7 +809,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -638,7 +822,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -647,7 +835,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -656,7 +848,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -680,7 +876,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -689,7 +889,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -698,7 +902,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -707,7 +915,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -731,7 +943,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -740,7 +956,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -749,7 +969,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -758,7 +982,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -784,7 +1012,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -793,7 +1025,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -802,7 +1038,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -811,7 +1051,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -835,7 +1079,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -844,7 +1092,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -853,7 +1105,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -862,7 +1118,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -888,7 +1148,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -897,7 +1161,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -906,7 +1174,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -915,7 +1187,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -939,7 +1215,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -948,7 +1228,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -957,7 +1241,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -966,7 +1254,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -992,7 +1284,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1001,7 +1297,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1010,7 +1310,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1019,7 +1323,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1043,7 +1351,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1052,7 +1364,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1061,7 +1377,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1070,7 +1390,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ zed: 'other_value_1' }, { zed: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1094,7 +1418,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1103,7 +1431,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1112,7 +1444,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1121,7 +1457,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1145,7 +1485,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1154,7 +1498,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1', 'other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1163,7 +1511,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1172,7 +1524,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': [{ mars: 'other_value_1' }, { mars: 'other_value_2' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1198,7 +1554,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1210,7 +1570,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['value_1', 'value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1234,7 +1598,11 @@ describe('merge_missing_fields_with_source', () => { 'bar.keyword': ['bar_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1252,7 +1620,11 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1272,7 +1644,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1283,7 +1659,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1301,7 +1681,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1317,7 +1701,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1335,7 +1723,11 @@ describe('merge_missing_fields_with_source', () => { 'host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1351,7 +1743,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.host.hostname.keyword': ['hostname_other_value_keyword_1'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1363,7 +1759,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.bar.zed': ['zed_other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ foo: 'other_value_1', }); @@ -1380,7 +1780,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1395,7 +1799,11 @@ describe('merge_missing_fields_with_source', () => { '@timestamp': ['2023-02-10T10:15:50.000Z'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1410,15 +1818,15 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ - email: { - headers: { - 'x-test': 'from fields', - }, + 'email.headers': { + 'x-test': 'from fields', }, - // preserves conflicting keys if values contain empty objects - 'email.headers': {}, }); }); @@ -1431,7 +1839,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({ 'email.headers': { 'x-test': 'from fields', @@ -1448,7 +1860,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1461,7 +1877,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['from fields'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1474,7 +1894,11 @@ describe('merge_missing_fields_with_source', () => { 'a.b.c.d': ['1', '2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1487,7 +1911,11 @@ describe('merge_missing_fields_with_source', () => { 'a.b.c': [{ d: '3 ' }, { d: '4' }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1500,7 +1928,11 @@ describe('merge_missing_fields_with_source', () => { 'email.headers.x-test': ['a'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1525,7 +1957,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); @@ -1545,7 +1981,11 @@ describe('merge_missing_fields_with_source', () => { 'foo.zed.mars': ['other_value_2'], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1560,7 +2000,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual({}); }); @@ -1577,7 +2021,11 @@ describe('merge_missing_fields_with_source', () => { foo: [{ bar: ['single_value'], zed: ['single_value'] }], }; const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; - const merged = mergeMissingFieldsWithSource({ doc, ignoreFields: [] })._source; + const merged = mergeMissingFieldsWithSource({ + doc, + ignoreFields: {}, + ignoreFieldsRegexes: [], + })._source; expect(merged).toEqual(_source); }); }); @@ -1596,7 +2044,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: ['value.should.ignore', '/[_]+/'], + ignoreFields: { 'value.should.ignore': true }, + ignoreFieldsRegexes: ['/[_]+/'], })._source; expect(merged).toEqual({ foo: { @@ -1615,7 +2064,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: ['other.string', '/[z]+/'], // Neither of these two should match anything + ignoreFields: { 'other.string': true }, + ignoreFieldsRegexes: ['/[z]+/'], // Neither of these two should match anything })._source; expect(merged).toEqual({ foo: { @@ -1646,7 +2096,8 @@ describe('merge_missing_fields_with_source', () => { const doc: SignalSourceHit = { ...emptyEsResult(), _source: cloneDeep(_source), fields }; const merged = mergeMissingFieldsWithSource({ doc, - ignoreFields: [], + ignoreFields: {}, + ignoreFieldsRegexes: [], })._source; expect(merged).toEqual({ foo: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts index b4de7a9d4bb1a..ad9d8bd34705b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/strategies/merge_missing_fields_with_source.ts @@ -5,17 +5,13 @@ * 2.0. */ -import { get } from 'lodash/fp'; -import { set } from '@kbn/safer-lodash-set'; - -import type { SignalSource } from '../../../types'; -import { filterFieldEntries } from '../utils/filter_field_entries'; +import { filterFieldEntry } from '../utils/filter_field_entry'; import type { FieldsType, MergeStrategyFunction } from '../types'; import { recursiveUnboxingFields } from '../utils/recursive_unboxing_fields'; import { isTypeObject } from '../utils/is_type_object'; import { isNestedObject } from '../utils/is_nested_object'; -import { isPathValid } from '../utils/is_path_valid'; -import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map'; +import { robustGet, robustSet } from '../utils/robust_field_access'; +import { robustIsPathValid } from '../utils/is_path_valid'; /** * Merges only missing sections of "doc._source" with its "doc.fields" on a "best effort" basis. See ../README.md for more information @@ -26,36 +22,33 @@ import { buildFieldsKeyAsArrayMap } from '../utils/build_fields_key_as_array_map * it will not be added from fields. * @returns The two merged together in one object where we can */ -export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignoreFields }) => { +export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ + doc, + ignoreFields, + ignoreFieldsRegexes, +}) => { const source = doc._source ?? {}; const fields = doc.fields ?? {}; - const fieldEntries = Object.entries(fields); - const filteredEntries = filterFieldEntries(fieldEntries, ignoreFields); - const fieldsKeyMap = buildFieldsKeyAsArrayMap(source); + const fieldKeys = Object.keys(fields); + + fieldKeys.forEach((fieldKey) => { + const valueInMergedDocument = robustGet({ key: fieldKey, document: source }); + if (valueInMergedDocument == null && robustIsPathValid(fieldKey, source)) { + const fieldsValue = fields[fieldKey]; - const transformedSource = filteredEntries.reduce( - (merged, [fieldsKeyAsString, fieldsValue]: [string, FieldsType]) => { - const fieldsKey = fieldsKeyMap[fieldsKeyAsString] ?? fieldsKeyAsString; if ( - hasEarlyReturnConditions({ - fieldsValue, - fieldsKey, - merged, - }) + !hasEarlyReturnConditions(fieldsValue) && + filterFieldEntry([fieldKey, fieldsValue], fieldKeys, ignoreFields, ignoreFieldsRegexes) ) { - return merged; + const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); + return robustSet({ document: source, key: fieldKey, valueToSet: valueToMerge }); } - - const valueInMergedDocument = get(fieldsKey, merged); - const valueToMerge = recursiveUnboxingFields(fieldsValue, valueInMergedDocument); - return set(merged, fieldsKey, valueToMerge); - }, - { ...source } - ); + } + }); return { ...doc, - _source: transformedSource, + _source: source, }; }; @@ -70,21 +63,6 @@ export const mergeMissingFieldsWithSource: MergeStrategyFunction = ({ doc, ignor * @param merged The merge document which is what we are testing conditions against * @returns true if we should return early, otherwise false */ -const hasEarlyReturnConditions = ({ - fieldsValue, - fieldsKey, - merged, -}: { - fieldsValue: FieldsType; - fieldsKey: string[] | string; - merged: SignalSource; -}) => { - const valueInMergedDocument = get(fieldsKey, merged); - return ( - fieldsValue.length === 0 || - valueInMergedDocument !== undefined || - !isPathValid(fieldsKey, merged) || - isNestedObject(fieldsValue) || - isTypeObject(fieldsValue) - ); +const hasEarlyReturnConditions = (fieldsValue: FieldsType) => { + return fieldsValue.length === 0 || isNestedObject(fieldsValue) || isTypeObject(fieldsValue); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts index 68d0c2f047727..670fba02b2aec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/types.ts @@ -20,7 +20,9 @@ export type FieldsType = string[] | number[] | boolean[] | object[]; export type MergeStrategyFunction = ({ doc, ignoreFields, + ignoreFieldsRegexes, }: { doc: SignalSourceHit; - ignoreFields: string[]; + ignoreFields: Record; + ignoreFieldsRegexes: string[]; }) => SignalSourceHit; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts deleted file mode 100644 index 1495fd4b9aaca..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { buildFieldsKeyAsArrayMap } from './build_fields_key_as_array_map'; - -describe('buildFieldsKeyAsArrayMap()', () => { - it('returns primitive type if it passed as source', () => { - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(1)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(Infinity)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(NaN)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(false)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(null)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap(undefined)).toEqual({}); - // @ts-expect-error - expect(buildFieldsKeyAsArrayMap([])).toEqual({}); - }); - it('builds map for nested source', () => { - expect(buildFieldsKeyAsArrayMap({ a: 'b' })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ a: ['b'] })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ a: { b: { c: 1 } } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - 'a.b.c': ['a', 'b', 'c'], - }); - expect(buildFieldsKeyAsArrayMap({ a: { b: 'c' }, d: { e: 'f' } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - d: ['d'], - 'd.e': ['d', 'e'], - }); - }); - - it('builds map for flattened source', () => { - expect(buildFieldsKeyAsArrayMap({ a: 'b' })).toEqual({ a: ['a'] }); - expect(buildFieldsKeyAsArrayMap({ 'a.b.c': 1 })).toEqual({ 'a.b.c': ['a.b.c'] }); - expect(buildFieldsKeyAsArrayMap({ 'a.b': 'c', 'd.e': 'f' })).toEqual({ - 'a.b': ['a.b'], - 'd.e': ['d.e'], - }); - }); - - it('builds map for arrays in a path', () => { - expect(buildFieldsKeyAsArrayMap({ a: { b: [{ c: 1 }, { c: 2 }] } })).toEqual({ - a: ['a'], - 'a.b': ['a', 'b'], - 'a.b.c': ['a', 'b', 'c'], - }); - }); - - it('builds map for mixed nested and flattened in a path', () => { - expect( - buildFieldsKeyAsArrayMap({ - 'a.b': { c: { d: 1 } }, - }) - ).toEqual({ - 'a.b': ['a.b'], - 'a.b.c': ['a.b', 'c'], - 'a.b.c.d': ['a.b', 'c', 'd'], - }); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts deleted file mode 100644 index 33e3814104921..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/build_fields_key_as_array_map.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isPlainObject, isArray } from 'lodash'; - -import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; - -const isObjectTypeGuard = (value: SearchTypes): value is Record => { - return isPlainObject(value); -}; - -function traverseSource( - document: SearchTypes, - result: Record = {}, - prefix: string[] = [] -): Record { - if (prefix.length) { - result[prefix.join('.')] = prefix; - } - - if (isObjectTypeGuard(document)) { - for (const [key, value] of Object.entries(document)) { - const path = [...prefix, key]; - - traverseSource(value, result, path); - } - } else if (isArray(document)) { - // for array of primitive values we can call traverseSource once - if (isPlainObject(document[0])) { - traverseSource(document[0], result, prefix); - } else { - document.forEach((doc) => { - traverseSource(doc, result, prefix); - }); - } - } - - return result; -} - -/** - * takes object document and creates map of string field keys to array field keys - * source `{ 'a.b': { c: { d: 1 } } }` - * will result in map: `{ - * 'a.b': ['a.b'], - * 'a.b.c': ['a.b', 'c'], - * 'a.b.c.d': ['a.b', 'c', 'd'], - * }` - * @param document - Record - **/ -export function buildFieldsKeyAsArrayMap( - document: Record -): Record { - if (!isPlainObject(document)) { - return {}; - } - - return traverseSource(document); -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts deleted file mode 100644 index 7288e7d1e9b80..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { filterFieldEntries } from './filter_field_entries'; -import type { FieldsType } from '../types'; - -describe('filter_field_entries', () => { - beforeAll(() => { - jest.resetAllMocks(); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - /** Dummy test value */ - const dummyValue = ['value']; - - /** - * Get the return type of the mergeFieldsWithSource for TypeScript checks against expected - */ - type ReturnTypeFilterFieldEntries = ReturnType; - - test('returns a single valid fieldEntries as expected', () => { - const fieldEntries: Array<[string, FieldsType]> = [['foo.bar', dummyValue]]; - expect(filterFieldEntries(fieldEntries, [])).toEqual( - fieldEntries - ); - }); - - test('removes invalid dotted entries', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['.', dummyValue], - ['foo.bar', dummyValue], - ['..', dummyValue], - ['foo..bar', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.bar', dummyValue], - ]); - }); - - test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['foo', dummyValue], - ['foo.keyword', dummyValue], // <-- "foo.keyword" multi-field should be removed - ['bar.keyword', dummyValue], // <-- "bar.keyword" multi-field should be removed - ['bar', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo', dummyValue], - ['bar', dummyValue], - ]); - }); - - test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['host.name', dummyValue], - ['host.name.keyword', dummyValue], // <-- multi-field should be removed - ['host.hostname', dummyValue], - ['host.hostname.keyword', dummyValue], // <-- multi-field should be removed - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['host.name', dummyValue], - ['host.hostname', dummyValue], - ]); - }); - - test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['foo.host.name', dummyValue], - ['foo.host.name.keyword', dummyValue], // <-- multi-field should be removed - ['foo.host.hostname', dummyValue], - ['foo.host.hostname.keyword', dummyValue], // <-- multi-field should be removed - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.host.name', dummyValue], - ['foo.host.hostname', dummyValue], - ]); - }); - - test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['_ignored', dummyValue], - ['foo.host.hostname', dummyValue], - ]; - expect(filterFieldEntries(fieldEntries, [])).toEqual([ - ['foo.host.hostname', dummyValue], - ]); - }); - - test('ignores fields given strings and regular expressions in the ignoreFields list', () => { - const fieldEntries: Array<[string, FieldsType]> = [ - ['host.name', dummyValue], - ['user.name', dummyValue], // <-- string from ignoreFields should ignore this - ['host.hostname', dummyValue], - ['_odd.value', dummyValue], // <-- regular expression from ignoreFields should ignore this - ]; - expect( - filterFieldEntries(fieldEntries, ['user.name', '/[_]+/']) - ).toEqual([ - ['host.name', dummyValue], - ['host.hostname', dummyValue], - ]); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts new file mode 100644 index 0000000000000..b9ae033f425fa --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filterFieldEntry } from './filter_field_entry'; + +describe('filter_field_entry', () => { + beforeAll(() => { + jest.resetAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + /** Dummy test value */ + const dummyValue = ['value']; + + test('returns true for a valid field entry', () => { + const fieldsKeys: string[] = ['foo.bar']; + expect(filterFieldEntry(['foo.bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('returns false for invalid dotted entries', () => { + const fieldsKeys: string[] = ['.', 'foo.bar', '..', 'foo..bar']; + expect(filterFieldEntry(['.', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo.bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['..', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo..bar', dummyValue], fieldsKeys, {}, [])).toEqual(false); + }); + + test('removes multi-field values such "foo.keyword" mixed with "foo" and prefers just "foo" for 1st level', () => { + const fieldsKeys: string[] = [ + 'foo', + 'foo.keyword', // <-- "foo.keyword" multi-field should be removed + 'bar.keyword', // <-- "bar.keyword" multi-field should be removed + 'bar', + ]; + expect(filterFieldEntry(['foo', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['bar.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['bar', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('removes multi-field values such "host.name.keyword" mixed with "host.name" and prefers just "host.name" for 2nd level', () => { + const fieldsKeys: string[] = [ + 'host.name', + 'host.name.keyword', // <-- multi-field should be removed + 'host.hostname', + 'host.hostname.keyword', // <-- multi-field should be removed + ]; + expect(filterFieldEntry(['host.name', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['host.name.keyword', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['host.hostname.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + }); + + test('removes multi-field values such "foo.host.name.keyword" mixed with "foo.host.name" and prefers just "foo.host.name" for 3rd level', () => { + const fieldsKeys: string[] = [ + 'foo.host.name', + 'foo.host.name.keyword', // <-- multi-field should be removed + 'foo.host.hostname', + 'foo.host.hostname.keyword', // <-- multi-field should be removed + ]; + expect(filterFieldEntry(['foo.host.name', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.host.name.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + expect(filterFieldEntry(['foo.host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + expect(filterFieldEntry(['foo.host.hostname.keyword', dummyValue], fieldsKeys, {}, [])).toEqual( + false + ); + }); + + test('ignores fields of "_ignore", for eql bug https://github.com/elastic/elasticsearch/issues/77152', () => { + const fieldsKeys: string[] = ['_ignored', 'foo.host.hostname']; + expect(filterFieldEntry(['_ignored', dummyValue], fieldsKeys, {}, [])).toEqual(false); + expect(filterFieldEntry(['foo.host.hostname', dummyValue], fieldsKeys, {}, [])).toEqual(true); + }); + + test('ignores fields given strings and regular expressions in the ignoreFields list', () => { + const fieldsKeys: string[] = [ + 'host.name', + 'user.name', // <-- string from ignoreFields should ignore this + 'host.hostname', + '_odd.value', // <-- regular expression from ignoreFields should ignore this + ]; + expect( + filterFieldEntry(['host.name', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(true); + expect( + filterFieldEntry(['user.name', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(false); + expect( + filterFieldEntry(['host.hostname', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(true); + expect( + filterFieldEntry(['_odd.value', dummyValue], fieldsKeys, { 'user.name': true }, ['/[_]+/']) + ).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts similarity index 58% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts index 581dd7ffbff1a..3a18d36334070 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entries.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/filter_field_entry.ts @@ -18,23 +18,25 @@ import { isEqlBug77152 } from './is_eql_bug_77152'; * are invalid runtime field names. Also matches type objects such as geo-points and we ignore * those and don't try to merge those. * - * @param fieldEntries The field entries to filter + * @param fieldEntry The specific entry to test + * @param fieldEntries The full list of field entries, so we can check for multifields * @param ignoreFields Array of fields to ignore. If a value starts and ends with "/", such as: "/[_]+/" then the field will be treated as a regular expression. * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" - * @returns The field entries filtered + * @returns boolean Whether or not the field entry is valid for merging into _source */ -export const filterFieldEntries = ( - fieldEntries: Array<[string, FieldsType]>, - ignoreFields: string[] -): Array<[string, FieldsType]> => { - return fieldEntries.filter(([fieldsKey, fieldsValue]: [string, FieldsType]) => { - return ( - // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields - !isEqlBug77152(fieldsKey) && - !isIgnored(fieldsKey, ignoreFields) && - !isInvalidKey(fieldsKey) && - !isMultiField(fieldsKey, fieldEntries) && - !isTypeObject(fieldsValue) - ); - }); +export const filterFieldEntry = ( + fieldEntry: [string, FieldsType], + fieldsKeys: string[], + ignoreFields: Record, + ignoreFieldRegexes: string[] +): boolean => { + const [fieldsKey, fieldsValue] = fieldEntry; + // TODO: Look at not filtering this and instead transform it so it can be inserted correctly in the strategies which does an overwrite of everything from fields + return ( + !isEqlBug77152(fieldsKey) && + !isIgnored(fieldsKey, ignoreFields, ignoreFieldRegexes) && + !isInvalidKey(fieldsKey) && + !isMultiField(fieldsKey, fieldsKeys) && + !isTypeObject(fieldsValue) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts index 87b1097dd9bca..4c48b585167b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/index.ts @@ -5,7 +5,7 @@ * 2.0. */ export * from './array_in_path_exists'; -export * from './filter_field_entries'; +export * from './filter_field_entry'; export * from './is_array_of_primitives'; export * from './is_ignored'; export * from './is_invalid_key'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts index e4a7093ef127c..a4aa55cbabcb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.test.ts @@ -17,68 +17,72 @@ describe('is_ignored', () => { }); describe('string matching', () => { - test('it returns false if given an empty array', () => { - expect(isIgnored('simple.value', [])).toEqual(false); + test('it returns false if given an empty object and empty array', () => { + expect(isIgnored('simple.value', {}, [])).toEqual(false); }); test('it returns true if a simple string value matches', () => { - expect(isIgnored('simple.value', ['simple.value'])).toEqual(true); + expect(isIgnored('simple.value', { 'simple.value': true }, [])).toEqual(true); }); test('it returns false if a simple string value does not match', () => { - expect(isIgnored('simple', ['simple.value'])).toEqual(false); + expect(isIgnored('simple', { 'simple.value': true }, [])).toEqual(false); }); test('it returns true if a simple string value matches with two strings', () => { - expect(isIgnored('simple.value', ['simple.value', 'simple.second.value'])).toEqual(true); + expect( + isIgnored('simple.value', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(true); }); test('it returns true if a simple string value matches the second string', () => { - expect(isIgnored('simple.second.value', ['simple.value', 'simple.second.value'])).toEqual( - true - ); + expect( + isIgnored('simple.second.value', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(true); }); test('it returns false if a simple string value does not match two strings', () => { - expect(isIgnored('simple', ['simple.value', 'simple.second.value'])).toEqual(false); + expect( + isIgnored('simple', { 'simple.value': true, 'simple.second.value': true }, []) + ).toEqual(false); }); test('it returns true if mixed with a regular expression in the list', () => { - expect(isIgnored('simple', ['simple', '/[_]+/'])).toEqual(true); + expect(isIgnored('simple', { simple: true }, ['/[_]+/'])).toEqual(true); }); }); describe('regular expression matching', () => { test('it returns true if a simple regular expression matches', () => { - expect(isIgnored('_ignored', ['/[_]+/'])).toEqual(true); + expect(isIgnored('_ignored', {}, ['/[_]+/'])).toEqual(true); }); test('it returns false if a simple regular expression does not match', () => { - expect(isIgnored('simple', ['/[_]+/'])).toEqual(false); + expect(isIgnored('simple', {}, ['/[_]+/'])).toEqual(false); }); test('it returns true if a simple regular expression matches a longer string', () => { - expect(isIgnored('___ignored', ['/[_]+/'])).toEqual(true); + expect(isIgnored('___ignored', {}, ['/[_]+/'])).toEqual(true); }); test('it returns true if mixed with regular stings', () => { - expect(isIgnored('___ignored', ['simple', '/[_]+/'])).toEqual(true); + expect(isIgnored('___ignored', { simple: true }, ['/[_]+/'])).toEqual(true); }); test('it returns true with start anchor', () => { - expect(isIgnored('_ignored', ['simple', '/^[_]+/'])).toEqual(true); + expect(isIgnored('_ignored', { simple: true }, ['/^[_]+/'])).toEqual(true); }); test('it returns false with start anchor', () => { - expect(isIgnored('simple.something_', ['simple', '/^[_]+/'])).toEqual(false); + expect(isIgnored('simple.something_', { simple: true }, ['/^[_]+/'])).toEqual(false); }); test('it returns true with end anchor', () => { - expect(isIgnored('something_', ['simple', '/[_]+$/'])).toEqual(true); + expect(isIgnored('something_', { simple: true }, ['/[_]+$/'])).toEqual(true); }); test('it returns false with end anchor', () => { - expect(isIgnored('_something', ['simple', '/[_]+$/'])).toEqual(false); + expect(isIgnored('_something', { simple: true }, ['/[_]+$/'])).toEqual(false); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts index a418ce735626d..46638293bce17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_ignored.ts @@ -12,12 +12,13 @@ * If you have an object structure to ignore such as "{ a: { b: c: {} } } ", then you need to ignore it as the string "a.b.c" * @returns true if it is a field to ignore, otherwise false */ -export const isIgnored = (fieldsKey: string, ignoreFields: string[]): boolean => { - return ignoreFields.some((ignoreField) => { - if (ignoreField.startsWith('/') && ignoreField.endsWith('/')) { - return new RegExp(ignoreField.slice(1, -1)).test(fieldsKey); - } else { - return fieldsKey === ignoreField; - } - }); +export const isIgnored = ( + fieldsKey: string, + ignoreFields: Record, + ignoreFieldRegexes: string[] +): boolean => { + return ( + (ignoreFields[fieldsKey] ? true : false) || + ignoreFieldRegexes.some((regex) => new RegExp(regex.slice(1, -1)).test(fieldsKey)) + ); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts index a8050b600b464..8b85557ee521a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.test.ts @@ -16,25 +16,23 @@ describe('is_multifield', () => { jest.resetAllMocks(); }); - const dummyValue = ['value']; - test('it returns true if the string "foo.bar" is a multiField', () => { - expect(isMultiField('foo.bar', [['foo', dummyValue]])).toEqual(true); + expect(isMultiField('foo.bar', ['foo'])).toEqual(true); }); test('it returns false if the string "foo" is not a multiField', () => { - expect(isMultiField('foo', [['foo', dummyValue]])).toEqual(false); + expect(isMultiField('foo', ['foo'])).toEqual(false); }); test('it returns false if we have a sibling string and are not a multiField', () => { - expect(isMultiField('foo.bar', [['foo.mar', dummyValue]])).toEqual(false); + expect(isMultiField('foo.bar', ['foo.mar'])).toEqual(false); }); test('it returns true for a 3rd level match of being a sub-object. Runtime fields can have multiple layers of multiFields', () => { - expect(isMultiField('foo.mars.bar', [['foo', dummyValue]])).toEqual(true); + expect(isMultiField('foo.mars.bar', ['foo'])).toEqual(true); }); test('it returns true for a 3rd level match against a 2nd level sub-object. Runtime fields can have multiple layers of multiFields', () => { - expect(isMultiField('foo.mars.bar', [['foo.mars', dummyValue]])).toEqual(true); + expect(isMultiField('foo.mars.bar', ['foo.mars'])).toEqual(true); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts index e9e51f9f50389..e19ff40a32e90 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_multifield.ts @@ -5,8 +5,6 @@ * 2.0. */ -import type { FieldsType } from '../types'; - /** * Returns true if we are a multiField when passed in a fields entry and a fields key, * otherwise false. Notice that runtime fields can have multiple levels of multiFields which is kind a problem @@ -16,18 +14,15 @@ import type { FieldsType } from '../types'; * @param fieldEntries The entries to check against. * @returns True if we are a subObject, otherwise false. */ -export const isMultiField = ( - fieldsKey: string, - fieldEntries: Array<[string, FieldsType]> -): boolean => { +export const isMultiField = (fieldsKey: string, fieldsKeys: string[]): boolean => { const splitPath = fieldsKey.split('.'); return splitPath.some((_, index, array) => { if (index + 1 === array.length) { return false; } else { const newPath = [...array].splice(0, index + 1).join('.'); - return fieldEntries.some(([fieldKeyToCheck]) => { - return fieldKeyToCheck === newPath; + return fieldsKeys.some((key) => { + return key === newPath; }); } }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts index 39d215d4c5062..be4f35f1948f9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_objectlike_or_array_of_objectlikes.ts @@ -6,6 +6,7 @@ */ import { isObjectLike } from 'lodash/fp'; +import { isPlainObject } from 'lodash'; import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; /** @@ -23,3 +24,7 @@ export const isObjectLikeOrArrayOfObjectLikes = ( return isObjectLike(valueInMergedDocument); } }; + +export const isObjectTypeGuard = (value: SearchTypes): value is Record => { + return isPlainObject(value); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts index 896adb2326a8f..6cdac6c1662e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.test.ts @@ -5,138 +5,97 @@ * 2.0. */ -import { isPathValid } from './is_path_valid'; - -describe('isPathValid', () => { - test('not valid when empty array is key', () => { - expect(isPathValid([], {})).toEqual(false); - }); +import { robustIsPathValid } from './is_path_valid'; +describe('robustIsPathValid', () => { test('valid when empty string is key', () => { - expect(isPathValid('', {})).toEqual(true); - expect(isPathValid([''], {})).toEqual(true); + expect(robustIsPathValid('', {})).toEqual(true); }); test('valid when a path and empty object', () => { - expect(isPathValid(['a', 'b', 'c'], {})).toEqual(true); - expect(isPathValid('a.b.c', {})).toEqual(true); + expect(robustIsPathValid('a.b.c', {})).toEqual(true); }); test('not valid when a path and an array exists', () => { - expect(isPathValid(['a'], { a: [] })).toEqual(false); - expect(isPathValid('a', { a: [] })).toEqual(false); + expect(robustIsPathValid('a', { a: [] })).toEqual(false); }); test('not valid when a path and primitive value exists', () => { - expect(isPathValid(['a'], { a: 'test' })).toEqual(false); - expect(isPathValid(['a'], { a: 1 })).toEqual(false); - expect(isPathValid(['a'], { a: true })).toEqual(false); - - expect(isPathValid('a', { a: 'test' })).toEqual(false); - expect(isPathValid('a', { a: 1 })).toEqual(false); - expect(isPathValid('a', { a: true })).toEqual(false); + expect(robustIsPathValid('a', { a: 'test' })).toEqual(false); + expect(robustIsPathValid('a', { a: 1 })).toEqual(false); + expect(robustIsPathValid('a', { a: true })).toEqual(false); }); test('valid when a path and object value exists', () => { - expect(isPathValid(['a'], { a: {} })).toEqual(true); - - expect(isPathValid('a', { a: {} })).toEqual(true); + expect(robustIsPathValid('a', { a: {} })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: [] })).toEqual(false); - - expect(isPathValid('a.b', { a: [] })).toEqual(false); + expect(robustIsPathValid('a.b', { a: [] })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: 'test' })).toEqual(false); - expect(isPathValid(['a', 'b'], { a: 1 })).toEqual(false); - expect(isPathValid(['a', 'b'], { a: true })).toEqual(false); - - expect(isPathValid('a.b', { a: 'test' })).toEqual(false); - expect(isPathValid('a.b', { a: 1 })).toEqual(false); - expect(isPathValid('a.b', { a: true })).toEqual(false); + expect(robustIsPathValid('a.b', { a: 'test' })).toEqual(false); + expect(robustIsPathValid('a.b', { a: 1 })).toEqual(false); + expect(robustIsPathValid('a.b', { a: true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at level 1', () => { - expect(isPathValid(['a', 'b'], { a: {} })).toEqual(true); - - expect(isPathValid('a.b', { a: {} })).toEqual(true); + expect(robustIsPathValid('a.b', { a: {} })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at level 2', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: [] } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b': [] })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: [] } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b': [] })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': [] })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at level 2', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: 'test' } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: 1 } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: true } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b': true })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: 'test' } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: 1 } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: true } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b': true })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: 'test' } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: true } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at the last level 2', () => { - expect(isPathValid(['a', 'b'], { a: { b: {} } })).toEqual(true); - - expect(isPathValid('a.b', { a: { b: {} } })).toEqual(true); + expect(robustIsPathValid('a.b', { a: { b: {} } })).toEqual(true); }); test('not valid when a path and an array exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: [] } } })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: [] } } })).toEqual(false); }); test('not valid when a path and primitive value exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: 'test' } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: 1 } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: true } } })).toEqual(false); - expect(isPathValid(['a', 'b', 'c'], { 'a.b.c': true })).toEqual(false); - - expect(isPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false); - expect(isPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false); - expect(isPathValid('a.b.c', { 'a.b.c': true })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: 'test' } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: 1 } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { b: { c: true } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b.c': true })).toEqual(false); }); test('valid when a path and object value exists within the parent path at the last level 3', () => { - expect(isPathValid(['a', 'b', 'c'], { a: { b: { c: {} } } })).toEqual(true); - expect(isPathValid(['a', 'b', 'c'], { 'a.b.c': {} })).toEqual(true); - - expect(isPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true); - expect(isPathValid('a.b.c', { 'a.b.c': {} })).toEqual(true); + expect(robustIsPathValid('a.b.c', { a: { b: { c: {} } } })).toEqual(true); + expect(robustIsPathValid('a.b.c', { 'a.b.c': {} })).toEqual(true); }); - test('valid when any key has dot notation', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': {} } })).toEqual(true); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: {} } })).toEqual(true); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: {} } } })).toEqual(true); + test('valid when using dot notation', () => { + expect(robustIsPathValid('a.b.c', { a: { 'b.c': {} } })).toEqual(true); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: {} } })).toEqual(true); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: {} } } })).toEqual(true); }); - test('not valid when any key has dot notation and array is present in source on the last level', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': [] } })).toEqual(false); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: [] } })).toEqual(false); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: [] } } })).toEqual(false); + test('not valid when using dot notation and array is present in source on the last level', () => { + expect(robustIsPathValid('a.b.c', { a: { 'b.c': [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: [] } } })).toEqual(false); }); test('not valid when any key has dot notation and primitive value is present in source on the last level', () => { - expect(isPathValid(['a', 'b.c'], { a: { 'b.c': 1 } })).toEqual(false); - expect(isPathValid(['a.b', 'c'], { 'a.b': { c: 1 } })).toEqual(false); - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': { d: 1 } } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { a: { 'b.c': 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c', { 'a.b': { c: 1 } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': { d: 1 } } })).toEqual(false); }); test('not valid when any key has dot notation and array is present in source on level 2', () => { - expect(isPathValid(['a', 'b.c', 'd'], { a: { 'b.c': [] } })).toEqual(false); - expect(isPathValid(['a.b', 'c', 'd'], { 'a.b': { c: [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { a: { 'b.c': [] } })).toEqual(false); + expect(robustIsPathValid('a.b.c.d', { 'a.b': { c: [] } })).toEqual(false); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts index 6e193e237696d..f9c87088bd25b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/is_path_valid.ts @@ -5,32 +5,32 @@ * 2.0. */ -import { get, isPlainObject } from 'lodash/fp'; -import type { SignalSource } from '../../../types'; +import { isObjectTypeGuard } from './is_objectlike_or_array_of_objectlikes'; /** * Returns true if path in SignalSource object is valid * Path is valid if each field in hierarchy is object or undefined * Path is not valid if ANY of field in hierarchy is not object or undefined - * @param path in source to check within source - * @param source The source document + * The function is robust in that it can handle any mix of dot and nested notation in the document + * @param key Path (dot-notation) to check for validity + * @param document Document to search * @returns boolean */ -export const isPathValid = (path: string[] | string, source: SignalSource): boolean => { - if (path == null) { - return false; +export const robustIsPathValid = (key: string, document: Record): boolean => { + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (!isObjectTypeGuard(value)) { + return false; + } else { + return robustIsPathValid(splitKey.slice(i + 1).join('.'), value); + } + } } - const pathAsArray = typeof path === 'string' ? path.split('.') : path; - - if (pathAsArray.length === 0) { - return false; - } - - return pathAsArray.every((_, index, array) => { - const newPath = [...array].splice(0, index + 1); - // _.get won't retrieve value of flattened key 'a.b' when receives path ['a', 'b']. - // so we would try to call _.get with dot-notation path if array path results in undefined - const valueToCheck = get(newPath, source) ?? get(newPath.join('.'), source); - return valueToCheck === undefined || isPlainObject(valueToCheck); - }); + return true; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts new file mode 100644 index 0000000000000..646068b59eec7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { robustGet, robustSet } from './robust_field_access'; + +describe('robust field access', () => { + describe('get', () => { + it('fetches a value with basic key', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { b: { c: 'my-value' } } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value with mixed notation', () => { + expect(robustGet({ key: 'a.b.c', document: { 'a.b': { c: 'my-value' } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value with different mixed notation', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { 'b.c': 'my-value' } } })).toEqual( + 'my-value' + ); + }); + + it('fetches a value using only dot notation', () => { + expect(robustGet({ key: 'a.b.c', document: { 'a.b.c': 'my-value' } })).toEqual('my-value'); + }); + + it('returns undefined if the key does not exist', () => { + expect(robustGet({ key: 'a.b.c', document: { a: { b: 'my-value' } } })).toEqual(undefined); + }); + }); + + describe('set', () => { + it('sets a value with a basic key', () => { + expect(robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: {} })).toEqual({ + a: { b: { c: 'test-value' } }, + }); + }); + + it('sets a value inside an object at a dot notation path', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: { 'a.b': {} } }) + ).toEqual({ + 'a.b': { c: 'test-value' }, + }); + }); + + it('sets a value inside an object at a nested notation path', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-value', document: { a: { b: {} } } }) + ).toEqual({ + a: { b: { c: 'test-value' } }, + }); + }); + + it('sets a value and overwrites the existing value with dot notation', () => { + expect( + robustSet({ key: 'a.b.c', valueToSet: 'test-new', document: { 'a.b.c': 'test-original' } }) + ).toEqual({ + 'a.b.c': 'test-new', + }); + }); + + it('sets a value and overwrites the existing value with nested notation', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { a: { b: { c: 'test-original' } } }, + }) + ).toEqual({ + a: { b: { c: 'test-new' } }, + }); + }); + + it('sets a value and overwrites the existing value with mixed notation', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { 'a.b': { c: 'test-original' } }, + }) + ).toEqual({ + 'a.b': { c: 'test-new' }, + }); + }); + + it('sets a value and ignores non-object values on the path', () => { + expect( + robustSet({ + key: 'a.b.c', + valueToSet: 'test-new', + document: { 'a.b': 'test-ignore' }, + }) + ).toEqual({ + 'a.b': 'test-ignore', + a: { b: { c: 'test-new' } }, + }); + }); + + it('sets the value correctly if an object already exists at the path', () => { + expect( + robustSet({ + key: 'a.b', + valueToSet: 'test-new', + document: { 'a.b': {} }, + }) + ).toEqual({ 'a.b': 'test-new' }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts new file mode 100644 index 0000000000000..30d716896056d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { set } from '@kbn/safer-lodash-set'; + +import type { SearchTypes } from '../../../../../../../common/detection_engine/types'; +import { isObjectTypeGuard } from './is_objectlike_or_array_of_objectlikes'; + +/** + * Similar to lodash `get`, but instead of handling only pure dot or nested notation this function handles any mix of dot and nested notation + * + * Note: this function makes no attempt to handle arrays in the middle of the path because it's only used to fetch values based on paths from + * the `fields` option on search requests, which can never have arrays in the middle of the path + * @param key Path to field, in dot notation + * @param document Object to fetch value from + * @returns Fetched value or undefined + */ +export const robustGet = ({ + key, + document, +}: { + key: string; + document: Record; +}): SearchTypes => { + const fastPathValue = document[key]; + if (fastPathValue != null) { + return fastPathValue; + } + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length - 1; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (isObjectTypeGuard(value)) { + return robustGet({ key: splitKey.slice(i + 1).join('.'), document: value }); + } else { + return undefined; + } + } + } + return undefined; +}; + +/** + * Similar to lodash set, but instead of handling only pure dot or nested notation this function handles any mix of dot and nested notation + * @param key Path to field, in dot notation + * @param valueToSet Value to insert into document + * @param document Object to insert value into + * @returns Updated document + */ +export const robustSet = >({ + key, + valueToSet, + document, +}: { + key: string; + valueToSet: SearchTypes; + document: T; +}) => { + const splitKey = key.split('.'); + let tempKey = splitKey[0]; + for (let i = 0; i < splitKey.length - 1; i++) { + if (i > 0) { + tempKey += `.${splitKey[i]}`; + } + const value = document[tempKey]; + if (value != null) { + if (isObjectTypeGuard(value)) { + robustSet({ key: splitKey.slice(i + 1).join('.'), valueToSet, document: value }); + return document; + } + } + } + return set(document, key, valueToSet); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts index 5baa84ff913b2..4d47c279f1495 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.test.ts @@ -16,13 +16,13 @@ import { } from '@kbn/rule-data-utils'; import type { CompleteRule, ThreatRuleParams } from '../../rule_schema'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; -jest.mock('../factories/utils/build_bulk_body', () => ({ buildBulkBody: jest.fn() })); +jest.mock('../factories/utils/transform_hit_to_alert', () => ({ transformHitToAlert: jest.fn() })); -const buildBulkBodyMock = buildBulkBody as jest.Mock; +const transformHitToAlertMock = transformHitToAlert as jest.Mock; const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); @@ -118,7 +118,7 @@ const wrappedParams = { }; describe('wrapSuppressedAlerts', () => { - buildBulkBodyMock.mockReturnValue({ 'mock-props': true }); + transformHitToAlertMock.mockReturnValue({ 'mock-props': true }); it('should wrap event with alert fields and correctly set suppression fields', () => { const expectedTimestamp = '2020-10-28T06:30:00.000Z'; @@ -137,10 +137,10 @@ describe('wrapSuppressedAlerts', () => { ...wrappedParams, }); - expect(buildBulkBodyMock).toHaveBeenCalledWith( - 'default', - wrappedParams.completeRule, - { + expect(transformHitToAlertMock).toHaveBeenCalledWith({ + spaceId: 'default', + completeRule: wrappedParams.completeRule, + doc: { fields: { '@timestamp': [expectedTimestamp], 'agent.name': ['agent-0'], @@ -149,16 +149,17 @@ describe('wrapSuppressedAlerts', () => { _id: '1', _index: 'test*', }, - 'missingFields', - [], - true, - wrappedParams.buildReasonMessage, - ['test*'], - undefined, + mergeStrategy: 'missingFields', + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, + buildReasonMessage: wrappedParams.buildReasonMessage, + indicesToQuery: ['test*'], + alertTimestampOverride: undefined, ruleExecutionLogger, - expect.any(String), - 'public-url-mock' - ); + alertUuid: expect.any(String), + publicBaseUrl: 'public-url-mock', + }); expect(wrappedAlerts[0]._source).toEqual( expect.objectContaining({ [ALERT_SUPPRESSION_TERMS]: [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 70fee20116fc4..87d1e64d8ece9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -23,7 +23,7 @@ import type { ThreatRuleParams, } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; import { generateId } from './utils'; @@ -77,20 +77,21 @@ export const wrapSuppressedAlerts = ({ const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); - const baseAlert: BaseFieldsLatest = buildBulkBody( + const baseAlert: BaseFieldsLatest = transformHitToAlert({ spaceId, completeRule, - event, + doc: event, mergeStrategy, - [], - true, + ignoreFields: {}, + ignoreFieldsRegexes: [], + applyOverrides: true, buildReasonMessage, indicesToQuery, alertTimestampOverride, ruleExecutionLogger, - id, - publicBaseUrl - ); + alertUuid: id, + publicBaseUrl, + }); return { _id: id, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts index f08893fe4ded4..9c8d50631cc53 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/ess_specific_index_logic/alerts_compatibility.ts @@ -300,9 +300,11 @@ export default ({ getService }: FtrProviderContext) => { type: 'logs', dataset: 'elastic_agent.filebeat', }, - 'event.agent_id_status': 'verified', - 'event.ingested': '2022-03-23T16:50:28.994Z', - 'event.dataset': 'elastic_agent.filebeat', + event: { + agent_id_status: 'verified', + ingested: '2022-03-23T16:50:28.994Z', + dataset: 'elastic_agent.filebeat', + }, 'event.kind': 'signal', 'kibana.alert.ancestors': [ { @@ -466,9 +468,11 @@ export default ({ getService }: FtrProviderContext) => { type: 'logs', dataset: 'elastic_agent.filebeat', }, - 'event.agent_id_status': 'verified', - 'event.ingested': '2022-03-23T16:50:28.994Z', - 'event.dataset': 'elastic_agent.filebeat', + event: { + agent_id_status: 'verified', + ingested: '2022-03-23T16:50:28.994Z', + dataset: 'elastic_agent.filebeat', + }, 'event.kind': 'signal', 'kibana.alert.ancestors': [ { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts index cd67498c421c9..c0060912d6a52 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/ip_array.ts @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverless @serverlessQA @ess Rule exception operators for data type ip', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type ip array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/ip_as_array'); }); @@ -62,10 +62,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.1', '127.0.0.2', '127.0.0.3', '127.0.0.4'], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -86,9 +86,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -116,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('should filter 3 ips if all 3 are set as exceptions', async () => { @@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); it('should filter a CIDR range of "127.0.0.1/30"', async () => { @@ -171,9 +171,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -193,7 +193,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); }); @@ -305,9 +305,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -327,7 +327,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('should filter 3 ips if all 3 are set as exceptions', async () => { @@ -346,7 +346,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); }); @@ -408,7 +408,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); }); @@ -458,9 +458,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); expect(ips).to.eql([ - [], ['127.0.0.5', null, '127.0.0.6', '127.0.0.7'], ['127.0.0.8', '127.0.0.9', '127.0.0.10'], + undefined, ]); }); @@ -484,7 +484,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('will return 1 result if we have a list that includes all ips', async () => { @@ -513,7 +513,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips.flat(10)).to.eql([]); + expect(ips.flat(10)).to.eql([undefined]); }); it('will return 2 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => { @@ -551,7 +551,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); it('will return 2 results if we have a list which contains the range syntax of "127.0.0.1-127.0.0.7', async () => { @@ -582,7 +582,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const ips = alertsOpen.hits.hits.map((hit) => hit._source?.ip).sort(); - expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]); + expect(ips).to.eql([['127.0.0.8', '127.0.0.9', '127.0.0.10'], undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts index c4d50860ea5c2..016d83e1587be 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/keyword_array.ts @@ -33,7 +33,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverles @serverlessQA @ess Rule exception operators for data type keyword', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type keyword array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/keyword_as_array'); }); @@ -65,10 +65,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -89,9 +89,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -154,7 +154,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -241,9 +241,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -263,7 +263,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -282,7 +282,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -344,7 +344,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -404,10 +404,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -442,9 +442,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -469,9 +469,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -495,7 +495,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('will return only the empty array for results if we have a list that includes all keyword', async () => { @@ -524,7 +524,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -653,9 +653,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -675,7 +675,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits).to.eql([[], ['word five', null, 'word six', 'word seven']]); + expect(hits).to.eql([['word five', null, 'word six', 'word seven'], undefined]); }); it('should filter 3 keyword if all 3 are set as exceptions', async () => { @@ -694,7 +694,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.keyword).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts index bfc528cf6ad6b..e0271b7ddb934 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/text_array.ts @@ -32,7 +32,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - describe('@serverless @serverlessQA @ess Rule exception operators for data type text', () => { + describe('@serverless @serverlessQA @ess Rule exception operators for data type text array', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/text_as_array'); }); @@ -62,10 +62,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -86,9 +86,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -116,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 text if all 3 are set as exceptions', async () => { @@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -238,9 +238,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -260,7 +260,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('should filter 3 text if all 3 are set as exceptions', async () => { @@ -279,7 +279,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -341,7 +341,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); @@ -401,10 +401,10 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], ['word one', 'word two', 'word three', 'word four'], + undefined, ]); }); @@ -439,9 +439,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -466,9 +466,9 @@ export default ({ getService }: FtrProviderContext) => { const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); expect(hits).to.eql([ - [], ['word eight', 'word nine', 'word ten'], ['word five', null, 'word six', 'word seven'], + undefined, ]); }); @@ -492,7 +492,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 2, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]); + expect(hits).to.eql([['word eight', 'word nine', 'word ten'], undefined]); }); it('will return only the empty array for results if we have a list that includes all text', async () => { @@ -521,7 +521,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForAlertsToBePresent(supertest, log, 1, [id]); const alertsOpen = await getAlertsById(supertest, log, id); const hits = alertsOpen.hits.hits.map((hit) => hit._source?.text).sort(); - expect(hits.flat(10)).to.eql([]); + expect(hits.flat(10)).to.eql([undefined]); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts index 608b7da833b6c..0b39a7287bacb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts @@ -25,7 +25,7 @@ import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { Rule } from '@kbn/alerting-plugin/common'; import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import moment from 'moment'; -import { orderBy } from 'lodash'; +import { get, orderBy } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; import { @@ -2830,7 +2830,7 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[0]?._source?.destination).toEqual( expect.objectContaining({ domain: 'aaa.stage.11111111.hello' }) ); - expect(previewAlerts[0]?._source?.['event.dataset']).toEqual('network_traffic.tls'); + expect(get(previewAlerts[0]?._source, 'event.dataset')).toEqual('network_traffic.tls'); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts index 6a59d5244b88c..53b2843399c62 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts @@ -158,12 +158,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'changed-audit-configuration', category: 'configuration', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -300,12 +300,11 @@ export default ({ getService }: FtrProviderContext) => { }, }, }, - ...flattenWithPrefix('event', { + event: { action: 'changed-audit-configuration', category: 'configuration', module: 'auditd', - kind: 'signal', - }), + }, service: { type: 'auditd', }, @@ -427,12 +426,11 @@ export default ({ getService }: FtrProviderContext) => { }, cloud: { instance: { id: '133551048' }, provider: 'digitalocean', region: 'ams3' }, ecs: { version: '1.0.0-beta2' }, - ...flattenWithPrefix('event', { + event: { action: 'changed-promiscuous-mode-on-device', category: 'anomoly', module: 'auditd', - kind: 'signal', - }), + }, host: { architecture: 'x86_64', containerized: false, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts index b688f53e288b1..0dd5a93bb9e60 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts @@ -282,12 +282,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'error', category: 'user-login', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -464,12 +464,12 @@ export default ({ getService }: FtrProviderContext) => { ecs: { version: '1.0.0-beta2', }, - ...flattenWithPrefix('event', { + event: { action: 'error', category: 'user-login', module: 'auditd', - kind: 'signal', - }), + }, + 'event.kind': 'signal', host: { architecture: 'x86_64', containerized: false, @@ -749,7 +749,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -774,7 +773,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -826,7 +824,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -844,7 +841,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -901,7 +897,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -924,7 +919,6 @@ export default ({ getService }: FtrProviderContext) => { // threat.indicator.matched data). That's the case with the // first and third indicators matched, here. { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -943,7 +937,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1003,7 +996,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[0].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1024,7 +1016,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1042,7 +1033,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1063,7 +1053,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[1].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1172,7 +1161,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1197,7 +1185,6 @@ export default ({ getService }: FtrProviderContext) => { { enrichments: [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1274,7 +1261,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threat.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1292,7 +1278,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1390,7 +1375,6 @@ export default ({ getService }: FtrProviderContext) => { }>; assertContains(threatTerm.enrichments, [ { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1413,7 +1397,6 @@ export default ({ getService }: FtrProviderContext) => { // threat.indicator.matched data). That's the case with the // first and third indicators matched, here. { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1432,7 +1415,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1505,7 +1487,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[0].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', @@ -1526,7 +1507,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1544,7 +1524,6 @@ export default ({ getService }: FtrProviderContext) => { }, }, { - feed: {}, indicator: { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', @@ -1565,7 +1544,6 @@ export default ({ getService }: FtrProviderContext) => { assertContains(threats[1].enrichments, [ { - feed: {}, indicator: { description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts index dfb5b3492b17a..e274366e54aa7 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/new_terms.ts @@ -160,14 +160,17 @@ export default ({ getService }: FtrProviderContext) => { name: expect.stringMatching(/(root|bob)/), terminal: 'pts/0', }, - 'event.action': 'user_login', - 'event.category': 'authentication', - 'event.dataset': 'login', + event: { + action: 'user_login', + category: 'authentication', + dataset: 'login', + + module: 'system', + origin: '/var/log/wtmp', + outcome: 'success', + type: 'authentication_success', + }, 'event.kind': 'signal', - 'event.module': 'system', - 'event.origin': '/var/log/wtmp', - 'event.outcome': 'success', - 'event.type': 'authentication_success', 'kibana.alert.original_time': '2019-02-19T20:42:08.230Z', 'kibana.alert.ancestors': [ { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts index 15ea0c02b6bc2..a4c59313389e3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/non_ecs_fields.ts @@ -213,8 +213,7 @@ export default ({ getService }: FtrProviderContext) => { expect(errors).toEqual([]); - // event properties getting flattened - expect(alertSource).toHaveProperty(['event.created'], validDates); + expect(alertSource).toHaveProperty(['event', 'created'], validDates); }); // source threat.enrichments is keyword, ECS mapping for threat.enrichments is nested diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts index e97b9436612aa..79b118b3b3d94 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/const_keyword.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, ThresholdRuleCreateProps, @@ -73,7 +74,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -107,7 +108,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts index 5ef047ecd2de8..1be8274079663 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, @@ -60,7 +61,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -81,7 +82,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 4, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts index c0b42709847be..9d63a84a1cbe5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/keyword_family/keyword_mixed_with_const.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import { get } from 'lodash'; import { EqlRuleCreateProps, ThresholdRuleCreateProps, @@ -75,7 +76,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 8, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1', @@ -113,7 +114,7 @@ export default ({ getService }: FtrProviderContext) => { await waitForRuleSuccess({ supertest, log, id }); await waitForAlertsToBePresent(supertest, log, 8, [id]); const alertsOpen = await getAlertsById(supertest, log, id); - const hits = alertsOpen.hits.hits.map((hit) => hit._source?.['event.dataset']).sort(); + const hits = alertsOpen.hits.hits.map((hit) => get(hit, '_source.event.dataset')).sort(); expect(hits).to.eql([ 'dataset_name_1', 'dataset_name_1',