diff --git a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts index dc2497bb19cec..5152132cda4bb 100644 --- a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts +++ b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts @@ -291,6 +291,29 @@ export const eventDetailsFormattedFields = [ originalValue: [`{"lon":118.7778,"lat":32.0617}`], values: [`{"lon":118.7778,"lat":32.0617}`], }, + { + category: 'threat', + field: 'threat.enrichments', + isObjectArray: true, + originalValue: [ + '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', + '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + ], + values: [ + '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', + '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + ], + }, { category: 'threat', field: 'threat.enrichments.matched.field', @@ -376,27 +399,4 @@ export const eventDetailsFormattedFields = [ originalValue: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'], values: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'], }, - { - category: 'threat', - field: 'threat.enrichments', - isObjectArray: true, - originalValue: [ - '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', - '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - ], - values: [ - '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', - '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - ], - }, ]; diff --git a/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts b/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts deleted file mode 100644 index 33d0c226fc44e..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { EventHit } from '../search_strategy'; -import { getDataFromFieldsHits, getDataSafety } from './field_formatters'; -import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid'; - -describe('Events Details Helpers', () => { - const fields: EventHit['fields'] = eventHit.fields; - const resultFields = eventDetailsFormattedFields; - describe('#getDataFromFieldsHits', () => { - it('happy path', () => { - const result = getDataFromFieldsHits(fields); - expect(result).toEqual(resultFields); - }); - it('lets get weird', () => { - const whackFields = { - 'crazy.pants': [ - { - 'matched.field': ['matched_field'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['yourself'], - type: ['custom'], - 'matched.atomic': ['matched_atomic'], - lazer: [ - { - 'great.field': ['grrrrr'], - lazer: [ - { - lazer: [ - { - cool: true, - lazer: [ - { - lazer: [ - { - lazer: [ - { - lazer: [ - { - whoa: false, - }, - ], - }, - ], - }, - ], - }, - ], - }, - ], - }, - { - lazer: [ - { - cool: false, - }, - ], - }, - ], - }, - { - 'great.field': ['grrrrr_2'], - }, - ], - }, - ], - }; - const whackResultFields = [ - { - category: 'crazy', - field: 'crazy.pants', - values: [ - '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}', - ], - originalValue: [ - '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}', - ], - isObjectArray: true, - }, - ]; - const result = getDataFromFieldsHits(whackFields); - expect(result).toEqual(whackResultFields); - }); - }); - it('#getDataSafety', async () => { - const result = await getDataSafety(getDataFromFieldsHits, fields); - expect(result).toEqual(resultFields); - }); -}); diff --git a/x-pack/plugins/security_solution/common/utils/field_formatters.ts b/x-pack/plugins/security_solution/common/utils/field_formatters.ts deleted file mode 100644 index 1d8c05ec9ccf7..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/field_formatters.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; -import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; -import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { isEmpty } from 'lodash/fp'; -import { ENRICHMENT_DESTINATION_PATH } from '../constants'; - -import type { Fields, TimelineEventsDetailsItem } from '../search_strategy'; -import { toObjectArrayOfStrings, toStringArray } from './to_array'; - -export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags']; - -export const getFieldCategory = (field: string): string => { - const fieldCategory = field.split('.')[0]; - if (!isEmpty(fieldCategory) && baseCategoryFields.includes(fieldCategory)) { - return 'base'; - } - return fieldCategory; -}; - -export const formatGeoLocation = (item: unknown[]) => { - const itemGeo = item.length > 0 ? (item[0] as { coordinates: number[] }) : null; - if (itemGeo != null && !isEmpty(itemGeo.coordinates)) { - try { - return toStringArray({ - lon: itemGeo.coordinates[0], - lat: itemGeo.coordinates[1], - }); - } catch { - return toStringArray(item); - } - } - return toStringArray(item); -}; - -export const isGeoField = (field: string) => - field.includes('geo.location') || field.includes('geoip.location'); - -export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) => - prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH; - -export const getDataFromFieldsHits = ( - fields: Fields, - prependField?: string, - prependFieldCategory?: string -): TimelineEventsDetailsItem[] => - Object.keys(fields).reduce((accumulator, field) => { - const item: unknown[] = fields[field]; - - const fieldCategory = - prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); - if (isGeoField(field)) { - return [ - ...accumulator, - { - category: fieldCategory, - field, - values: formatGeoLocation(item), - originalValue: formatGeoLocation(item), - isObjectArray: true, // important for UI - }, - ]; - } - - const objArrStr = toObjectArrayOfStrings(item); - const strArr = objArrStr.map(({ str }) => str); - const isObjectArray = objArrStr.some((o) => o.isObjectArray); - const dotField = prependField ? `${prependField}.${field}` : field; - - // return simple field value (non-esc object, non-array) - if ( - !isObjectArray || - Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...legacyExperimentalFieldMap }).find( - (ecsField) => ecsField === field - ) === undefined - ) { - return [ - ...accumulator, - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ]; - } - - const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField) - ? [ - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ] - : []; - - // format nested fields - const nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, prependField, fieldCategory); - - // combine duplicate fields - const flat: Record = [ - ...accumulator, - ...nestedFields, - ...threatEnrichmentObject, - ].reduce( - (acc, f) => ({ - ...acc, - // acc/flat is hashmap to determine if we already have the field or not without an array iteration - // its converted back to array in return with Object.values - ...(acc[f.field] != null - ? { - [f.field]: { - ...f, - originalValue: acc[f.field].originalValue.includes(f.originalValue[0]) - ? acc[f.field].originalValue - : [...acc[f.field].originalValue, ...f.originalValue], - values: acc[f.field].values?.includes(f.values?.[0] || '') - ? acc[f.field].values - : [...(acc[f.field].values || []), ...(f.values || [])], - }, - } - : { [f.field]: f }), - }), - {} as Record - ); - - return Object.values(flat); - }, []); - -export const getDataSafety = (fn: (args: A) => T, args: A): Promise => - new Promise((resolve) => setTimeout(() => resolve(fn(args)))); diff --git a/x-pack/plugins/security_solution/common/utils/to_array.ts b/x-pack/plugins/security_solution/common/utils/to_array.ts deleted file mode 100644 index b6945708ff0db..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/to_array.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const toArray = (value: T | T[] | null | undefined): T[] => - Array.isArray(value) ? value : value == null ? [] : [value]; - -export const toStringArray = (value: T | T[] | null): string[] => { - if (Array.isArray(value)) { - return value.reduce((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, v.toString()]; - case 'object': - try { - return [...acc, JSON.stringify(v)]; - } catch { - return [...acc, 'Invalid Object']; - } - case 'string': - return [...acc, v]; - default: - return [...acc, `${v}`]; - } - } - return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [JSON.stringify(value)]; - } catch { - return ['Invalid Object']; - } - } else { - return [`${value}`]; - } -}; - -export const toObjectArrayOfStrings = ( - value: T | T[] | null -): Array<{ - str: string; - isObjectArray?: boolean; -}> => { - if (Array.isArray(value)) { - return value.reduce< - Array<{ - str: string; - isObjectArray?: boolean; - }> - >((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, { str: v.toString() }]; - case 'object': - try { - return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value - } catch { - return [...acc, { str: 'Invalid Object' }]; - } - case 'string': - return [...acc, { str: v }]; - default: - return [...acc, { str: `${v}` }]; - } - } - return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [{ str: JSON.stringify(value), isObjectArray: true }]; - } catch { - return [{ str: 'Invalid Object' }]; - } - } else { - return [{ str: `${value}` }]; - } -}; diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx index 2f5bd6510430b..fefacf3e7fc14 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx @@ -7,11 +7,11 @@ import { groupBy, isObject } from 'lodash'; import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common'; +import { getDataFromFieldsHits } from '@kbn/timelines-plugin/common'; import { i18n } from '@kbn/i18n'; import type { ThreatDetailsRow } from '../../left/components/threat_details_view_enrichment_accordion'; import type { CtiEnrichment, EventFields } from '../../../../../common/search_strategy'; import { isValidEventField } from '../../../../../common/search_strategy'; -import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters'; import { DEFAULT_INDICATOR_SOURCE_PATH, ENRICHMENT_DESTINATION_PATH, diff --git a/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts b/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts index f44ad77e67929..d80be5f4a421b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts @@ -7,9 +7,7 @@ import { mapValues, isObject, isArray } from 'lodash/fp'; import { set } from '@kbn/safer-lodash-set'; - -import { toArray } from '../../../common/utils/to_array'; -import { isGeoField } from '../../../common/utils/field_formatters'; +import { toArray, isGeoField } from '@kbn/timelines-plugin/common'; export const mapObjectValuesToStringArray = (object: object): object => mapValues((o) => { diff --git a/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts b/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts index f40edfc5914df..baed5b3c35605 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts @@ -6,7 +6,7 @@ */ import { set } from '@kbn/safer-lodash-set'; import { get, isEmpty } from 'lodash/fp'; -import { toObjectArrayOfStrings } from '../../../common/utils/to_array'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; export function getFlattenedFields( fields: string[], diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts index 8707f10ed01cb..61ab1c5bca583 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts @@ -8,12 +8,12 @@ import { set } from '@kbn/safer-lodash-set/fp'; import { get, has } from 'lodash/fp'; import { hostFieldsMap } from '@kbn/securitysolution-ecs'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import type { HostAggEsItem, HostsEdges, HostValue, } from '../../../../../../common/search_strategy/security_solution/hosts'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; export const HOSTS_FIELDS: readonly string[] = [ '_id', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts index 94d45b2b63e0e..cc1a084f6b5a7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts @@ -13,6 +13,7 @@ import type { SavedObjectsClientContract, } from '@kbn/core/server'; import { hostFieldsMap } from '@kbn/securitysolution-ecs'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import { Direction } from '../../../../../../common/search_strategy/common'; import type { AggregationRequest, @@ -22,7 +23,6 @@ import type { HostItem, HostValue, } from '../../../../../../common/search_strategy/security_solution/hosts'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; import type { EndpointAppContext } from '../../../../../endpoint/types'; import { getPendingActionsSummary } from '../../../../../endpoint/services'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts index 06452c915009c..b92b67f244ebe 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts @@ -8,7 +8,7 @@ import { get, getOr, isEmpty } from 'lodash/fp'; import { set } from '@kbn/safer-lodash-set/fp'; import { sourceFieldsMap, hostFieldsMap } from '@kbn/securitysolution-ecs'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import type { AuthenticationsEdges, AuthenticationHit, diff --git a/x-pack/plugins/timelines/common/index.ts b/x-pack/plugins/timelines/common/index.ts index 0a96a22fb2679..3e335a7edd278 100644 --- a/x-pack/plugins/timelines/common/index.ts +++ b/x-pack/plugins/timelines/common/index.ts @@ -67,3 +67,5 @@ export type { } from './search_strategy'; export { Direction, EntityType, EMPTY_BROWSER_FIELDS } from './search_strategy'; + +export { getDataFromFieldsHits, toArray, isGeoField, toObjectArrayOfStrings } from './utils'; diff --git a/x-pack/plugins/timelines/common/utils/field_formatters.test.ts b/x-pack/plugins/timelines/common/utils/field_formatters.test.ts index 43babd374d991..46eb77c8f7f32 100644 --- a/x-pack/plugins/timelines/common/utils/field_formatters.test.ts +++ b/x-pack/plugins/timelines/common/utils/field_formatters.test.ts @@ -7,7 +7,7 @@ import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid'; import { EventHit } from '../search_strategy'; -import { getDataFromFieldsHits, getDataSafety } from './field_formatters'; +import { getDataFromFieldsHits } from './field_formatters'; describe('Events Details Helpers', () => { const fields: EventHit['fields'] = eventHit.fields; @@ -84,7 +84,7 @@ describe('Events Details Helpers', () => { }, ]; const result = getDataFromFieldsHits(whackFields); - expect(result).toEqual(whackResultFields); + expect(result).toMatchObject(whackResultFields); }); it('flattens alert parameters', () => { const ruleParameterFields = { @@ -191,7 +191,7 @@ describe('Events Details Helpers', () => { ]; const result = getDataFromFieldsHits(ruleParameterFields); - expect(result).toEqual(ruleParametersResultFields); + expect(result).toMatchObject(ruleParametersResultFields); }); it('get data from threat enrichments', () => { @@ -546,6 +546,17 @@ describe('Events Details Helpers', () => { originalValue: ['495ad7a7-316e-4544-8a0f-9c098daee76e'], values: ['495ad7a7-316e-4544-8a0f-9c098daee76e'], }, + { + category: 'threat', + field: 'threat.enrichments', + isObjectArray: true, + originalValue: [ + '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', + ], + values: [ + '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', + ], + }, { category: 'threat', field: 'threat.enrichments.matched.field', @@ -581,25 +592,9 @@ describe('Events Details Helpers', () => { originalValue: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], values: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], }, - { - category: 'threat', - field: 'threat.enrichments', - isObjectArray: true, - originalValue: [ - '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', - ], - values: [ - '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', - ], - }, ]; const result = getDataFromFieldsHits(data); - expect(result).toEqual(ruleParametersResultFields); + expect(result).toMatchObject(ruleParametersResultFields); }); }); - - it('#getDataSafety', async () => { - const result = await getDataSafety(getDataFromFieldsHits, fields); - expect(result).toEqual(resultFields); - }); }); diff --git a/x-pack/plugins/timelines/common/utils/field_formatters.ts b/x-pack/plugins/timelines/common/utils/field_formatters.ts index c9292987f59b2..2e3785633bc3f 100644 --- a/x-pack/plugins/timelines/common/utils/field_formatters.ts +++ b/x-pack/plugins/timelines/common/utils/field_formatters.ts @@ -8,9 +8,15 @@ import { isEmpty } from 'lodash/fp'; import { ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; -import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; -import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; +import { + ecsFieldMap, + EcsFieldMap, +} from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; +import { + technicalRuleFieldMap, + TechnicalRuleFieldMap, +} from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; +import { legacyExperimentalFieldMap, ExperimentalRuleFieldMap } from '@kbn/alerts-as-data-utils'; import { Fields, TimelineEventsDetailsItem } from '../search_strategy'; import { toObjectArrayOfStrings, toStringArray } from './to_array'; import { ENRICHMENT_DESTINATION_PATH } from '../constants'; @@ -51,117 +57,141 @@ export const isRuleParametersFieldOrSubfield = (field: string, prependField?: st export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) => prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH; +// Helper functions +const createFieldItem = ( + fieldCategory: string, + field: string, + values: string[], + isObjectArray: boolean +): TimelineEventsDetailsItem => ({ + category: fieldCategory, + field, + values, + originalValue: values, + isObjectArray, +}); + +const processGeoField = ( + field: string, + item: unknown[], + fieldCategory: string +): TimelineEventsDetailsItem => { + const formattedLocation = formatGeoLocation(item); + return createFieldItem(fieldCategory, field, formattedLocation, true); +}; + +const processSimpleField = ( + dotField: string, + strArr: string[], + isObjectArray: boolean, + fieldCategory: string +): TimelineEventsDetailsItem => createFieldItem(fieldCategory, dotField, strArr, isObjectArray); + +const processNestedFields = ( + item: unknown, + dotField: string, + fieldCategory: string, + prependDotField: boolean +): TimelineEventsDetailsItem[] => { + if (Array.isArray(item)) { + return item.flatMap((curr) => + getDataFromFieldsHits(curr as Fields, prependDotField ? dotField : undefined, fieldCategory) + ); + } + + return getDataFromFieldsHits( + item as Fields, + prependDotField ? dotField : undefined, + fieldCategory + ); +}; + +type DisjointFieldNames = 'ecs.version' | 'event.action' | 'event.kind' | 'event.original'; + +// Memoized field maps +const fieldMaps: EcsFieldMap & + Omit & + ExperimentalRuleFieldMap = { + ...technicalRuleFieldMap, + ...ecsFieldMap, + ...legacyExperimentalFieldMap, +}; + export const getDataFromFieldsHits = ( fields: Fields, prependField?: string, prependFieldCategory?: string -): TimelineEventsDetailsItem[] => - Object.keys(fields).reduce((accumulator, field) => { +): TimelineEventsDetailsItem[] => { + const resultMap = new Map(); + const fieldNames = Object.keys(fields); + for (let i = 0; i < fieldNames.length; i++) { + const field = fieldNames[i]; const item: unknown[] = fields[field]; - const fieldCategory = - prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); + const fieldCategory = prependFieldCategory ?? getFieldCategory(field); + const dotField = prependField ? `${prependField}.${field}` : field; + + // Handle geo fields if (isGeoField(field)) { - return [ - ...accumulator, - { - category: fieldCategory, - field, - values: formatGeoLocation(item), - originalValue: formatGeoLocation(item), - isObjectArray: true, // important for UI - }, - ]; + const geoItem = processGeoField(field, item, fieldCategory); + resultMap.set(field, geoItem); + // eslint-disable-next-line no-continue + continue; } + const objArrStr = toObjectArrayOfStrings(item); const strArr = objArrStr.map(({ str }) => str); const isObjectArray = objArrStr.some((o) => o.isObjectArray); - const dotField = prependField ? `${prependField}.${field}` : field; - // return simple field value (non-ecs object, non-array) - if ( - !isObjectArray || - (Object.keys({ - ...ecsFieldMap, - ...technicalRuleFieldMap, - ...legacyExperimentalFieldMap, - }).find((ecsField) => ecsField === field) === undefined && - !isRuleParametersFieldOrSubfield(field, prependField)) - ) { - return [ - ...accumulator, - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ]; + const isEcsField = fieldMaps[field as keyof typeof fieldMaps] !== undefined; + const isRuleParameters = isRuleParametersFieldOrSubfield(field, prependField); + + // Handle simple fields + if (!isObjectArray || (!isEcsField && !isRuleParameters)) { + const simpleItem = processSimpleField(dotField, strArr, isObjectArray, fieldCategory); + resultMap.set(dotField, simpleItem); + // eslint-disable-next-line no-continue + continue; } - const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField) - ? [ - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ] - : []; - - // format nested fields - let nestedFields: TimelineEventsDetailsItem[] = []; - if (isRuleParametersFieldOrSubfield(field, prependField)) { - nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, dotField, fieldCategory); - } else { - nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, prependField, fieldCategory); + // Handle threat enrichment + if (isThreatEnrichmentFieldOrSubfield(field, prependField)) { + const enrichmentItem = createFieldItem(fieldCategory, dotField, strArr, isObjectArray); + resultMap.set(dotField, enrichmentItem); } - // combine duplicate fields - const flat: Record = [ - ...accumulator, - ...nestedFields, - ...threatEnrichmentObject, - ].reduce( - (acc, f) => ({ - ...acc, - // acc/flat is hashmap to determine if we already have the field or not without an array iteration - // its converted back to array in return with Object.values - ...(acc[f.field] != null - ? { - [f.field]: { - ...f, - originalValue: acc[f.field].originalValue.includes(f.originalValue[0]) - ? acc[f.field].originalValue - : [...acc[f.field].originalValue, ...f.originalValue], - values: acc[f.field].values?.includes(f.values?.[0] || '') - ? acc[f.field].values - : [...(acc[f.field].values || []), ...(f.values || [])], - }, - } - : { [f.field]: f }), - }), - {} as Record + // Process nested fields + const nestedFields = processNestedFields( + item, + dotField, + fieldCategory, + isRuleParameters || isThreatEnrichmentFieldOrSubfield(field, prependField) ); + // Merge results + for (const nestedItem of nestedFields) { + const existing = resultMap.get(nestedItem.field); + + if (!existing) { + resultMap.set(nestedItem.field, nestedItem); + // eslint-disable-next-line no-continue + continue; + } - return Object.values(flat); - }, []); + // Merge values and originalValue arrays + const mergedValues = existing.values?.includes(nestedItem.values?.[0] || '') + ? existing.values + : [...(existing.values || []), ...(nestedItem.values || [])]; -export const getDataSafety = (fn: (args: A) => T, args: A): Promise => - new Promise((resolve) => setTimeout(() => resolve(fn(args)))); + const mergedOriginal = existing.originalValue.includes(nestedItem.originalValue[0]) + ? existing.originalValue + : [...existing.originalValue, ...nestedItem.originalValue]; + + resultMap.set(nestedItem.field, { + ...nestedItem, + values: mergedValues, + originalValue: mergedOriginal, + }); + } + } + + return Array.from(resultMap.values()); +}; diff --git a/x-pack/plugins/timelines/common/utils/index.ts b/x-pack/plugins/timelines/common/utils/index.ts new file mode 100644 index 0000000000000..e4b7eefec454f --- /dev/null +++ b/x-pack/plugins/timelines/common/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getDataFromFieldsHits, isGeoField } from './field_formatters'; +export { toArray, toObjectArrayOfStrings } from './to_array'; diff --git a/x-pack/plugins/timelines/common/utils/to_array.ts b/x-pack/plugins/timelines/common/utils/to_array.ts index fbb2b8d48a250..d13eb0578008b 100644 --- a/x-pack/plugins/timelines/common/utils/to_array.ts +++ b/x-pack/plugins/timelines/common/utils/to_array.ts @@ -5,83 +5,55 @@ * 2.0. */ -export const toArray = (value: T | T[] | null): T[] => - Array.isArray(value) ? value : value == null ? [] : [value]; -export const toStringArray = (value: T | T[] | null): string[] => { - if (Array.isArray(value)) { - return value.reduce((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, v.toString()]; - case 'object': - try { - return [...acc, JSON.stringify(v)]; - } catch { - return [...acc, 'Invalid Object']; - } - case 'string': - return [...acc, v]; - default: - return [...acc, `${v}`]; - } +export const toArray = (value: T | T[] | null | undefined): T[] => + value == null ? [] : Array.isArray(value) ? value : [value]; + +export const toStringArray = (value: T | T[] | null): string[] => { + if (value == null) return []; + + const arr = Array.isArray(value) ? value : [value]; + return arr.reduce((acc, v) => { + if (v == null) return acc; + + if (typeof v === 'object') { + try { + acc.push(JSON.stringify(v)); + } catch { + acc.push('Invalid Object'); } return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [JSON.stringify(value)]; - } catch { - return ['Invalid Object']; } - } else { - return [`${value}`]; - } + + acc.push(String(v)); + return acc; + }, []); }; -export const toObjectArrayOfStrings = ( + +export const toObjectArrayOfStrings = ( value: T | T[] | null ): Array<{ str: string; isObjectArray?: boolean; }> => { - if (Array.isArray(value)) { - return value.reduce< - Array<{ - str: string; - isObjectArray?: boolean; - }> - >((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, { str: v.toString() }]; - case 'object': - try { - return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value - } catch { - return [...acc, { str: 'Invalid Object' }]; - } - case 'string': - return [...acc, { str: v }]; - default: - return [...acc, { str: `${v}` }]; - } + if (value == null) return []; + + const arr = Array.isArray(value) ? value : [value]; + return arr.reduce>((acc, v) => { + if (v == null) return acc; + + if (typeof v === 'object') { + try { + acc.push({ + str: JSON.stringify(v), + isObjectArray: true, + }); + } catch { + acc.push({ str: 'Invalid Object' }); } return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [{ str: JSON.stringify(value), isObjectArray: true }]; - } catch { - return [{ str: 'Invalid Object' }]; } - } else { - return [{ str: `${value}` }]; - } + + acc.push({ str: String(v) }); + return acc; + }, []); }; diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts index 2b4f562954df1..645f6daa5727d 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts @@ -17,7 +17,6 @@ import { } from '../../../../common/search_strategy'; import { TimelineEqlResponse } from '../../../../common/search_strategy/timeline/events/eql'; import { inspectStringifyObject } from '../../../utils/build_query'; -import { TIMELINE_EVENTS_FIELDS } from '../factory/helpers/constants'; import { formatTimelineData } from '../factory/helpers/format_timeline_data'; export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record => { @@ -68,38 +67,38 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record>, fieldRequested: string[]) => - sequences.reduce>(async (acc, sequence, sequenceIndex) => { +const parseSequences = async (sequences: Array>, fieldRequested: string[]) => { + let result: TimelineEdges[] = []; + + for (const [sequenceIndex, sequence] of sequences.entries()) { const sequenceParentId = sequence.events[0]?._id ?? null; - const data = await acc; - const allData = await Promise.all( - sequence.events.map(async (event, eventIndex) => { - const item = await formatTimelineData( - fieldRequested, - TIMELINE_EVENTS_FIELDS, - event as EventHit - ); - return Promise.resolve({ - ...item, - node: { - ...item.node, - ecs: { - ...item.node.ecs, - ...(sequenceParentId != null - ? { - eql: { - parentId: sequenceParentId, - sequenceNumber: `${sequenceIndex}-${eventIndex}`, - }, - } - : {}), - }, - }, - }); - }) + const formattedEvents = await formatTimelineData( + sequence.events as EventHit[], + fieldRequested, + false ); - return Promise.resolve([...data, ...allData]); - }, Promise.resolve([])); + + const eventsWithEql = formattedEvents.map((item, eventIndex) => ({ + ...item, + node: { + ...item.node, + ecs: { + ...item.node.ecs, + ...(sequenceParentId && { + eql: { + parentId: sequenceParentId, + sequenceNumber: `${sequenceIndex}-${eventIndex}`, + }, + }), + }, + }, + })); + + result = result.concat(eventsWithEql); + } + + return result; +}; export const parseEqlResponse = async ( options: TimelineEqlRequestOptions, @@ -116,10 +115,10 @@ export const parseEqlResponse = async ( if (response.rawResponse.hits.sequences !== undefined) { edges = await parseSequences(response.rawResponse.hits.sequences, options.fieldRequested); } else if (response.rawResponse.hits.events !== undefined) { - edges = await Promise.all( - response.rawResponse.hits.events.map(async (event) => - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) - ) + edges = await formatTimelineData( + response.rawResponse.hits.events as EventHit[], + options.fieldRequested, + false ); } diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts index 4ed857b4885e3..2ee2b64162c13 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts @@ -14,13 +14,11 @@ import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants import { EventHit, TimelineEventsAllStrategyResponse, - TimelineEdges, } from '../../../../../../common/search_strategy'; import { TimelineFactory } from '../../types'; import { buildTimelineEventsAllQuery } from './query.events_all.dsl'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { formatTimelineData } from '../../helpers/format_timeline_data'; -import { TIMELINE_EVENTS_FIELDS } from '../../helpers/constants'; export const timelineEventsAll: TimelineFactory = { buildDsl: ({ authFilter, ...options }) => { @@ -54,14 +52,10 @@ export const timelineEventsAll: TimelineFactory = { fieldRequested = [...new Set(fieldsReturned)]; } - const edges: TimelineEdges[] = await Promise.all( - hits.map((hit) => - formatTimelineData( - fieldRequested, - options.excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS, - hit as EventHit - ) - ) + const edges = await formatTimelineData( + hits as EventHit[], + fieldRequested, + options.excludeEcsData ?? false ); const consumers = producerBuckets.reduce( diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts index 46edcf926d061..fcb90b73c241e 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts @@ -12,15 +12,11 @@ import { TimelineEventsQueries } from '../../../../../../common/api/search_strat import { EventHit, TimelineEventsDetailsStrategyResponse, - TimelineEventsDetailsItem, } from '../../../../../../common/search_strategy'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { TimelineFactory } from '../../types'; import { buildTimelineDetailsQuery } from './query.events_details.dsl'; -import { - getDataFromFieldsHits, - getDataSafety, -} from '../../../../../../common/utils/field_formatters'; +import { getDataFromFieldsHits } from '../../../../../../common/utils/field_formatters'; import { buildEcsObjects } from '../../helpers/build_ecs_objects'; export const timelineEventsDetails: TimelineFactory = { @@ -57,10 +53,7 @@ export const timelineEventsDetails: TimelineFactory( - getDataFromFieldsHits, - merge(fields, hitsData) - ); + const fieldsData = getDataFromFieldsHits(merge(fields, hitsData)); const rawEventData = response.rawResponse.hits.hits[0]; const ecs = buildEcsObjects(rawEventData as EventHit); diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts index 5117f8dc889ed..3e96494c88313 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts @@ -7,12 +7,12 @@ import { eventHit } from '@kbn/securitysolution-t-grid'; import { EventHit } from '../../../../../common/search_strategy'; -import { TIMELINE_EVENTS_FIELDS } from './constants'; import { formatTimelineData } from './format_timeline_data'; describe('formatTimelineData', () => { it('should properly format the timeline data', async () => { const res = await formatTimelineData( + [eventHit], [ '@timestamp', 'host.name', @@ -21,187 +21,188 @@ describe('formatTimelineData', () => { 'source.geo.location', 'threat.enrichments.matched.field', ], - TIMELINE_EVENTS_FIELDS, - eventHit + false ); - expect(res).toEqual({ - cursor: { - tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', - value: '1605624488922', - }, - node: { - _id: 'tkCt1nUBaEgqnrVSZ8R_', - _index: 'auditbeat-7.8.0-2020.11.05-000003', - data: [ - { - field: '@timestamp', - value: ['2020-11-17T14:48:08.922Z'], - }, - { - field: 'host.name', - value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - }, - { - field: 'threat.enrichments.matched.field', - value: [ - 'matched_field', - 'other_matched_field', - 'matched_field_2', - 'host.name', - 'host.hostname', - 'host.architecture', - ], - }, - { - field: 'source.geo.location', - value: [`{"lon":118.7778,"lat":32.0617}`], - }, - ], - ecs: { - '@timestamp': ['2020-11-17T14:48:08.922Z'], + expect(res).toEqual([ + { + cursor: { + tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', + value: '1605624488922', + }, + node: { _id: 'tkCt1nUBaEgqnrVSZ8R_', _index: 'auditbeat-7.8.0-2020.11.05-000003', - agent: { - type: ['auditbeat'], - }, - event: { - action: ['process_started'], - category: ['process'], - dataset: ['process'], - kind: ['event'], - module: ['system'], - type: ['start'], - }, - host: { - id: ['e59991e835905c65ed3e455b33e13bd6'], - ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - os: { - family: ['debian'], + data: [ + { + field: '@timestamp', + value: ['2020-11-17T14:48:08.922Z'], }, - }, - message: ['Process go (PID: 4313) by user jenkins STARTED'], - process: { - args: ['go', 'vet', './...'], - entity_id: ['Z59cIkAAIw8ZoK0H'], - executable: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - hash: { - sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - }, - name: ['go'], - pid: ['4313'], - ppid: ['3977'], - working_directory: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - }, - timestamp: '2020-11-17T14:48:08.922Z', - user: { - name: ['jenkins'], - }, - threat: { - enrichments: [ - { - feed: { name: [] }, - indicator: { - provider: ['yourself'], - reference: [], - }, - matched: { - atomic: ['matched_atomic'], - field: ['matched_field', 'other_matched_field'], - type: [], - }, - }, - { - feed: { name: [] }, - indicator: { - provider: ['other_you'], - reference: [], - }, - matched: { - atomic: ['matched_atomic_2'], - field: ['matched_field_2'], - type: [], - }, - }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], - }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.name'], - type: ['indicator_match_rule'], - }, + { + field: 'host.name', + value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + }, + { + field: 'threat.enrichments.matched.field', + value: [ + 'matched_field', + 'other_matched_field', + 'matched_field_2', + 'host.name', + 'host.hostname', + 'host.architecture', + ], + }, + { + field: 'source.geo.location', + value: [`{"lon":118.7778,"lat":32.0617}`], + }, + ], + ecs: { + '@timestamp': ['2020-11-17T14:48:08.922Z'], + _id: 'tkCt1nUBaEgqnrVSZ8R_', + _index: 'auditbeat-7.8.0-2020.11.05-000003', + agent: { + type: ['auditbeat'], + }, + event: { + action: ['process_started'], + category: ['process'], + dataset: ['process'], + kind: ['event'], + module: ['system'], + type: ['start'], + }, + host: { + id: ['e59991e835905c65ed3e455b33e13bd6'], + ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + os: { + family: ['debian'], }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], - }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.hostname'], - type: ['indicator_match_rule'], - }, + }, + message: ['Process go (PID: 4313) by user jenkins STARTED'], + process: { + args: ['go', 'vet', './...'], + entity_id: ['Z59cIkAAIw8ZoK0H'], + executable: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + hash: { + sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], + name: ['go'], + pid: ['4313'], + ppid: ['3977'], + working_directory: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + }, + timestamp: '2020-11-17T14:48:08.922Z', + user: { + name: ['jenkins'], + }, + threat: { + enrichments: [ + { + feed: { name: [] }, + indicator: { + provider: ['yourself'], + reference: [], + }, + matched: { + atomic: ['matched_atomic'], + field: ['matched_field', 'other_matched_field'], + type: [], + }, }, - matched: { - atomic: ['x86_64'], - field: ['host.architecture'], - type: ['indicator_match_rule'], + { + feed: { name: [] }, + indicator: { + provider: ['other_you'], + reference: [], + }, + matched: { + atomic: ['matched_atomic_2'], + field: ['matched_field_2'], + type: [], + }, }, - }, - { - feed: { - name: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.name'], + type: ['indicator_match_rule'], + }, }, - indicator: { - provider: [], - reference: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.hostname'], + type: ['indicator_match_rule'], + }, }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.name'], - type: ['indicator_match_rule'], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['x86_64'], + field: ['host.architecture'], + type: ['indicator_match_rule'], + }, }, - }, - { - feed: { - name: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.name'], + type: ['indicator_match_rule'], + }, }, - indicator: { - provider: [], - reference: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.hostname'], + type: ['indicator_match_rule'], + }, }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.hostname'], - type: ['indicator_match_rule'], - }, - }, - ], + ], + }, }, }, }, - }); + ]); }); it('should properly format the rule signal results', async () => { @@ -240,57 +241,61 @@ describe('formatTimelineData', () => { expect( await formatTimelineData( + [response], ['@timestamp', 'host.name', 'destination.ip', 'source.ip'], - TIMELINE_EVENTS_FIELDS, - response + false ) - ).toEqual({ - cursor: { - tiebreaker: null, - value: '', - }, - node: { - _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', - _index: '.siem-signals-patrykkopycinski-default-000007', - data: [ - { - field: '@timestamp', - value: ['2021-01-09T13:41:40.517Z'], - }, - ], - ecs: { - '@timestamp': ['2021-01-09T13:41:40.517Z'], - timestamp: '2021-01-09T13:41:40.517Z', + ).toEqual([ + { + cursor: { + tiebreaker: null, + value: '', + }, + node: { _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', _index: '.siem-signals-patrykkopycinski-default-000007', - event: { - kind: ['signal'], - }, - kibana: { - alert: { - original_time: ['2021-01-09T13:39:32.595Z'], - workflow_status: ['open'], - threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'], - severity: ['low'], - risk_score: ['21'], - rule: { - building_block_type: [], - exceptions_list: [], - from: ['now-360s'], - uuid: ['696c24e0-526d-11eb-836c-e1620268b945'], - name: ['Threshold test'], - to: ['now'], - type: ['threshold'], - version: ['1'], - timeline_id: [], - timeline_title: [], - note: [], + data: [ + { + field: '@timestamp', + value: ['2021-01-09T13:41:40.517Z'], + }, + ], + ecs: { + '@timestamp': ['2021-01-09T13:41:40.517Z'], + timestamp: '2021-01-09T13:41:40.517Z', + _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', + _index: '.siem-signals-patrykkopycinski-default-000007', + event: { + kind: ['signal'], + }, + kibana: { + alert: { + original_time: ['2021-01-09T13:39:32.595Z'], + workflow_status: ['open'], + threshold_result: [ + '{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}', + ], + severity: ['low'], + risk_score: ['21'], + rule: { + building_block_type: [], + exceptions_list: [], + from: ['now-360s'], + uuid: ['696c24e0-526d-11eb-836c-e1620268b945'], + name: ['Threshold test'], + to: ['now'], + type: ['threshold'], + version: ['1'], + timeline_id: [], + timeline_title: [], + note: [], + }, }, }, }, }, }, - }); + ]); }); it('should properly format the inventory rule signal results', async () => { @@ -347,6 +352,7 @@ describe('formatTimelineData', () => { expect( await formatTimelineData( + [response], [ 'kibana.alert.status', '@timestamp', @@ -376,168 +382,169 @@ describe('formatTimelineData', () => { 'event.kind', 'kibana.alert.rule.parameters', ], - TIMELINE_EVENTS_FIELDS, - response + false ) - ).toEqual({ - cursor: { - tiebreaker: null, - value: '', - }, - node: { - _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', - _index: '.internal.alerts-observability.metrics.alerts-default-000001', - data: [ - { - field: 'kibana.alert.rule.consumer', - value: ['infrastructure'], - }, - { - field: '@timestamp', - value: ['2022-07-21T22:38:57.888Z'], - }, - { - field: 'kibana.alert.workflow_status', - value: ['open'], - }, - { - field: 'kibana.alert.reason', - value: [ - 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', - ], - }, - { - field: 'kibana.alert.rule.name', - value: ['test 1212'], - }, - { - field: 'kibana.alert.rule.uuid', - value: ['15d82f10-0926-11ed-bece-6b0c033d0075'], - }, - { - field: 'kibana.alert.rule.parameters.sourceId', - value: ['default'], - }, - { - field: 'kibana.alert.rule.parameters.nodeType', - value: ['host'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.comparator', - value: ['>'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.timeSize', - value: ['1'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.metric', - value: ['cpu'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.threshold', - value: ['10'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation', - value: ['avg'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.id', - value: ['alert-custom-metric'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.field', - value: [''], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.type', - value: ['custom'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.timeUnit', - value: ['d'], - }, - { - field: 'event.action', - value: ['active'], - }, - { - field: 'event.kind', - value: ['signal'], - }, - { - field: 'kibana.alert.status', - value: ['active'], - }, - { - field: 'kibana.alert.duration.us', - value: ['9502040000'], - }, - { - field: 'kibana.alert.rule.category', - value: ['Inventory'], - }, - { - field: 'kibana.alert.uuid', - value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'], - }, - { - field: 'kibana.alert.start', - value: ['2022-07-21T20:00:35.848Z'], - }, - { - field: 'kibana.alert.rule.producer', - value: ['infrastructure'], - }, - { - field: 'kibana.alert.rule.rule_type_id', - value: ['metrics.alert.inventory.threshold'], - }, - { - field: 'kibana.alert.instance.id', - value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'], - }, - { - field: 'kibana.alert.rule.execution.uuid', - value: ['37498c42-0190-4a83-adfa-c7e5f817f977'], - }, - { - field: 'kibana.space_ids', - value: ['default'], - }, - { - field: 'kibana.version', - value: ['8.4.0'], - }, - ], - ecs: { - '@timestamp': ['2022-07-21T22:38:57.888Z'], + ).toEqual([ + { + cursor: { + tiebreaker: null, + value: '', + }, + node: { _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', _index: '.internal.alerts-observability.metrics.alerts-default-000001', - event: { - action: ['active'], - kind: ['signal'], - }, - kibana: { - alert: { - reason: [ + data: [ + { + field: 'kibana.alert.rule.consumer', + value: ['infrastructure'], + }, + { + field: '@timestamp', + value: ['2022-07-21T22:38:57.888Z'], + }, + { + field: 'kibana.alert.workflow_status', + value: ['open'], + }, + { + field: 'kibana.alert.reason', + value: [ 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', ], - rule: { - consumer: ['infrastructure'], - name: ['test 1212'], - uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'], - parameters: [ - '{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}', + }, + { + field: 'kibana.alert.rule.name', + value: ['test 1212'], + }, + { + field: 'kibana.alert.rule.uuid', + value: ['15d82f10-0926-11ed-bece-6b0c033d0075'], + }, + { + field: 'kibana.alert.rule.parameters.sourceId', + value: ['default'], + }, + { + field: 'kibana.alert.rule.parameters.nodeType', + value: ['host'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.comparator', + value: ['>'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.timeSize', + value: ['1'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.metric', + value: ['cpu'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.threshold', + value: ['10'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation', + value: ['avg'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.id', + value: ['alert-custom-metric'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.field', + value: [''], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.type', + value: ['custom'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.timeUnit', + value: ['d'], + }, + { + field: 'event.action', + value: ['active'], + }, + { + field: 'event.kind', + value: ['signal'], + }, + { + field: 'kibana.alert.status', + value: ['active'], + }, + { + field: 'kibana.alert.duration.us', + value: ['9502040000'], + }, + { + field: 'kibana.alert.rule.category', + value: ['Inventory'], + }, + { + field: 'kibana.alert.uuid', + value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'], + }, + { + field: 'kibana.alert.start', + value: ['2022-07-21T20:00:35.848Z'], + }, + { + field: 'kibana.alert.rule.producer', + value: ['infrastructure'], + }, + { + field: 'kibana.alert.rule.rule_type_id', + value: ['metrics.alert.inventory.threshold'], + }, + { + field: 'kibana.alert.instance.id', + value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'], + }, + { + field: 'kibana.alert.rule.execution.uuid', + value: ['37498c42-0190-4a83-adfa-c7e5f817f977'], + }, + { + field: 'kibana.space_ids', + value: ['default'], + }, + { + field: 'kibana.version', + value: ['8.4.0'], + }, + ], + ecs: { + '@timestamp': ['2022-07-21T22:38:57.888Z'], + _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', + _index: '.internal.alerts-observability.metrics.alerts-default-000001', + event: { + action: ['active'], + kind: ['signal'], + }, + kibana: { + alert: { + reason: [ + 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', ], + rule: { + consumer: ['infrastructure'], + name: ['test 1212'], + uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'], + parameters: [ + '{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}', + ], + }, + workflow_status: ['open'], }, - workflow_status: ['open'], }, + timestamp: '2022-07-21T22:38:57.888Z', }, - timestamp: '2022-07-21T22:38:57.888Z', }, }, - }); + ]); }); }); diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts index f56cfd32391d4..485ec64badd5c 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts @@ -5,118 +5,130 @@ * 2.0. */ -import { get, has, merge, uniq } from 'lodash/fp'; -import { EventHit, TimelineEdges, TimelineNonEcsData } from '../../../../../common/search_strategy'; +import { get, has } from 'lodash/fp'; +import { + EventHit, + TimelineEdges, + TimelineNonEcsData, + EventSource, +} from '../../../../../common/search_strategy'; import { toStringArray } from '../../../../../common/utils/to_array'; -import { getDataFromFieldsHits, getDataSafety } from '../../../../../common/utils/field_formatters'; +import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters'; import { getTimestamp } from './get_timestamp'; import { getNestedParentPath } from './get_nested_parent_path'; import { buildObjectRecursive } from './build_object_recursive'; -import { ECS_METADATA_FIELDS } from './constants'; +import { ECS_METADATA_FIELDS, TIMELINE_EVENTS_FIELDS } from './constants'; -export const formatTimelineData = async ( - dataFields: readonly string[], - ecsFields: readonly string[], - hit: EventHit -) => - uniq([...ecsFields, ...dataFields]).reduce>( - async (acc, fieldName) => { - const flattenedFields: TimelineEdges = await acc; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - flattenedFields.node._id = hit._id!; - flattenedFields.node._index = hit._index; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - flattenedFields.node.ecs._id = hit._id!; - flattenedFields.node.ecs.timestamp = getTimestamp(hit); - flattenedFields.node.ecs._index = hit._index; - if (hit.sort && hit.sort.length > 1) { - flattenedFields.cursor.value = hit.sort[0]; - flattenedFields.cursor.tiebreaker = hit.sort[1]; - } - const waitForIt = await mergeTimelineFieldsWithHit( - fieldName, - flattenedFields, - hit, - dataFields, - ecsFields - ); - return Promise.resolve(waitForIt); - }, - Promise.resolve({ - node: { ecs: { _id: '' }, data: [], _id: '', _index: '' }, - cursor: { - value: '', - tiebreaker: null, - }, - }) - ); - -const getValuesFromFields = async ( +const createBaseTimelineEdges = (): TimelineEdges => ({ + node: { + ecs: { _id: '' }, + data: [], + _id: '', + _index: '', + }, + cursor: { + value: '', + tiebreaker: null, + }, +}); + +function deepMerge(target: EventSource, source: EventSource) { + for (const key in source) { + if (source[key] instanceof Object && key in target) { + deepMerge(target[key], source[key]); + } else { + target[key] = source[key]; + } + } + return target; +} + +const processMetadataField = (fieldName: string, hit: EventHit): TimelineNonEcsData[] => [ + { + field: fieldName, + value: toStringArray(get(fieldName, hit)), + }, +]; + +const processFieldData = ( fieldName: string, hit: EventHit, nestedParentFieldName?: string -): Promise => { - if (ECS_METADATA_FIELDS.includes(fieldName)) { - return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }]; - } +): TimelineNonEcsData[] => { + const fieldToEval = nestedParentFieldName + ? { [nestedParentFieldName]: hit.fields[nestedParentFieldName] } + : { [fieldName]: hit.fields[fieldName] }; - let fieldToEval; - - if (nestedParentFieldName == null) { - fieldToEval = { - [fieldName]: hit.fields[fieldName], - }; - } else { - fieldToEval = { - [nestedParentFieldName]: hit.fields[nestedParentFieldName], - }; - } - const formattedData = await getDataSafety(getDataFromFieldsHits, fieldToEval); - return formattedData.reduce((acc: TimelineNonEcsData[], { field, values }) => { - // nested fields return all field values, pick only the one we asked for + const formattedData = getDataFromFieldsHits(fieldToEval); + const fieldsData: TimelineNonEcsData[] = []; + return formattedData.reduce((agg, { field, values }) => { if (field.includes(fieldName)) { - acc.push({ field, value: values }); + agg.push({ + field, + value: values, + }); } - return acc; - }, []); + return agg; + }, fieldsData); }; -const mergeTimelineFieldsWithHit = async ( - fieldName: string, - flattenedFields: T, - hit: EventHit, - dataFields: readonly string[], - ecsFields: readonly string[] -) => { - if (fieldName != null) { - const nestedParentPath = getNestedParentPath(fieldName, hit.fields); - if ( - nestedParentPath != null || - has(fieldName, hit.fields) || - ECS_METADATA_FIELDS.includes(fieldName) - ) { - const objectWithProperty = { - node: { - ...get('node', flattenedFields), - data: dataFields.includes(fieldName) - ? [ - ...get('node.data', flattenedFields), - ...(await getValuesFromFields(fieldName, hit, nestedParentPath)), - ] - : get('node.data', flattenedFields), - ecs: ecsFields.includes(fieldName) - ? { - ...get('node.ecs', flattenedFields), - ...buildObjectRecursive(fieldName, hit.fields), - } - : get('node.ecs', flattenedFields), - }, - }; - return merge(flattenedFields, objectWithProperty); +export const formatTimelineData = async ( + hits: EventHit[], + fieldRequested: readonly string[], + excludeEcsData: boolean +): Promise => { + const ecsFields = excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS; + + const uniqueFields = new Set([...ecsFields, ...fieldRequested]); + const dataFieldSet = new Set(fieldRequested); + const ecsFieldSet = new Set(ecsFields); + + const results: TimelineEdges[] = new Array(hits.length); + + for (let i = 0; i < hits.length; i++) { + const hit = hits[i]; + if (hit._id) { + const result = createBaseTimelineEdges(); + + result.node._id = hit._id; + result.node._index = hit._index; + result.node.ecs._id = hit._id; + result.node.ecs.timestamp = getTimestamp(hit); + result.node.ecs._index = hit._index; + + if (hit.sort?.length > 1) { + result.cursor.value = hit.sort[0]; + result.cursor.tiebreaker = hit.sort[1]; + } + + result.node.data = []; + + for (const fieldName of uniqueFields) { + const nestedParentPath = getNestedParentPath(fieldName, hit.fields); + const isEcs = ECS_METADATA_FIELDS.includes(fieldName); + if (!nestedParentPath && !has(fieldName, hit.fields) && !isEcs) { + // eslint-disable-next-line no-continue + continue; + } + + if (dataFieldSet.has(fieldName)) { + const values = isEcs + ? processMetadataField(fieldName, hit) + : processFieldData(fieldName, hit, nestedParentPath); + + result.node.data.push(...values); + } + + if (ecsFieldSet.has(fieldName)) { + deepMerge(result.node.ecs, buildObjectRecursive(fieldName, hit.fields)); + } + } + + results[i] = result; } else { - return flattenedFields; + results[i] = createBaseTimelineEdges(); } - } else { - return flattenedFields; } + + return results; };