From e2a798c7e2291a3df9149a244ece182776903255 Mon Sep 17 00:00:00 2001 From: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:24:54 -0500 Subject: [PATCH] [Security Solution] [Timeline] Consolidate reduces, remove unneeded async/awaits, other small fixes (#197168) ## Summary For most of 8.x, both anecdotally from users and in development, timeline search strategy based apis would often seem slower than the equivalent search in discover or elsewhere in kibana, and I have long suspected that this came from how the timeline sever code formatted the elasticsearch responses for use in the UI, and while working on something else, noticed even higher than normal occurrences in logs of "][http.server.Kibana] Event loop utilization for /internal/search/timelineSearchStrategy exceeded threshold of..." and so I tried to refactor all of the functions in place as much as possible, keeping the apis similar, most of the unit tests, etc, but removing as many as possible of the Promise.alls, reduce within reduce, etc. This has lead to a substantial improvement in performance, as you can see below, and with larger result sets, I think the difference would only be more noticeable. After fix: ~40 ms for formatTimelineData with ~1000 docs image Before fix: ~18000 ms for formatTimelineData with ~1000 docs image [chrome_profile_timeline_slow.cpuprofile](https://github.com/user-attachments/files/17825602/chrome_profile_timeline_slow.cpuprofile) [chrome_profile_timeline_fast.cpuprofile](https://github.com/user-attachments/files/17825606/chrome_profile_timeline_fast.cpuprofile) I've attached the chrome devtools profiles for each, the time was measured with the function: ``` async function measureAwait(promise: Promise, label: string): Promise { const start = performance.now(); try { const result = await promise; const duration = performance.now() - start; console.log(`${label} took ${duration}ms`); return result; } catch (error) { const duration = performance.now() - start; console.log(`${label} failed after ${duration}ms`); throw error; } } ``` Wrapped around the call to formatTimelineData in x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../src/mock/mock_event_details.ts | 46 +- .../common/utils/field_formatters.test.ts | 94 --- .../common/utils/field_formatters.ts | 148 ---- .../common/utils/to_array.ts | 89 --- .../shared/utils/threat_intelligence.tsx | 2 +- .../helpers/format_response_object_values.ts | 4 +- .../helpers/get_flattened_fields.ts | 2 +- .../factory/hosts/all/helpers.ts | 2 +- .../factory/hosts/details/helpers.ts | 2 +- .../factory/users/authentications/helpers.ts | 2 +- x-pack/plugins/timelines/common/index.ts | 2 + .../common/utils/field_formatters.test.ts | 35 +- .../common/utils/field_formatters.ts | 228 +++--- .../plugins/timelines/common/utils/index.ts | 9 + .../timelines/common/utils/to_array.ts | 106 +-- .../search_strategy/timeline/eql/helpers.ts | 69 +- .../timeline/factory/events/all/index.ts | 14 +- .../timeline/factory/events/details/index.ts | 11 +- .../helpers/format_timeline_data.test.ts | 727 +++++++++--------- .../factory/helpers/format_timeline_data.ts | 208 ++--- 20 files changed, 740 insertions(+), 1060 deletions(-) delete mode 100644 x-pack/plugins/security_solution/common/utils/field_formatters.test.ts delete mode 100644 x-pack/plugins/security_solution/common/utils/field_formatters.ts delete mode 100644 x-pack/plugins/security_solution/common/utils/to_array.ts create mode 100644 x-pack/plugins/timelines/common/utils/index.ts diff --git a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts index dc2497bb19cec..5152132cda4bb 100644 --- a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts +++ b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts @@ -291,6 +291,29 @@ export const eventDetailsFormattedFields = [ originalValue: [`{"lon":118.7778,"lat":32.0617}`], values: [`{"lon":118.7778,"lat":32.0617}`], }, + { + category: 'threat', + field: 'threat.enrichments', + isObjectArray: true, + originalValue: [ + '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', + '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + ], + values: [ + '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', + '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', + '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', + ], + }, { category: 'threat', field: 'threat.enrichments.matched.field', @@ -376,27 +399,4 @@ export const eventDetailsFormattedFields = [ originalValue: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'], values: ['FFEtSYIBZ61VHL7LvV2j', 'E1EtSYIBZ61VHL7Ltl3m', 'CFErSYIBZ61VHL7LIV1N'], }, - { - category: 'threat', - field: 'threat.enrichments', - isObjectArray: true, - originalValue: [ - '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', - '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - ], - values: [ - '{"matched.field":["matched_field","other_matched_field"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["yourself"],"indicator.type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"]},{"great.field":["grrrrr_2"]}]}', - '{"matched.field":["matched_field_2"],"indicator.first_seen":["2021-02-22T17:29:25.195Z"],"indicator.provider":["other_you"],"indicator.type":["custom"],"matched.atomic":["matched_atomic_2"],"lazer":[{"great.field":[{"wowoe":[{"fooooo":["grrrrr"]}],"astring":"cool","aNumber":1,"neat":true}]}]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["FFEtSYIBZ61VHL7LvV2j"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.architecture"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["x86_64"]}', - '{"matched.field":["host.name"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["E1EtSYIBZ61VHL7Ltl3m"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - '{"matched.field":["host.hostname"],"matched.index":["im"],"matched.type":["indicator_match_rule"],"matched.id":["CFErSYIBZ61VHL7LIV1N"],"matched.atomic":["MacBook-Pro-de-Gloria.local"]}', - ], - }, ]; diff --git a/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts b/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts deleted file mode 100644 index 33d0c226fc44e..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/field_formatters.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { EventHit } from '../search_strategy'; -import { getDataFromFieldsHits, getDataSafety } from './field_formatters'; -import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid'; - -describe('Events Details Helpers', () => { - const fields: EventHit['fields'] = eventHit.fields; - const resultFields = eventDetailsFormattedFields; - describe('#getDataFromFieldsHits', () => { - it('happy path', () => { - const result = getDataFromFieldsHits(fields); - expect(result).toEqual(resultFields); - }); - it('lets get weird', () => { - const whackFields = { - 'crazy.pants': [ - { - 'matched.field': ['matched_field'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['yourself'], - type: ['custom'], - 'matched.atomic': ['matched_atomic'], - lazer: [ - { - 'great.field': ['grrrrr'], - lazer: [ - { - lazer: [ - { - cool: true, - lazer: [ - { - lazer: [ - { - lazer: [ - { - lazer: [ - { - whoa: false, - }, - ], - }, - ], - }, - ], - }, - ], - }, - ], - }, - { - lazer: [ - { - cool: false, - }, - ], - }, - ], - }, - { - 'great.field': ['grrrrr_2'], - }, - ], - }, - ], - }; - const whackResultFields = [ - { - category: 'crazy', - field: 'crazy.pants', - values: [ - '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}', - ], - originalValue: [ - '{"matched.field":["matched_field"],"first_seen":["2021-02-22T17:29:25.195Z"],"provider":["yourself"],"type":["custom"],"matched.atomic":["matched_atomic"],"lazer":[{"great.field":["grrrrr"],"lazer":[{"lazer":[{"cool":true,"lazer":[{"lazer":[{"lazer":[{"lazer":[{"whoa":false}]}]}]}]}]},{"lazer":[{"cool":false}]}]},{"great.field":["grrrrr_2"]}]}', - ], - isObjectArray: true, - }, - ]; - const result = getDataFromFieldsHits(whackFields); - expect(result).toEqual(whackResultFields); - }); - }); - it('#getDataSafety', async () => { - const result = await getDataSafety(getDataFromFieldsHits, fields); - expect(result).toEqual(resultFields); - }); -}); diff --git a/x-pack/plugins/security_solution/common/utils/field_formatters.ts b/x-pack/plugins/security_solution/common/utils/field_formatters.ts deleted file mode 100644 index 1d8c05ec9ccf7..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/field_formatters.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; -import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; -import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { isEmpty } from 'lodash/fp'; -import { ENRICHMENT_DESTINATION_PATH } from '../constants'; - -import type { Fields, TimelineEventsDetailsItem } from '../search_strategy'; -import { toObjectArrayOfStrings, toStringArray } from './to_array'; - -export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags']; - -export const getFieldCategory = (field: string): string => { - const fieldCategory = field.split('.')[0]; - if (!isEmpty(fieldCategory) && baseCategoryFields.includes(fieldCategory)) { - return 'base'; - } - return fieldCategory; -}; - -export const formatGeoLocation = (item: unknown[]) => { - const itemGeo = item.length > 0 ? (item[0] as { coordinates: number[] }) : null; - if (itemGeo != null && !isEmpty(itemGeo.coordinates)) { - try { - return toStringArray({ - lon: itemGeo.coordinates[0], - lat: itemGeo.coordinates[1], - }); - } catch { - return toStringArray(item); - } - } - return toStringArray(item); -}; - -export const isGeoField = (field: string) => - field.includes('geo.location') || field.includes('geoip.location'); - -export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) => - prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH; - -export const getDataFromFieldsHits = ( - fields: Fields, - prependField?: string, - prependFieldCategory?: string -): TimelineEventsDetailsItem[] => - Object.keys(fields).reduce((accumulator, field) => { - const item: unknown[] = fields[field]; - - const fieldCategory = - prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); - if (isGeoField(field)) { - return [ - ...accumulator, - { - category: fieldCategory, - field, - values: formatGeoLocation(item), - originalValue: formatGeoLocation(item), - isObjectArray: true, // important for UI - }, - ]; - } - - const objArrStr = toObjectArrayOfStrings(item); - const strArr = objArrStr.map(({ str }) => str); - const isObjectArray = objArrStr.some((o) => o.isObjectArray); - const dotField = prependField ? `${prependField}.${field}` : field; - - // return simple field value (non-esc object, non-array) - if ( - !isObjectArray || - Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...legacyExperimentalFieldMap }).find( - (ecsField) => ecsField === field - ) === undefined - ) { - return [ - ...accumulator, - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ]; - } - - const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField) - ? [ - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ] - : []; - - // format nested fields - const nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, prependField, fieldCategory); - - // combine duplicate fields - const flat: Record = [ - ...accumulator, - ...nestedFields, - ...threatEnrichmentObject, - ].reduce( - (acc, f) => ({ - ...acc, - // acc/flat is hashmap to determine if we already have the field or not without an array iteration - // its converted back to array in return with Object.values - ...(acc[f.field] != null - ? { - [f.field]: { - ...f, - originalValue: acc[f.field].originalValue.includes(f.originalValue[0]) - ? acc[f.field].originalValue - : [...acc[f.field].originalValue, ...f.originalValue], - values: acc[f.field].values?.includes(f.values?.[0] || '') - ? acc[f.field].values - : [...(acc[f.field].values || []), ...(f.values || [])], - }, - } - : { [f.field]: f }), - }), - {} as Record - ); - - return Object.values(flat); - }, []); - -export const getDataSafety = (fn: (args: A) => T, args: A): Promise => - new Promise((resolve) => setTimeout(() => resolve(fn(args)))); diff --git a/x-pack/plugins/security_solution/common/utils/to_array.ts b/x-pack/plugins/security_solution/common/utils/to_array.ts deleted file mode 100644 index b6945708ff0db..0000000000000 --- a/x-pack/plugins/security_solution/common/utils/to_array.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const toArray = (value: T | T[] | null | undefined): T[] => - Array.isArray(value) ? value : value == null ? [] : [value]; - -export const toStringArray = (value: T | T[] | null): string[] => { - if (Array.isArray(value)) { - return value.reduce((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, v.toString()]; - case 'object': - try { - return [...acc, JSON.stringify(v)]; - } catch { - return [...acc, 'Invalid Object']; - } - case 'string': - return [...acc, v]; - default: - return [...acc, `${v}`]; - } - } - return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [JSON.stringify(value)]; - } catch { - return ['Invalid Object']; - } - } else { - return [`${value}`]; - } -}; - -export const toObjectArrayOfStrings = ( - value: T | T[] | null -): Array<{ - str: string; - isObjectArray?: boolean; -}> => { - if (Array.isArray(value)) { - return value.reduce< - Array<{ - str: string; - isObjectArray?: boolean; - }> - >((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, { str: v.toString() }]; - case 'object': - try { - return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value - } catch { - return [...acc, { str: 'Invalid Object' }]; - } - case 'string': - return [...acc, { str: v }]; - default: - return [...acc, { str: `${v}` }]; - } - } - return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [{ str: JSON.stringify(value), isObjectArray: true }]; - } catch { - return [{ str: 'Invalid Object' }]; - } - } else { - return [{ str: `${value}` }]; - } -}; diff --git a/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx b/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx index 2f5bd6510430b..fefacf3e7fc14 100644 --- a/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx +++ b/x-pack/plugins/security_solution/public/flyout/document_details/shared/utils/threat_intelligence.tsx @@ -7,11 +7,11 @@ import { groupBy, isObject } from 'lodash'; import type { TimelineEventsDetailsItem } from '@kbn/timelines-plugin/common'; +import { getDataFromFieldsHits } from '@kbn/timelines-plugin/common'; import { i18n } from '@kbn/i18n'; import type { ThreatDetailsRow } from '../../left/components/threat_details_view_enrichment_accordion'; import type { CtiEnrichment, EventFields } from '../../../../../common/search_strategy'; import { isValidEventField } from '../../../../../common/search_strategy'; -import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters'; import { DEFAULT_INDICATOR_SOURCE_PATH, ENRICHMENT_DESTINATION_PATH, diff --git a/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts b/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts index f44ad77e67929..d80be5f4a421b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/helpers/format_response_object_values.ts @@ -7,9 +7,7 @@ import { mapValues, isObject, isArray } from 'lodash/fp'; import { set } from '@kbn/safer-lodash-set'; - -import { toArray } from '../../../common/utils/to_array'; -import { isGeoField } from '../../../common/utils/field_formatters'; +import { toArray, isGeoField } from '@kbn/timelines-plugin/common'; export const mapObjectValuesToStringArray = (object: object): object => mapValues((o) => { diff --git a/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts b/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts index f40edfc5914df..baed5b3c35605 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/helpers/get_flattened_fields.ts @@ -6,7 +6,7 @@ */ import { set } from '@kbn/safer-lodash-set'; import { get, isEmpty } from 'lodash/fp'; -import { toObjectArrayOfStrings } from '../../../common/utils/to_array'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; export function getFlattenedFields( fields: string[], diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts index 8707f10ed01cb..61ab1c5bca583 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/helpers.ts @@ -8,12 +8,12 @@ import { set } from '@kbn/safer-lodash-set/fp'; import { get, has } from 'lodash/fp'; import { hostFieldsMap } from '@kbn/securitysolution-ecs'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import type { HostAggEsItem, HostsEdges, HostValue, } from '../../../../../../common/search_strategy/security_solution/hosts'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; export const HOSTS_FIELDS: readonly string[] = [ '_id', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts index 94d45b2b63e0e..cc1a084f6b5a7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts @@ -13,6 +13,7 @@ import type { SavedObjectsClientContract, } from '@kbn/core/server'; import { hostFieldsMap } from '@kbn/securitysolution-ecs'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import { Direction } from '../../../../../../common/search_strategy/common'; import type { AggregationRequest, @@ -22,7 +23,6 @@ import type { HostItem, HostValue, } from '../../../../../../common/search_strategy/security_solution/hosts'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; import type { EndpointAppContext } from '../../../../../endpoint/types'; import { getPendingActionsSummary } from '../../../../../endpoint/services'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts index 06452c915009c..b92b67f244ebe 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users/authentications/helpers.ts @@ -8,7 +8,7 @@ import { get, getOr, isEmpty } from 'lodash/fp'; import { set } from '@kbn/safer-lodash-set/fp'; import { sourceFieldsMap, hostFieldsMap } from '@kbn/securitysolution-ecs'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; +import { toObjectArrayOfStrings } from '@kbn/timelines-plugin/common'; import type { AuthenticationsEdges, AuthenticationHit, diff --git a/x-pack/plugins/timelines/common/index.ts b/x-pack/plugins/timelines/common/index.ts index 0a96a22fb2679..3e335a7edd278 100644 --- a/x-pack/plugins/timelines/common/index.ts +++ b/x-pack/plugins/timelines/common/index.ts @@ -67,3 +67,5 @@ export type { } from './search_strategy'; export { Direction, EntityType, EMPTY_BROWSER_FIELDS } from './search_strategy'; + +export { getDataFromFieldsHits, toArray, isGeoField, toObjectArrayOfStrings } from './utils'; diff --git a/x-pack/plugins/timelines/common/utils/field_formatters.test.ts b/x-pack/plugins/timelines/common/utils/field_formatters.test.ts index 43babd374d991..46eb77c8f7f32 100644 --- a/x-pack/plugins/timelines/common/utils/field_formatters.test.ts +++ b/x-pack/plugins/timelines/common/utils/field_formatters.test.ts @@ -7,7 +7,7 @@ import { eventDetailsFormattedFields, eventHit } from '@kbn/securitysolution-t-grid'; import { EventHit } from '../search_strategy'; -import { getDataFromFieldsHits, getDataSafety } from './field_formatters'; +import { getDataFromFieldsHits } from './field_formatters'; describe('Events Details Helpers', () => { const fields: EventHit['fields'] = eventHit.fields; @@ -84,7 +84,7 @@ describe('Events Details Helpers', () => { }, ]; const result = getDataFromFieldsHits(whackFields); - expect(result).toEqual(whackResultFields); + expect(result).toMatchObject(whackResultFields); }); it('flattens alert parameters', () => { const ruleParameterFields = { @@ -191,7 +191,7 @@ describe('Events Details Helpers', () => { ]; const result = getDataFromFieldsHits(ruleParameterFields); - expect(result).toEqual(ruleParametersResultFields); + expect(result).toMatchObject(ruleParametersResultFields); }); it('get data from threat enrichments', () => { @@ -546,6 +546,17 @@ describe('Events Details Helpers', () => { originalValue: ['495ad7a7-316e-4544-8a0f-9c098daee76e'], values: ['495ad7a7-316e-4544-8a0f-9c098daee76e'], }, + { + category: 'threat', + field: 'threat.enrichments', + isObjectArray: true, + originalValue: [ + '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', + ], + values: [ + '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', + ], + }, { category: 'threat', field: 'threat.enrichments.matched.field', @@ -581,25 +592,9 @@ describe('Events Details Helpers', () => { originalValue: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], values: ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], }, - { - category: 'threat', - field: 'threat.enrichments', - isObjectArray: true, - originalValue: [ - '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', - ], - values: [ - '{"matched.field":["myhash.mysha256"],"matched.index":["logs-ti_abusech.malware"],"matched.type":["indicator_match_rule"],"feed.name":["AbuseCH malware"],"matched.atomic":["a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3"]}', - ], - }, ]; const result = getDataFromFieldsHits(data); - expect(result).toEqual(ruleParametersResultFields); + expect(result).toMatchObject(ruleParametersResultFields); }); }); - - it('#getDataSafety', async () => { - const result = await getDataSafety(getDataFromFieldsHits, fields); - expect(result).toEqual(resultFields); - }); }); diff --git a/x-pack/plugins/timelines/common/utils/field_formatters.ts b/x-pack/plugins/timelines/common/utils/field_formatters.ts index c9292987f59b2..2e3785633bc3f 100644 --- a/x-pack/plugins/timelines/common/utils/field_formatters.ts +++ b/x-pack/plugins/timelines/common/utils/field_formatters.ts @@ -8,9 +8,15 @@ import { isEmpty } from 'lodash/fp'; import { ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; -import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; -import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; +import { + ecsFieldMap, + EcsFieldMap, +} from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; +import { + technicalRuleFieldMap, + TechnicalRuleFieldMap, +} from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; +import { legacyExperimentalFieldMap, ExperimentalRuleFieldMap } from '@kbn/alerts-as-data-utils'; import { Fields, TimelineEventsDetailsItem } from '../search_strategy'; import { toObjectArrayOfStrings, toStringArray } from './to_array'; import { ENRICHMENT_DESTINATION_PATH } from '../constants'; @@ -51,117 +57,141 @@ export const isRuleParametersFieldOrSubfield = (field: string, prependField?: st export const isThreatEnrichmentFieldOrSubfield = (field: string, prependField?: string) => prependField?.includes(ENRICHMENT_DESTINATION_PATH) || field === ENRICHMENT_DESTINATION_PATH; +// Helper functions +const createFieldItem = ( + fieldCategory: string, + field: string, + values: string[], + isObjectArray: boolean +): TimelineEventsDetailsItem => ({ + category: fieldCategory, + field, + values, + originalValue: values, + isObjectArray, +}); + +const processGeoField = ( + field: string, + item: unknown[], + fieldCategory: string +): TimelineEventsDetailsItem => { + const formattedLocation = formatGeoLocation(item); + return createFieldItem(fieldCategory, field, formattedLocation, true); +}; + +const processSimpleField = ( + dotField: string, + strArr: string[], + isObjectArray: boolean, + fieldCategory: string +): TimelineEventsDetailsItem => createFieldItem(fieldCategory, dotField, strArr, isObjectArray); + +const processNestedFields = ( + item: unknown, + dotField: string, + fieldCategory: string, + prependDotField: boolean +): TimelineEventsDetailsItem[] => { + if (Array.isArray(item)) { + return item.flatMap((curr) => + getDataFromFieldsHits(curr as Fields, prependDotField ? dotField : undefined, fieldCategory) + ); + } + + return getDataFromFieldsHits( + item as Fields, + prependDotField ? dotField : undefined, + fieldCategory + ); +}; + +type DisjointFieldNames = 'ecs.version' | 'event.action' | 'event.kind' | 'event.original'; + +// Memoized field maps +const fieldMaps: EcsFieldMap & + Omit & + ExperimentalRuleFieldMap = { + ...technicalRuleFieldMap, + ...ecsFieldMap, + ...legacyExperimentalFieldMap, +}; + export const getDataFromFieldsHits = ( fields: Fields, prependField?: string, prependFieldCategory?: string -): TimelineEventsDetailsItem[] => - Object.keys(fields).reduce((accumulator, field) => { +): TimelineEventsDetailsItem[] => { + const resultMap = new Map(); + const fieldNames = Object.keys(fields); + for (let i = 0; i < fieldNames.length; i++) { + const field = fieldNames[i]; const item: unknown[] = fields[field]; - const fieldCategory = - prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); + const fieldCategory = prependFieldCategory ?? getFieldCategory(field); + const dotField = prependField ? `${prependField}.${field}` : field; + + // Handle geo fields if (isGeoField(field)) { - return [ - ...accumulator, - { - category: fieldCategory, - field, - values: formatGeoLocation(item), - originalValue: formatGeoLocation(item), - isObjectArray: true, // important for UI - }, - ]; + const geoItem = processGeoField(field, item, fieldCategory); + resultMap.set(field, geoItem); + // eslint-disable-next-line no-continue + continue; } + const objArrStr = toObjectArrayOfStrings(item); const strArr = objArrStr.map(({ str }) => str); const isObjectArray = objArrStr.some((o) => o.isObjectArray); - const dotField = prependField ? `${prependField}.${field}` : field; - // return simple field value (non-ecs object, non-array) - if ( - !isObjectArray || - (Object.keys({ - ...ecsFieldMap, - ...technicalRuleFieldMap, - ...legacyExperimentalFieldMap, - }).find((ecsField) => ecsField === field) === undefined && - !isRuleParametersFieldOrSubfield(field, prependField)) - ) { - return [ - ...accumulator, - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ]; + const isEcsField = fieldMaps[field as keyof typeof fieldMaps] !== undefined; + const isRuleParameters = isRuleParametersFieldOrSubfield(field, prependField); + + // Handle simple fields + if (!isObjectArray || (!isEcsField && !isRuleParameters)) { + const simpleItem = processSimpleField(dotField, strArr, isObjectArray, fieldCategory); + resultMap.set(dotField, simpleItem); + // eslint-disable-next-line no-continue + continue; } - const threatEnrichmentObject = isThreatEnrichmentFieldOrSubfield(field, prependField) - ? [ - { - category: fieldCategory, - field: dotField, - values: strArr, - originalValue: strArr, - isObjectArray, - }, - ] - : []; - - // format nested fields - let nestedFields: TimelineEventsDetailsItem[] = []; - if (isRuleParametersFieldOrSubfield(field, prependField)) { - nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, dotField, fieldCategory); - } else { - nestedFields = Array.isArray(item) - ? item - .reduce((acc, curr) => { - acc.push(getDataFromFieldsHits(curr as Fields, dotField, fieldCategory)); - return acc; - }, []) - .flat() - : getDataFromFieldsHits(item, prependField, fieldCategory); + // Handle threat enrichment + if (isThreatEnrichmentFieldOrSubfield(field, prependField)) { + const enrichmentItem = createFieldItem(fieldCategory, dotField, strArr, isObjectArray); + resultMap.set(dotField, enrichmentItem); } - // combine duplicate fields - const flat: Record = [ - ...accumulator, - ...nestedFields, - ...threatEnrichmentObject, - ].reduce( - (acc, f) => ({ - ...acc, - // acc/flat is hashmap to determine if we already have the field or not without an array iteration - // its converted back to array in return with Object.values - ...(acc[f.field] != null - ? { - [f.field]: { - ...f, - originalValue: acc[f.field].originalValue.includes(f.originalValue[0]) - ? acc[f.field].originalValue - : [...acc[f.field].originalValue, ...f.originalValue], - values: acc[f.field].values?.includes(f.values?.[0] || '') - ? acc[f.field].values - : [...(acc[f.field].values || []), ...(f.values || [])], - }, - } - : { [f.field]: f }), - }), - {} as Record + // Process nested fields + const nestedFields = processNestedFields( + item, + dotField, + fieldCategory, + isRuleParameters || isThreatEnrichmentFieldOrSubfield(field, prependField) ); + // Merge results + for (const nestedItem of nestedFields) { + const existing = resultMap.get(nestedItem.field); + + if (!existing) { + resultMap.set(nestedItem.field, nestedItem); + // eslint-disable-next-line no-continue + continue; + } - return Object.values(flat); - }, []); + // Merge values and originalValue arrays + const mergedValues = existing.values?.includes(nestedItem.values?.[0] || '') + ? existing.values + : [...(existing.values || []), ...(nestedItem.values || [])]; -export const getDataSafety = (fn: (args: A) => T, args: A): Promise => - new Promise((resolve) => setTimeout(() => resolve(fn(args)))); + const mergedOriginal = existing.originalValue.includes(nestedItem.originalValue[0]) + ? existing.originalValue + : [...existing.originalValue, ...nestedItem.originalValue]; + + resultMap.set(nestedItem.field, { + ...nestedItem, + values: mergedValues, + originalValue: mergedOriginal, + }); + } + } + + return Array.from(resultMap.values()); +}; diff --git a/x-pack/plugins/timelines/common/utils/index.ts b/x-pack/plugins/timelines/common/utils/index.ts new file mode 100644 index 0000000000000..e4b7eefec454f --- /dev/null +++ b/x-pack/plugins/timelines/common/utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getDataFromFieldsHits, isGeoField } from './field_formatters'; +export { toArray, toObjectArrayOfStrings } from './to_array'; diff --git a/x-pack/plugins/timelines/common/utils/to_array.ts b/x-pack/plugins/timelines/common/utils/to_array.ts index fbb2b8d48a250..d13eb0578008b 100644 --- a/x-pack/plugins/timelines/common/utils/to_array.ts +++ b/x-pack/plugins/timelines/common/utils/to_array.ts @@ -5,83 +5,55 @@ * 2.0. */ -export const toArray = (value: T | T[] | null): T[] => - Array.isArray(value) ? value : value == null ? [] : [value]; -export const toStringArray = (value: T | T[] | null): string[] => { - if (Array.isArray(value)) { - return value.reduce((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, v.toString()]; - case 'object': - try { - return [...acc, JSON.stringify(v)]; - } catch { - return [...acc, 'Invalid Object']; - } - case 'string': - return [...acc, v]; - default: - return [...acc, `${v}`]; - } +export const toArray = (value: T | T[] | null | undefined): T[] => + value == null ? [] : Array.isArray(value) ? value : [value]; + +export const toStringArray = (value: T | T[] | null): string[] => { + if (value == null) return []; + + const arr = Array.isArray(value) ? value : [value]; + return arr.reduce((acc, v) => { + if (v == null) return acc; + + if (typeof v === 'object') { + try { + acc.push(JSON.stringify(v)); + } catch { + acc.push('Invalid Object'); } return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [JSON.stringify(value)]; - } catch { - return ['Invalid Object']; } - } else { - return [`${value}`]; - } + + acc.push(String(v)); + return acc; + }, []); }; -export const toObjectArrayOfStrings = ( + +export const toObjectArrayOfStrings = ( value: T | T[] | null ): Array<{ str: string; isObjectArray?: boolean; }> => { - if (Array.isArray(value)) { - return value.reduce< - Array<{ - str: string; - isObjectArray?: boolean; - }> - >((acc, v) => { - if (v != null) { - switch (typeof v) { - case 'number': - case 'boolean': - return [...acc, { str: v.toString() }]; - case 'object': - try { - return [...acc, { str: JSON.stringify(v), isObjectArray: true }]; // need to track when string is not a simple value - } catch { - return [...acc, { str: 'Invalid Object' }]; - } - case 'string': - return [...acc, { str: v }]; - default: - return [...acc, { str: `${v}` }]; - } + if (value == null) return []; + + const arr = Array.isArray(value) ? value : [value]; + return arr.reduce>((acc, v) => { + if (v == null) return acc; + + if (typeof v === 'object') { + try { + acc.push({ + str: JSON.stringify(v), + isObjectArray: true, + }); + } catch { + acc.push({ str: 'Invalid Object' }); } return acc; - }, []); - } else if (value == null) { - return []; - } else if (!Array.isArray(value) && typeof value === 'object') { - try { - return [{ str: JSON.stringify(value), isObjectArray: true }]; - } catch { - return [{ str: 'Invalid Object' }]; } - } else { - return [{ str: `${value}` }]; - } + + acc.push({ str: String(v) }); + return acc; + }, []); }; diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts index 2b4f562954df1..645f6daa5727d 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts @@ -17,7 +17,6 @@ import { } from '../../../../common/search_strategy'; import { TimelineEqlResponse } from '../../../../common/search_strategy/timeline/events/eql'; import { inspectStringifyObject } from '../../../utils/build_query'; -import { TIMELINE_EVENTS_FIELDS } from '../factory/helpers/constants'; import { formatTimelineData } from '../factory/helpers/format_timeline_data'; export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record => { @@ -68,38 +67,38 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record>, fieldRequested: string[]) => - sequences.reduce>(async (acc, sequence, sequenceIndex) => { +const parseSequences = async (sequences: Array>, fieldRequested: string[]) => { + let result: TimelineEdges[] = []; + + for (const [sequenceIndex, sequence] of sequences.entries()) { const sequenceParentId = sequence.events[0]?._id ?? null; - const data = await acc; - const allData = await Promise.all( - sequence.events.map(async (event, eventIndex) => { - const item = await formatTimelineData( - fieldRequested, - TIMELINE_EVENTS_FIELDS, - event as EventHit - ); - return Promise.resolve({ - ...item, - node: { - ...item.node, - ecs: { - ...item.node.ecs, - ...(sequenceParentId != null - ? { - eql: { - parentId: sequenceParentId, - sequenceNumber: `${sequenceIndex}-${eventIndex}`, - }, - } - : {}), - }, - }, - }); - }) + const formattedEvents = await formatTimelineData( + sequence.events as EventHit[], + fieldRequested, + false ); - return Promise.resolve([...data, ...allData]); - }, Promise.resolve([])); + + const eventsWithEql = formattedEvents.map((item, eventIndex) => ({ + ...item, + node: { + ...item.node, + ecs: { + ...item.node.ecs, + ...(sequenceParentId && { + eql: { + parentId: sequenceParentId, + sequenceNumber: `${sequenceIndex}-${eventIndex}`, + }, + }), + }, + }, + })); + + result = result.concat(eventsWithEql); + } + + return result; +}; export const parseEqlResponse = async ( options: TimelineEqlRequestOptions, @@ -116,10 +115,10 @@ export const parseEqlResponse = async ( if (response.rawResponse.hits.sequences !== undefined) { edges = await parseSequences(response.rawResponse.hits.sequences, options.fieldRequested); } else if (response.rawResponse.hits.events !== undefined) { - edges = await Promise.all( - response.rawResponse.hits.events.map(async (event) => - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) - ) + edges = await formatTimelineData( + response.rawResponse.hits.events as EventHit[], + options.fieldRequested, + false ); } diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts index 4ed857b4885e3..2ee2b64162c13 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts @@ -14,13 +14,11 @@ import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants import { EventHit, TimelineEventsAllStrategyResponse, - TimelineEdges, } from '../../../../../../common/search_strategy'; import { TimelineFactory } from '../../types'; import { buildTimelineEventsAllQuery } from './query.events_all.dsl'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { formatTimelineData } from '../../helpers/format_timeline_data'; -import { TIMELINE_EVENTS_FIELDS } from '../../helpers/constants'; export const timelineEventsAll: TimelineFactory = { buildDsl: ({ authFilter, ...options }) => { @@ -54,14 +52,10 @@ export const timelineEventsAll: TimelineFactory = { fieldRequested = [...new Set(fieldsReturned)]; } - const edges: TimelineEdges[] = await Promise.all( - hits.map((hit) => - formatTimelineData( - fieldRequested, - options.excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS, - hit as EventHit - ) - ) + const edges = await formatTimelineData( + hits as EventHit[], + fieldRequested, + options.excludeEcsData ?? false ); const consumers = producerBuckets.reduce( diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts index 46edcf926d061..fcb90b73c241e 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/details/index.ts @@ -12,15 +12,11 @@ import { TimelineEventsQueries } from '../../../../../../common/api/search_strat import { EventHit, TimelineEventsDetailsStrategyResponse, - TimelineEventsDetailsItem, } from '../../../../../../common/search_strategy'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { TimelineFactory } from '../../types'; import { buildTimelineDetailsQuery } from './query.events_details.dsl'; -import { - getDataFromFieldsHits, - getDataSafety, -} from '../../../../../../common/utils/field_formatters'; +import { getDataFromFieldsHits } from '../../../../../../common/utils/field_formatters'; import { buildEcsObjects } from '../../helpers/build_ecs_objects'; export const timelineEventsDetails: TimelineFactory = { @@ -57,10 +53,7 @@ export const timelineEventsDetails: TimelineFactory( - getDataFromFieldsHits, - merge(fields, hitsData) - ); + const fieldsData = getDataFromFieldsHits(merge(fields, hitsData)); const rawEventData = response.rawResponse.hits.hits[0]; const ecs = buildEcsObjects(rawEventData as EventHit); diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts index 5117f8dc889ed..3e96494c88313 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.test.ts @@ -7,12 +7,12 @@ import { eventHit } from '@kbn/securitysolution-t-grid'; import { EventHit } from '../../../../../common/search_strategy'; -import { TIMELINE_EVENTS_FIELDS } from './constants'; import { formatTimelineData } from './format_timeline_data'; describe('formatTimelineData', () => { it('should properly format the timeline data', async () => { const res = await formatTimelineData( + [eventHit], [ '@timestamp', 'host.name', @@ -21,187 +21,188 @@ describe('formatTimelineData', () => { 'source.geo.location', 'threat.enrichments.matched.field', ], - TIMELINE_EVENTS_FIELDS, - eventHit + false ); - expect(res).toEqual({ - cursor: { - tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', - value: '1605624488922', - }, - node: { - _id: 'tkCt1nUBaEgqnrVSZ8R_', - _index: 'auditbeat-7.8.0-2020.11.05-000003', - data: [ - { - field: '@timestamp', - value: ['2020-11-17T14:48:08.922Z'], - }, - { - field: 'host.name', - value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - }, - { - field: 'threat.enrichments.matched.field', - value: [ - 'matched_field', - 'other_matched_field', - 'matched_field_2', - 'host.name', - 'host.hostname', - 'host.architecture', - ], - }, - { - field: 'source.geo.location', - value: [`{"lon":118.7778,"lat":32.0617}`], - }, - ], - ecs: { - '@timestamp': ['2020-11-17T14:48:08.922Z'], + expect(res).toEqual([ + { + cursor: { + tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', + value: '1605624488922', + }, + node: { _id: 'tkCt1nUBaEgqnrVSZ8R_', _index: 'auditbeat-7.8.0-2020.11.05-000003', - agent: { - type: ['auditbeat'], - }, - event: { - action: ['process_started'], - category: ['process'], - dataset: ['process'], - kind: ['event'], - module: ['system'], - type: ['start'], - }, - host: { - id: ['e59991e835905c65ed3e455b33e13bd6'], - ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - os: { - family: ['debian'], + data: [ + { + field: '@timestamp', + value: ['2020-11-17T14:48:08.922Z'], }, - }, - message: ['Process go (PID: 4313) by user jenkins STARTED'], - process: { - args: ['go', 'vet', './...'], - entity_id: ['Z59cIkAAIw8ZoK0H'], - executable: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - hash: { - sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - }, - name: ['go'], - pid: ['4313'], - ppid: ['3977'], - working_directory: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - }, - timestamp: '2020-11-17T14:48:08.922Z', - user: { - name: ['jenkins'], - }, - threat: { - enrichments: [ - { - feed: { name: [] }, - indicator: { - provider: ['yourself'], - reference: [], - }, - matched: { - atomic: ['matched_atomic'], - field: ['matched_field', 'other_matched_field'], - type: [], - }, - }, - { - feed: { name: [] }, - indicator: { - provider: ['other_you'], - reference: [], - }, - matched: { - atomic: ['matched_atomic_2'], - field: ['matched_field_2'], - type: [], - }, - }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], - }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.name'], - type: ['indicator_match_rule'], - }, + { + field: 'host.name', + value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + }, + { + field: 'threat.enrichments.matched.field', + value: [ + 'matched_field', + 'other_matched_field', + 'matched_field_2', + 'host.name', + 'host.hostname', + 'host.architecture', + ], + }, + { + field: 'source.geo.location', + value: [`{"lon":118.7778,"lat":32.0617}`], + }, + ], + ecs: { + '@timestamp': ['2020-11-17T14:48:08.922Z'], + _id: 'tkCt1nUBaEgqnrVSZ8R_', + _index: 'auditbeat-7.8.0-2020.11.05-000003', + agent: { + type: ['auditbeat'], + }, + event: { + action: ['process_started'], + category: ['process'], + dataset: ['process'], + kind: ['event'], + module: ['system'], + type: ['start'], + }, + host: { + id: ['e59991e835905c65ed3e455b33e13bd6'], + ip: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + name: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + os: { + family: ['debian'], }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], - }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.hostname'], - type: ['indicator_match_rule'], - }, + }, + message: ['Process go (PID: 4313) by user jenkins STARTED'], + process: { + args: ['go', 'vet', './...'], + entity_id: ['Z59cIkAAIw8ZoK0H'], + executable: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + hash: { + sha1: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], }, - { - feed: { - name: [], - }, - indicator: { - provider: [], - reference: [], + name: ['go'], + pid: ['4313'], + ppid: ['3977'], + working_directory: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + }, + timestamp: '2020-11-17T14:48:08.922Z', + user: { + name: ['jenkins'], + }, + threat: { + enrichments: [ + { + feed: { name: [] }, + indicator: { + provider: ['yourself'], + reference: [], + }, + matched: { + atomic: ['matched_atomic'], + field: ['matched_field', 'other_matched_field'], + type: [], + }, }, - matched: { - atomic: ['x86_64'], - field: ['host.architecture'], - type: ['indicator_match_rule'], + { + feed: { name: [] }, + indicator: { + provider: ['other_you'], + reference: [], + }, + matched: { + atomic: ['matched_atomic_2'], + field: ['matched_field_2'], + type: [], + }, }, - }, - { - feed: { - name: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.name'], + type: ['indicator_match_rule'], + }, }, - indicator: { - provider: [], - reference: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.hostname'], + type: ['indicator_match_rule'], + }, }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.name'], - type: ['indicator_match_rule'], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['x86_64'], + field: ['host.architecture'], + type: ['indicator_match_rule'], + }, }, - }, - { - feed: { - name: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.name'], + type: ['indicator_match_rule'], + }, }, - indicator: { - provider: [], - reference: [], + { + feed: { + name: [], + }, + indicator: { + provider: [], + reference: [], + }, + matched: { + atomic: ['MacBook-Pro-de-Gloria.local'], + field: ['host.hostname'], + type: ['indicator_match_rule'], + }, }, - matched: { - atomic: ['MacBook-Pro-de-Gloria.local'], - field: ['host.hostname'], - type: ['indicator_match_rule'], - }, - }, - ], + ], + }, }, }, }, - }); + ]); }); it('should properly format the rule signal results', async () => { @@ -240,57 +241,61 @@ describe('formatTimelineData', () => { expect( await formatTimelineData( + [response], ['@timestamp', 'host.name', 'destination.ip', 'source.ip'], - TIMELINE_EVENTS_FIELDS, - response + false ) - ).toEqual({ - cursor: { - tiebreaker: null, - value: '', - }, - node: { - _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', - _index: '.siem-signals-patrykkopycinski-default-000007', - data: [ - { - field: '@timestamp', - value: ['2021-01-09T13:41:40.517Z'], - }, - ], - ecs: { - '@timestamp': ['2021-01-09T13:41:40.517Z'], - timestamp: '2021-01-09T13:41:40.517Z', + ).toEqual([ + { + cursor: { + tiebreaker: null, + value: '', + }, + node: { _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', _index: '.siem-signals-patrykkopycinski-default-000007', - event: { - kind: ['signal'], - }, - kibana: { - alert: { - original_time: ['2021-01-09T13:39:32.595Z'], - workflow_status: ['open'], - threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'], - severity: ['low'], - risk_score: ['21'], - rule: { - building_block_type: [], - exceptions_list: [], - from: ['now-360s'], - uuid: ['696c24e0-526d-11eb-836c-e1620268b945'], - name: ['Threshold test'], - to: ['now'], - type: ['threshold'], - version: ['1'], - timeline_id: [], - timeline_title: [], - note: [], + data: [ + { + field: '@timestamp', + value: ['2021-01-09T13:41:40.517Z'], + }, + ], + ecs: { + '@timestamp': ['2021-01-09T13:41:40.517Z'], + timestamp: '2021-01-09T13:41:40.517Z', + _id: 'a77040f198355793c35bf22b900902371309be615381f0a2ec92c208b6132562', + _index: '.siem-signals-patrykkopycinski-default-000007', + event: { + kind: ['signal'], + }, + kibana: { + alert: { + original_time: ['2021-01-09T13:39:32.595Z'], + workflow_status: ['open'], + threshold_result: [ + '{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}', + ], + severity: ['low'], + risk_score: ['21'], + rule: { + building_block_type: [], + exceptions_list: [], + from: ['now-360s'], + uuid: ['696c24e0-526d-11eb-836c-e1620268b945'], + name: ['Threshold test'], + to: ['now'], + type: ['threshold'], + version: ['1'], + timeline_id: [], + timeline_title: [], + note: [], + }, }, }, }, }, }, - }); + ]); }); it('should properly format the inventory rule signal results', async () => { @@ -347,6 +352,7 @@ describe('formatTimelineData', () => { expect( await formatTimelineData( + [response], [ 'kibana.alert.status', '@timestamp', @@ -376,168 +382,169 @@ describe('formatTimelineData', () => { 'event.kind', 'kibana.alert.rule.parameters', ], - TIMELINE_EVENTS_FIELDS, - response + false ) - ).toEqual({ - cursor: { - tiebreaker: null, - value: '', - }, - node: { - _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', - _index: '.internal.alerts-observability.metrics.alerts-default-000001', - data: [ - { - field: 'kibana.alert.rule.consumer', - value: ['infrastructure'], - }, - { - field: '@timestamp', - value: ['2022-07-21T22:38:57.888Z'], - }, - { - field: 'kibana.alert.workflow_status', - value: ['open'], - }, - { - field: 'kibana.alert.reason', - value: [ - 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', - ], - }, - { - field: 'kibana.alert.rule.name', - value: ['test 1212'], - }, - { - field: 'kibana.alert.rule.uuid', - value: ['15d82f10-0926-11ed-bece-6b0c033d0075'], - }, - { - field: 'kibana.alert.rule.parameters.sourceId', - value: ['default'], - }, - { - field: 'kibana.alert.rule.parameters.nodeType', - value: ['host'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.comparator', - value: ['>'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.timeSize', - value: ['1'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.metric', - value: ['cpu'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.threshold', - value: ['10'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation', - value: ['avg'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.id', - value: ['alert-custom-metric'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.field', - value: [''], - }, - { - field: 'kibana.alert.rule.parameters.criteria.customMetric.type', - value: ['custom'], - }, - { - field: 'kibana.alert.rule.parameters.criteria.timeUnit', - value: ['d'], - }, - { - field: 'event.action', - value: ['active'], - }, - { - field: 'event.kind', - value: ['signal'], - }, - { - field: 'kibana.alert.status', - value: ['active'], - }, - { - field: 'kibana.alert.duration.us', - value: ['9502040000'], - }, - { - field: 'kibana.alert.rule.category', - value: ['Inventory'], - }, - { - field: 'kibana.alert.uuid', - value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'], - }, - { - field: 'kibana.alert.start', - value: ['2022-07-21T20:00:35.848Z'], - }, - { - field: 'kibana.alert.rule.producer', - value: ['infrastructure'], - }, - { - field: 'kibana.alert.rule.rule_type_id', - value: ['metrics.alert.inventory.threshold'], - }, - { - field: 'kibana.alert.instance.id', - value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'], - }, - { - field: 'kibana.alert.rule.execution.uuid', - value: ['37498c42-0190-4a83-adfa-c7e5f817f977'], - }, - { - field: 'kibana.space_ids', - value: ['default'], - }, - { - field: 'kibana.version', - value: ['8.4.0'], - }, - ], - ecs: { - '@timestamp': ['2022-07-21T22:38:57.888Z'], + ).toEqual([ + { + cursor: { + tiebreaker: null, + value: '', + }, + node: { _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', _index: '.internal.alerts-observability.metrics.alerts-default-000001', - event: { - action: ['active'], - kind: ['signal'], - }, - kibana: { - alert: { - reason: [ + data: [ + { + field: 'kibana.alert.rule.consumer', + value: ['infrastructure'], + }, + { + field: '@timestamp', + value: ['2022-07-21T22:38:57.888Z'], + }, + { + field: 'kibana.alert.workflow_status', + value: ['open'], + }, + { + field: 'kibana.alert.reason', + value: [ 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', ], - rule: { - consumer: ['infrastructure'], - name: ['test 1212'], - uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'], - parameters: [ - '{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}', + }, + { + field: 'kibana.alert.rule.name', + value: ['test 1212'], + }, + { + field: 'kibana.alert.rule.uuid', + value: ['15d82f10-0926-11ed-bece-6b0c033d0075'], + }, + { + field: 'kibana.alert.rule.parameters.sourceId', + value: ['default'], + }, + { + field: 'kibana.alert.rule.parameters.nodeType', + value: ['host'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.comparator', + value: ['>'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.timeSize', + value: ['1'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.metric', + value: ['cpu'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.threshold', + value: ['10'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.aggregation', + value: ['avg'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.id', + value: ['alert-custom-metric'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.field', + value: [''], + }, + { + field: 'kibana.alert.rule.parameters.criteria.customMetric.type', + value: ['custom'], + }, + { + field: 'kibana.alert.rule.parameters.criteria.timeUnit', + value: ['d'], + }, + { + field: 'event.action', + value: ['active'], + }, + { + field: 'event.kind', + value: ['signal'], + }, + { + field: 'kibana.alert.status', + value: ['active'], + }, + { + field: 'kibana.alert.duration.us', + value: ['9502040000'], + }, + { + field: 'kibana.alert.rule.category', + value: ['Inventory'], + }, + { + field: 'kibana.alert.uuid', + value: ['3fef4a4c-3d96-4e79-b4e5-158a0461d577'], + }, + { + field: 'kibana.alert.start', + value: ['2022-07-21T20:00:35.848Z'], + }, + { + field: 'kibana.alert.rule.producer', + value: ['infrastructure'], + }, + { + field: 'kibana.alert.rule.rule_type_id', + value: ['metrics.alert.inventory.threshold'], + }, + { + field: 'kibana.alert.instance.id', + value: ['gke-edge-oblt-pool-1-9a60016d-7dvq'], + }, + { + field: 'kibana.alert.rule.execution.uuid', + value: ['37498c42-0190-4a83-adfa-c7e5f817f977'], + }, + { + field: 'kibana.space_ids', + value: ['default'], + }, + { + field: 'kibana.version', + value: ['8.4.0'], + }, + ], + ecs: { + '@timestamp': ['2022-07-21T22:38:57.888Z'], + _id: '3fef4a4c-3d96-4e79-b4e5-158a0461d577', + _index: '.internal.alerts-observability.metrics.alerts-default-000001', + event: { + action: ['active'], + kind: ['signal'], + }, + kibana: { + alert: { + reason: [ + 'CPU usage is 37.8% in the last 1 day for gke-edge-oblt-pool-1-9a60016d-7dvq. Alert when > 10%.', ], + rule: { + consumer: ['infrastructure'], + name: ['test 1212'], + uuid: ['15d82f10-0926-11ed-bece-6b0c033d0075'], + parameters: [ + '{"sourceId":"default","nodeType":"host","criteria":[{"comparator":">","timeSize":1,"metric":"cpu","threshold":[10],"customMetric":{"aggregation":"avg","id":"alert-custom-metric","field":"","type":"custom"},"timeUnit":"d"}]}', + ], + }, + workflow_status: ['open'], }, - workflow_status: ['open'], }, + timestamp: '2022-07-21T22:38:57.888Z', }, - timestamp: '2022-07-21T22:38:57.888Z', }, }, - }); + ]); }); }); diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts index f56cfd32391d4..485ec64badd5c 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/helpers/format_timeline_data.ts @@ -5,118 +5,130 @@ * 2.0. */ -import { get, has, merge, uniq } from 'lodash/fp'; -import { EventHit, TimelineEdges, TimelineNonEcsData } from '../../../../../common/search_strategy'; +import { get, has } from 'lodash/fp'; +import { + EventHit, + TimelineEdges, + TimelineNonEcsData, + EventSource, +} from '../../../../../common/search_strategy'; import { toStringArray } from '../../../../../common/utils/to_array'; -import { getDataFromFieldsHits, getDataSafety } from '../../../../../common/utils/field_formatters'; +import { getDataFromFieldsHits } from '../../../../../common/utils/field_formatters'; import { getTimestamp } from './get_timestamp'; import { getNestedParentPath } from './get_nested_parent_path'; import { buildObjectRecursive } from './build_object_recursive'; -import { ECS_METADATA_FIELDS } from './constants'; +import { ECS_METADATA_FIELDS, TIMELINE_EVENTS_FIELDS } from './constants'; -export const formatTimelineData = async ( - dataFields: readonly string[], - ecsFields: readonly string[], - hit: EventHit -) => - uniq([...ecsFields, ...dataFields]).reduce>( - async (acc, fieldName) => { - const flattenedFields: TimelineEdges = await acc; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - flattenedFields.node._id = hit._id!; - flattenedFields.node._index = hit._index; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - flattenedFields.node.ecs._id = hit._id!; - flattenedFields.node.ecs.timestamp = getTimestamp(hit); - flattenedFields.node.ecs._index = hit._index; - if (hit.sort && hit.sort.length > 1) { - flattenedFields.cursor.value = hit.sort[0]; - flattenedFields.cursor.tiebreaker = hit.sort[1]; - } - const waitForIt = await mergeTimelineFieldsWithHit( - fieldName, - flattenedFields, - hit, - dataFields, - ecsFields - ); - return Promise.resolve(waitForIt); - }, - Promise.resolve({ - node: { ecs: { _id: '' }, data: [], _id: '', _index: '' }, - cursor: { - value: '', - tiebreaker: null, - }, - }) - ); - -const getValuesFromFields = async ( +const createBaseTimelineEdges = (): TimelineEdges => ({ + node: { + ecs: { _id: '' }, + data: [], + _id: '', + _index: '', + }, + cursor: { + value: '', + tiebreaker: null, + }, +}); + +function deepMerge(target: EventSource, source: EventSource) { + for (const key in source) { + if (source[key] instanceof Object && key in target) { + deepMerge(target[key], source[key]); + } else { + target[key] = source[key]; + } + } + return target; +} + +const processMetadataField = (fieldName: string, hit: EventHit): TimelineNonEcsData[] => [ + { + field: fieldName, + value: toStringArray(get(fieldName, hit)), + }, +]; + +const processFieldData = ( fieldName: string, hit: EventHit, nestedParentFieldName?: string -): Promise => { - if (ECS_METADATA_FIELDS.includes(fieldName)) { - return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }]; - } +): TimelineNonEcsData[] => { + const fieldToEval = nestedParentFieldName + ? { [nestedParentFieldName]: hit.fields[nestedParentFieldName] } + : { [fieldName]: hit.fields[fieldName] }; - let fieldToEval; - - if (nestedParentFieldName == null) { - fieldToEval = { - [fieldName]: hit.fields[fieldName], - }; - } else { - fieldToEval = { - [nestedParentFieldName]: hit.fields[nestedParentFieldName], - }; - } - const formattedData = await getDataSafety(getDataFromFieldsHits, fieldToEval); - return formattedData.reduce((acc: TimelineNonEcsData[], { field, values }) => { - // nested fields return all field values, pick only the one we asked for + const formattedData = getDataFromFieldsHits(fieldToEval); + const fieldsData: TimelineNonEcsData[] = []; + return formattedData.reduce((agg, { field, values }) => { if (field.includes(fieldName)) { - acc.push({ field, value: values }); + agg.push({ + field, + value: values, + }); } - return acc; - }, []); + return agg; + }, fieldsData); }; -const mergeTimelineFieldsWithHit = async ( - fieldName: string, - flattenedFields: T, - hit: EventHit, - dataFields: readonly string[], - ecsFields: readonly string[] -) => { - if (fieldName != null) { - const nestedParentPath = getNestedParentPath(fieldName, hit.fields); - if ( - nestedParentPath != null || - has(fieldName, hit.fields) || - ECS_METADATA_FIELDS.includes(fieldName) - ) { - const objectWithProperty = { - node: { - ...get('node', flattenedFields), - data: dataFields.includes(fieldName) - ? [ - ...get('node.data', flattenedFields), - ...(await getValuesFromFields(fieldName, hit, nestedParentPath)), - ] - : get('node.data', flattenedFields), - ecs: ecsFields.includes(fieldName) - ? { - ...get('node.ecs', flattenedFields), - ...buildObjectRecursive(fieldName, hit.fields), - } - : get('node.ecs', flattenedFields), - }, - }; - return merge(flattenedFields, objectWithProperty); +export const formatTimelineData = async ( + hits: EventHit[], + fieldRequested: readonly string[], + excludeEcsData: boolean +): Promise => { + const ecsFields = excludeEcsData ? [] : TIMELINE_EVENTS_FIELDS; + + const uniqueFields = new Set([...ecsFields, ...fieldRequested]); + const dataFieldSet = new Set(fieldRequested); + const ecsFieldSet = new Set(ecsFields); + + const results: TimelineEdges[] = new Array(hits.length); + + for (let i = 0; i < hits.length; i++) { + const hit = hits[i]; + if (hit._id) { + const result = createBaseTimelineEdges(); + + result.node._id = hit._id; + result.node._index = hit._index; + result.node.ecs._id = hit._id; + result.node.ecs.timestamp = getTimestamp(hit); + result.node.ecs._index = hit._index; + + if (hit.sort?.length > 1) { + result.cursor.value = hit.sort[0]; + result.cursor.tiebreaker = hit.sort[1]; + } + + result.node.data = []; + + for (const fieldName of uniqueFields) { + const nestedParentPath = getNestedParentPath(fieldName, hit.fields); + const isEcs = ECS_METADATA_FIELDS.includes(fieldName); + if (!nestedParentPath && !has(fieldName, hit.fields) && !isEcs) { + // eslint-disable-next-line no-continue + continue; + } + + if (dataFieldSet.has(fieldName)) { + const values = isEcs + ? processMetadataField(fieldName, hit) + : processFieldData(fieldName, hit, nestedParentPath); + + result.node.data.push(...values); + } + + if (ecsFieldSet.has(fieldName)) { + deepMerge(result.node.ecs, buildObjectRecursive(fieldName, hit.fields)); + } + } + + results[i] = result; } else { - return flattenedFields; + results[i] = createBaseTimelineEdges(); } - } else { - return flattenedFields; } + + return results; };