From 0131ae07b24b0b83f5c4d973acc90cbee0140e9a Mon Sep 17 00:00:00 2001 From: "Joey F. Poon" Date: Mon, 16 Dec 2024 09:07:34 -0800 Subject: [PATCH] Updates workflow insights to use a hashed id (#204157) ## Summary Updates workflow insights to use a hashed id for uniqueness. Create method updated to check if a duplicate insight exists and update instead if it does. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Konrad Szwarc --- .../workflow_insights/helpers.test.ts | 71 ++++++++++++++++++- .../services/workflow_insights/helpers.ts | 21 +++++- .../services/workflow_insights/index.test.ts | 67 +++++++++++------ .../services/workflow_insights/index.ts | 12 +++- 4 files changed, 144 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts index c3fd5b36ee1e2..50184413063cf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts @@ -5,25 +5,34 @@ * 2.0. */ +import moment from 'moment'; +import { merge } from 'lodash'; + import type { ElasticsearchClient } from '@kbn/core/server'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { kibanaPackageJson } from '@kbn/repo-info'; +import { DefendInsightType } from '@kbn/elastic-assistant-common'; import type { HostMetadata } from '../../../../common/endpoint/types'; -import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights'; +import type { + SearchParams, + SecurityWorkflowInsight, +} from '../../../../common/endpoint/types/workflow_insights'; import { ActionType, Category, SourceType, + TargetType, } from '../../../../common/endpoint/types/workflow_insights'; import type { EndpointMetadataService } from '../metadata'; import { buildEsQueryParams, createDatastream, createPipeline, + generateInsightId, groupEndpointIdsByOS, } from './helpers'; import { @@ -43,6 +52,57 @@ jest.mock('@kbn/data-stream-adapter', () => ({ })), })); +function getDefaultInsight(overrides?: Partial): SecurityWorkflowInsight { + const defaultInsight = { + '@timestamp': moment(), + message: 'This is a test message', + category: Category.Endpoint, + type: DefendInsightType.Enum.incompatible_antivirus, + source: { + type: SourceType.LlmConnector, + id: 'openai-connector-id', + data_range_start: moment(), + data_range_end: moment(), + }, + target: { + type: TargetType.Endpoint, + ids: ['endpoint-1', 'endpoint-2'], + }, + action: { + type: ActionType.Refreshed, + timestamp: moment(), + }, + value: 'unique-key', + remediation: { + exception_list_items: [ + { + list_id: 'example-list-id', + name: 'Example List Name', + description: 'Example description', + entries: [ + { + field: 'example-field', + operator: 'included', + type: 'match', + value: 'example-value', + }, + ], + tags: ['example-tag'], + os_types: ['windows', 'linux'], + }, + ], + }, + metadata: { + notes: { + key1: 'value1', + key2: 'value2', + }, + message_variables: ['variable1', 'variable2'], + }, + }; + return merge(defaultInsight, overrides); +} + describe('helpers', () => { describe('createDatastream', () => { it('should create a DataStreamSpacesAdapter with the correct configuration', () => { @@ -193,4 +253,13 @@ describe('helpers', () => { }); }); }); + + describe('generateInsightId', () => { + it('should generate the correct hashed id', () => { + const insight = getDefaultInsight(); + const result = generateInsightId(insight); + const expectedHash = '6b1a7a9625decbf899db4fbf78105a0eff9ef98e3f2dadc2781d59996b55445e'; + expect(result).toBe(expectedHash); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts index 67c852cc2720b..f7b477a17018d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { createHash } from 'crypto'; import { get as _get } from 'lodash'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; @@ -12,7 +13,10 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; -import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights'; +import type { + SearchParams, + SecurityWorkflowInsight, +} from '../../../../common/endpoint/types/workflow_insights'; import type { SupportedHostOsType } from '../../../../common/endpoint/constants'; import type { EndpointMetadataService } from '../metadata'; @@ -130,3 +134,18 @@ export async function groupEndpointIdsByOS( return acc; }, {}); } + +export function generateInsightId(insight: SecurityWorkflowInsight): string { + const { type, category, value, target } = insight; + const targetType = target.type; + const targetIds = target.ids.join(','); + + const hash = createHash('sha256'); + hash.update(type); + hash.update(category); + hash.update(value); + hash.update(targetType); + hash.update(targetIds); + + return hash.digest('hex'); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts index c742daa90d258..849c6431a09a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts @@ -5,12 +5,13 @@ * 2.0. */ -import { merge } from 'lodash'; +import { merge, cloneDeep } from 'lodash'; import moment from 'moment'; import { ReplaySubject } from 'rxjs'; import type { ElasticsearchClient, KibanaRequest, Logger } from '@kbn/core/server'; import type { DefendInsight, DefendInsightsPostRequestBody } from '@kbn/elastic-assistant-common'; +import type { SearchHit, UpdateResponse } from '@elastic/elasticsearch/lib/api/types'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { DefendInsightType } from '@kbn/elastic-assistant-common'; @@ -31,7 +32,7 @@ import { ActionType, } from '../../../../common/endpoint/types/workflow_insights'; import { createMockEndpointAppContext } from '../../mocks'; -import { createDatastream, createPipeline } from './helpers'; +import { createDatastream, createPipeline, generateInsightId } from './helpers'; import { securityWorkflowInsightsService } from '.'; import { DATA_STREAM_NAME } from './constants'; import { buildWorkflowInsights } from './builders'; @@ -108,6 +109,7 @@ describe('SecurityWorkflowInsightsService', () => { let logger: Logger; let esClient: ElasticsearchClient; let mockEndpointAppContextService: jest.Mocked; + let isInitializedSpy: jest.SpyInstance, [], boolean>; beforeEach(() => { logger = loggerMock.create(); @@ -115,6 +117,10 @@ describe('SecurityWorkflowInsightsService', () => { mockEndpointAppContextService = createMockEndpointAppContext() .service as jest.Mocked; + + isInitializedSpy = jest + .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') + .mockResolvedValueOnce([undefined, undefined]); }); afterEach(() => { @@ -217,10 +223,6 @@ describe('SecurityWorkflowInsightsService', () => { describe('createFromDefendInsights', () => { it('should create workflow insights from defend insights', async () => { - const isInitializedSpy = jest - .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') - .mockResolvedValueOnce([undefined, undefined]); - const defendInsights: DefendInsight[] = [ { group: 'AVGAntivirus', @@ -258,8 +260,8 @@ describe('SecurityWorkflowInsightsService', () => { request ); - // twice since it calls securityWorkflowInsightsService.create internally - expect(isInitializedSpy).toHaveBeenCalledTimes(2); + // three since it calls securityWorkflowInsightsService.create + fetch + expect(isInitializedSpy).toHaveBeenCalledTimes(3); expect(buildWorkflowInsightsMock).toHaveBeenCalledWith({ defendInsights, request, @@ -271,32 +273,55 @@ describe('SecurityWorkflowInsightsService', () => { describe('create', () => { it('should index the doc correctly', async () => { - const isInitializedSpy = jest - .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') - .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.start({ esClient }); const insight = getDefaultInsight(); await securityWorkflowInsightsService.create(insight); - // ensure it waits for initialization first - expect(isInitializedSpy).toHaveBeenCalledTimes(1); + // two since it calls fetch as well + expect(isInitializedSpy).toHaveBeenCalledTimes(2); // indexes the doc expect(esClient.index).toHaveBeenCalledTimes(1); expect(esClient.index).toHaveBeenCalledWith({ index: DATA_STREAM_NAME, - body: insight, + body: { ...insight, id: generateInsightId(insight) }, refresh: 'wait_for', }); }); + + it('should call update instead if insight already exists', async () => { + const indexName = 'backing-index'; + const fetchSpy = jest + .spyOn(securityWorkflowInsightsService, 'fetch') + .mockResolvedValueOnce([{ _index: indexName }] as Array< + SearchHit + >); + const updateSpy = jest + .spyOn(securityWorkflowInsightsService, 'update') + .mockResolvedValueOnce({} as UpdateResponse); + await securityWorkflowInsightsService.start({ esClient }); + const insight = getDefaultInsight(); + await securityWorkflowInsightsService.create(insight); + + const expectedInsight = cloneDeep(insight); + expectedInsight['@timestamp'] = expect.any(moment); + expectedInsight.action.timestamp = expect.any(moment); + expectedInsight.source.data_range_start = expect.any(moment); + expectedInsight.source.data_range_end = expect.any(moment); + + // ensure it waits for initialization first + expect(isInitializedSpy).toHaveBeenCalledTimes(1); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledWith( + generateInsightId(insight), + expect.objectContaining(insight), + indexName + ); + }); }); describe('update', () => { it('should update the doc correctly', async () => { - const isInitializedSpy = jest - .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') - .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.start({ esClient }); const insightId = 'some-insight-id'; const insight = getDefaultInsight(); @@ -318,10 +343,6 @@ describe('SecurityWorkflowInsightsService', () => { describe('fetch', () => { it('should fetch the docs with the correct params', async () => { - const isInitializedSpy = jest - .spyOn(securityWorkflowInsightsService, 'isInitialized', 'get') - .mockResolvedValueOnce([undefined, undefined]); - await securityWorkflowInsightsService.start({ esClient }); const searchParams: SearchParams = { size: 50, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts index d1c765196db29..1baeaf74e00f0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts @@ -23,7 +23,7 @@ import type { import type { EndpointAppContextService } from '../../endpoint_app_context_services'; import { SecurityWorkflowInsightsFailedInitialized } from './errors'; -import { buildEsQueryParams, createDatastream, createPipeline } from './helpers'; +import { buildEsQueryParams, createDatastream, createPipeline, generateInsightId } from './helpers'; import { DATA_STREAM_NAME } from './constants'; import { buildWorkflowInsights } from './builders'; @@ -135,9 +135,17 @@ class SecurityWorkflowInsightsService { public async create(insight: SecurityWorkflowInsight): Promise { await this.isInitialized; + const id = generateInsightId(insight); + + // if insight already exists, update instead + const existingInsights = await this.fetch({ ids: [id] }); + if (existingInsights.length) { + return this.update(id, insight, existingInsights[0]._index); + } + const response = await this.esClient.index({ index: DATA_STREAM_NAME, - body: insight, + body: { ...insight, id }, refresh: 'wait_for', });