Skip to content

Commit

Permalink
[8.x] Updates workflow insights to use a hashed id (elastic#204157) (e…
Browse files Browse the repository at this point in the history 
…lastic#204455)

# Backport

This will backport the following commits from `main` to `8.x`:
- [Updates workflow insights to use a hashed id
(elastic#204157)](elastic#204157)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Joey F.
Poon","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-12-16T17:07:34Z","message":"Updates
workflow insights to use a hashed id (elastic#204157)\n\n##
Summary\r\n\r\nUpdates workflow insights to use a hashed id for
uniqueness. Create\r\nmethod updated to check if a duplicate insight
exists and update instead\r\nif it does.\r\n\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Konrad Szwarc
<[email protected]>","sha":"edc79773f5bb9f698959aca2bce787d74bba784d","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Defend
Workflows","backport:prev-minor","v8.18.0"],"title":"Updates workflow
insights to use a hashed
id","number":204157,"url":"https://github.com/elastic/kibana/pull/204157","mergeCommit":{"message":"Updates
workflow insights to use a hashed id (elastic#204157)\n\n##
Summary\r\n\r\nUpdates workflow insights to use a hashed id for
uniqueness. Create\r\nmethod updated to check if a duplicate insight
exists and update instead\r\nif it does.\r\n\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Konrad Szwarc
<[email protected]>","sha":"edc79773f5bb9f698959aca2bce787d74bba784d"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/204157","number":204157,"mergeCommit":{"message":"Updates
workflow insights to use a hashed id (elastic#204157)\n\n##
Summary\r\n\r\nUpdates workflow insights to use a hashed id for
uniqueness. Create\r\nmethod updated to check if a duplicate insight
exists and update instead\r\nif it does.\r\n\r\n\r\n###
Checklist\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Konrad Szwarc
<[email protected]>","sha":"edc79773f5bb9f698959aca2bce787d74bba784d"}},{"branch":"8.x","label":"v8.18.0","branchLabelMappingKey":"^v8.18.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Joey F. Poon <[email protected]>
  • Loading branch information
@kibanamachine @joeypoon
kibanamachine and joeypoon authored Dec 16, 2024
1 parent 2fe297d commit 31fa4a2
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 27 deletions.
71 changes: 70 additions & 1 deletion x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,34 @@
* 2.0.
*/

import moment from 'moment';
import { merge } from 'lodash';

import type { ElasticsearchClient } from '@kbn/core/server';

import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks';
import { kibanaPackageJson } from '@kbn/repo-info';
import { DefendInsightType } from '@kbn/elastic-assistant-common';

import type { HostMetadata } from '../../../../common/endpoint/types';
import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights';
import type {
SearchParams,
SecurityWorkflowInsight,
} from '../../../../common/endpoint/types/workflow_insights';

import {
ActionType,
Category,
SourceType,
TargetType,
} from '../../../../common/endpoint/types/workflow_insights';
import type { EndpointMetadataService } from '../metadata';
import {
buildEsQueryParams,
createDatastream,
createPipeline,
generateInsightId,
groupEndpointIdsByOS,
} from './helpers';
import {
Expand All @@ -43,6 +52,57 @@ jest.mock('@kbn/data-stream-adapter', () => ({
})),
}));

function getDefaultInsight(overrides?: Partial<SecurityWorkflowInsight>): SecurityWorkflowInsight {
const defaultInsight = {
'@timestamp': moment(),
message: 'This is a test message',
category: Category.Endpoint,
type: DefendInsightType.Enum.incompatible_antivirus,
source: {
type: SourceType.LlmConnector,
id: 'openai-connector-id',
data_range_start: moment(),
data_range_end: moment(),
},
target: {
type: TargetType.Endpoint,
ids: ['endpoint-1', 'endpoint-2'],
},
action: {
type: ActionType.Refreshed,
timestamp: moment(),
},
value: 'unique-key',
remediation: {
exception_list_items: [
{
list_id: 'example-list-id',
name: 'Example List Name',
description: 'Example description',
entries: [
{
field: 'example-field',
operator: 'included',
type: 'match',
value: 'example-value',
},
],
tags: ['example-tag'],
os_types: ['windows', 'linux'],
},
],
},
metadata: {
notes: {
key1: 'value1',
key2: 'value2',
},
message_variables: ['variable1', 'variable2'],
},
};
return merge(defaultInsight, overrides);
}

describe('helpers', () => {
describe('createDatastream', () => {
it('should create a DataStreamSpacesAdapter with the correct configuration', () => {
Expand Down Expand Up @@ -193,4 +253,13 @@ describe('helpers', () => {
});
});
});

describe('generateInsightId', () => {
it('should generate the correct hashed id', () => {
const insight = getDefaultInsight();
const result = generateInsightId(insight);
const expectedHash = '6b1a7a9625decbf899db4fbf78105a0eff9ef98e3f2dadc2781d59996b55445e';
expect(result).toBe(expectedHash);
});
});
});
21 changes: 20 additions & 1 deletion x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@
* 2.0.
*/

import { createHash } from 'crypto';
import { get as _get } from 'lodash';

import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import type { ElasticsearchClient } from '@kbn/core/server';

import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';

import type { SearchParams } from '../../../../common/endpoint/types/workflow_insights';
import type {
SearchParams,
SecurityWorkflowInsight,
} from '../../../../common/endpoint/types/workflow_insights';
import type { SupportedHostOsType } from '../../../../common/endpoint/constants';

import type { EndpointMetadataService } from '../metadata';
Expand Down Expand Up @@ -130,3 +134,18 @@ export async function groupEndpointIdsByOS(
return acc;
}, {});
}

export function generateInsightId(insight: SecurityWorkflowInsight): string {
const { type, category, value, target } = insight;
const targetType = target.type;
const targetIds = target.ids.join(',');

const hash = createHash('sha256');
hash.update(type);
hash.update(category);
hash.update(value);
hash.update(targetType);
hash.update(targetIds);

return hash.digest('hex');
}
67 changes: 44 additions & 23 deletions x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
* 2.0.
*/

import { merge } from 'lodash';
import { merge, cloneDeep } from 'lodash';
import moment from 'moment';
import { ReplaySubject } from 'rxjs';

import type { ElasticsearchClient, KibanaRequest, Logger } from '@kbn/core/server';
import type { DefendInsight, DefendInsightsPostRequestBody } from '@kbn/elastic-assistant-common';
import type { SearchHit, UpdateResponse } from '@elastic/elasticsearch/lib/api/types';

import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter';
import { DefendInsightType } from '@kbn/elastic-assistant-common';
Expand All @@ -31,7 +32,7 @@ import {
ActionType,
} from '../../../../common/endpoint/types/workflow_insights';
import { createMockEndpointAppContext } from '../../mocks';
import { createDatastream, createPipeline } from './helpers';
import { createDatastream, createPipeline, generateInsightId } from './helpers';
import { securityWorkflowInsightsService } from '.';
import { DATA_STREAM_NAME } from './constants';
import { buildWorkflowInsights } from './builders';
Expand Down Expand Up @@ -108,13 +109,18 @@ describe('SecurityWorkflowInsightsService', () => {
let logger: Logger;
let esClient: ElasticsearchClient;
let mockEndpointAppContextService: jest.Mocked<EndpointAppContextService>;
let isInitializedSpy: jest.SpyInstance<Promise<[void, void]>, [], boolean>;

beforeEach(() => {
logger = loggerMock.create();
esClient = elasticsearchServiceMock.createElasticsearchClient();

mockEndpointAppContextService = createMockEndpointAppContext()
.service as jest.Mocked<EndpointAppContextService>;

isInitializedSpy = jest
.spyOn(securityWorkflowInsightsService, 'isInitialized', 'get')
.mockResolvedValueOnce([undefined, undefined]);
});

afterEach(() => {
Expand Down Expand Up @@ -217,10 +223,6 @@ describe('SecurityWorkflowInsightsService', () => {

describe('createFromDefendInsights', () => {
it('should create workflow insights from defend insights', async () => {
const isInitializedSpy = jest
.spyOn(securityWorkflowInsightsService, 'isInitialized', 'get')
.mockResolvedValueOnce([undefined, undefined]);

const defendInsights: DefendInsight[] = [
{
group: 'AVGAntivirus',
Expand Down Expand Up @@ -258,8 +260,8 @@ describe('SecurityWorkflowInsightsService', () => {
request
);

// twice since it calls securityWorkflowInsightsService.create internally
expect(isInitializedSpy).toHaveBeenCalledTimes(2);
// three since it calls securityWorkflowInsightsService.create + fetch
expect(isInitializedSpy).toHaveBeenCalledTimes(3);
expect(buildWorkflowInsightsMock).toHaveBeenCalledWith({
defendInsights,
request,
Expand All @@ -271,32 +273,55 @@ describe('SecurityWorkflowInsightsService', () => {

describe('create', () => {
it('should index the doc correctly', async () => {
const isInitializedSpy = jest
.spyOn(securityWorkflowInsightsService, 'isInitialized', 'get')
.mockResolvedValueOnce([undefined, undefined]);

await securityWorkflowInsightsService.start({ esClient });
const insight = getDefaultInsight();
await securityWorkflowInsightsService.create(insight);

// ensure it waits for initialization first
expect(isInitializedSpy).toHaveBeenCalledTimes(1);
// two since it calls fetch as well
expect(isInitializedSpy).toHaveBeenCalledTimes(2);
// indexes the doc
expect(esClient.index).toHaveBeenCalledTimes(1);
expect(esClient.index).toHaveBeenCalledWith({
index: DATA_STREAM_NAME,
body: insight,
body: { ...insight, id: generateInsightId(insight) },
refresh: 'wait_for',
});
});

it('should call update instead if insight already exists', async () => {
const indexName = 'backing-index';
const fetchSpy = jest
.spyOn(securityWorkflowInsightsService, 'fetch')
.mockResolvedValueOnce([{ _index: indexName }] as Array<
SearchHit<SecurityWorkflowInsight>
>);
const updateSpy = jest
.spyOn(securityWorkflowInsightsService, 'update')
.mockResolvedValueOnce({} as UpdateResponse);
await securityWorkflowInsightsService.start({ esClient });
const insight = getDefaultInsight();
await securityWorkflowInsightsService.create(insight);

const expectedInsight = cloneDeep(insight);
expectedInsight['@timestamp'] = expect.any(moment);
expectedInsight.action.timestamp = expect.any(moment);
expectedInsight.source.data_range_start = expect.any(moment);
expectedInsight.source.data_range_end = expect.any(moment);

// ensure it waits for initialization first
expect(isInitializedSpy).toHaveBeenCalledTimes(1);
expect(fetchSpy).toHaveBeenCalledTimes(1);
expect(updateSpy).toHaveBeenCalledTimes(1);
expect(updateSpy).toHaveBeenCalledWith(
generateInsightId(insight),
expect.objectContaining(insight),
indexName
);
});
});

describe('update', () => {
it('should update the doc correctly', async () => {
const isInitializedSpy = jest
.spyOn(securityWorkflowInsightsService, 'isInitialized', 'get')
.mockResolvedValueOnce([undefined, undefined]);

await securityWorkflowInsightsService.start({ esClient });
const insightId = 'some-insight-id';
const insight = getDefaultInsight();
Expand All @@ -318,10 +343,6 @@ describe('SecurityWorkflowInsightsService', () => {

describe('fetch', () => {
it('should fetch the docs with the correct params', async () => {
const isInitializedSpy = jest
.spyOn(securityWorkflowInsightsService, 'isInitialized', 'get')
.mockResolvedValueOnce([undefined, undefined]);

await securityWorkflowInsightsService.start({ esClient });
const searchParams: SearchParams = {
size: 50,
Expand Down
12 changes: 10 additions & 2 deletions x-pack/plugins/security_solution/server/endpoint/services/workflow_insights/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import type {
import type { EndpointAppContextService } from '../../endpoint_app_context_services';

import { SecurityWorkflowInsightsFailedInitialized } from './errors';
import { buildEsQueryParams, createDatastream, createPipeline } from './helpers';
import { buildEsQueryParams, createDatastream, createPipeline, generateInsightId } from './helpers';
import { DATA_STREAM_NAME } from './constants';
import { buildWorkflowInsights } from './builders';

Expand Down Expand Up @@ -135,9 +135,17 @@ class SecurityWorkflowInsightsService {
public async create(insight: SecurityWorkflowInsight): Promise<WriteResponseBase> {
await this.isInitialized;

const id = generateInsightId(insight);

// if insight already exists, update instead
const existingInsights = await this.fetch({ ids: [id] });
if (existingInsights.length) {
return this.update(id, insight, existingInsights[0]._index);
}

const response = await this.esClient.index<SecurityWorkflowInsight>({
index: DATA_STREAM_NAME,
body: insight,
body: { ...insight, id },
refresh: 'wait_for',
});

Expand Down

0 comments on commit 31fa4a2

Please sign in to comment.