From 489c0901ffd335879d9652424ab15ef9f39cc4cb Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Mon, 14 Oct 2024 22:56:58 +0200 Subject: [PATCH] [SecuritySolution] Load entity store indices from security solution data view (#195862) ## Summary * Update the Entity Store to retrieve indices from the security solution data view. * Create a new API that updates all installed entity engine indices (`api/entity_store/engines/apply_dataview_indices`) ### How to test it? * Install the entity store * Check if the transform index has the security solutions data view indices * Call `apply_dataview_indices` API; it should not return changes * Update the security solution data view indices * Call `apply_dataview_indices` API and if the API response contains the updated indices * Check if the transform index also got updated --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../output/kibana.serverless.staging.yaml | 65 +++++++++++ oas_docs/output/kibana.serverless.yaml | 65 +++++++++++ oas_docs/output/kibana.staging.yaml | 65 +++++++++++ oas_docs/output/kibana.yaml | 65 +++++++++++ .../entity_definition_update_conflict.ts | 13 +++ .../server/lib/entity_client.ts | 56 +++++++++- .../entity_store/common.gen.ts | 2 +- .../entity_store/common.schema.yaml | 1 + .../engine/apply_dataview_indices.gen.ts | 35 ++++++ .../engine/apply_dataview_indices.schema.yaml | 71 ++++++++++++ .../entity_store/engine/index.ts | 1 + .../common/api/quickstart_client.gen.ts | 13 +++ ...alytics_api_2023_10_31.bundled.schema.yaml | 63 +++++++++++ ...alytics_api_2023_10_31.bundled.schema.yaml | 63 +++++++++++ .../entity_store/constants.ts | 3 +- .../entity_store_data_client.test.ts | 4 + .../entity_store/entity_store_data_client.ts | 102 +++++++++++++++++- .../routes/apply_dataview_indices.ts | 85 +++++++++++++++ .../routes/register_entity_store_routes.ts | 2 + .../get_united_definition.test.ts | 27 +---- .../get_united_definition.ts | 9 +- .../united_entity_definition.ts | 19 ++-- .../entity_store/utils/entity_utils.ts | 50 +++++++++ .../server/request_context_factory.ts | 11 +- .../services/security_solution_api.gen.ts | 7 ++ .../trial_license_complete_tier/engine.ts | 67 ++++++++++-- .../entity_analytics/utils/data_view.ts | 44 ++++++++ .../entity_analytics/utils/entity_store.ts | 2 +- 28 files changed, 954 insertions(+), 56 deletions(-) create mode 100644 x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts create mode 100644 x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts create mode 100644 x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts diff --git a/oas_docs/output/kibana.serverless.staging.yaml b/oas_docs/output/kibana.serverless.staging.yaml index aad5256524f4a..9e63182949f25 100644 --- a/oas_docs/output/kibana.serverless.staging.yaml +++ b/oas_docs/output/kibana.serverless.staging.yaml @@ -8421,6 +8421,56 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -47909,6 +47959,20 @@ components: #/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel required: - criticality_level + Security_Entity_Analytics_API_EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type Security_Entity_Analytics_API_EngineDescriptor: type: object properties: @@ -47932,6 +47996,7 @@ components: - installing - started - stopped + - updating type: string Security_Entity_Analytics_API_Entity: oneOf: diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index aad5256524f4a..9e63182949f25 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -8421,6 +8421,56 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -47909,6 +47959,20 @@ components: #/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel required: - criticality_level + Security_Entity_Analytics_API_EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type Security_Entity_Analytics_API_EngineDescriptor: type: object properties: @@ -47932,6 +47996,7 @@ components: - installing - started - stopped + - updating type: string Security_Entity_Analytics_API_Entity: oneOf: diff --git a/oas_docs/output/kibana.staging.yaml b/oas_docs/output/kibana.staging.yaml index 740f52664dfe6..f32de75a62b26 100644 --- a/oas_docs/output/kibana.staging.yaml +++ b/oas_docs/output/kibana.staging.yaml @@ -11850,6 +11850,56 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -56675,6 +56725,20 @@ components: #/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel required: - criticality_level + Security_Entity_Analytics_API_EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type Security_Entity_Analytics_API_EngineDescriptor: type: object properties: @@ -56698,6 +56762,7 @@ components: - installing - started - stopped + - updating type: string Security_Entity_Analytics_API_Entity: oneOf: diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index 740f52664dfe6..f32de75a62b26 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -11850,6 +11850,56 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: >- + #/components/schemas/Security_Entity_Analytics_API_EngineDataviewUpdateResult + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json; Elastic-Api-Version=2023-10-31: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -56675,6 +56725,20 @@ components: #/components/schemas/Security_Entity_Analytics_API_AssetCriticalityLevel required: - criticality_level + Security_Entity_Analytics_API_EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type Security_Entity_Analytics_API_EngineDescriptor: type: object properties: @@ -56698,6 +56762,7 @@ components: - installing - started - stopped + - updating type: string Security_Entity_Analytics_API_Entity: oneOf: diff --git a/x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts b/x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts new file mode 100644 index 0000000000000..1e85d91a7bb01 --- /dev/null +++ b/x-pack/plugins/entity_manager/server/lib/entities/errors/entity_definition_update_conflict.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class EntityDefinitionUpdateConflict extends Error { + constructor(message: string) { + super(message); + this.name = 'EntityDefinitionUpdateConflict'; + } +} diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index 710872c04eda0..1bb1322be356f 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -5,17 +5,23 @@ * 2.0. */ -import { EntityDefinition } from '@kbn/entities-schema'; +import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; -import { installEntityDefinition } from './entities/install_entity_definition'; +import { + installEntityDefinition, + installationInProgress, + reinstallEntityDefinition, +} from './entities/install_entity_definition'; import { startTransforms } from './entities/start_transforms'; -import { findEntityDefinitions } from './entities/find_entity_definition'; +import { findEntityDefinitionById, findEntityDefinitions } from './entities/find_entity_definition'; import { uninstallEntityDefinition } from './entities/uninstall_entity_definition'; import { EntityDefinitionNotFound } from './entities/errors/entity_not_found'; import { stopTransforms } from './entities/stop_transforms'; +import { EntityDefinitionWithState } from './entities/types'; +import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict'; export class EntityClient { constructor( @@ -47,6 +53,50 @@ export class EntityClient { return installedDefinition; } + async updateEntityDefinition({ + id, + definitionUpdate, + }: { + id: string; + definitionUpdate: EntityDefinitionUpdate; + }) { + const definition = await findEntityDefinitionById({ + id, + soClient: this.options.soClient, + esClient: this.options.esClient, + includeState: true, + }); + + if (!definition) { + const message = `Unable to find entity definition with [${id}]`; + this.options.logger.error(message); + throw new EntityDefinitionNotFound(message); + } + + if (installationInProgress(definition)) { + const message = `Entity definition [${definition.id}] has changes in progress`; + this.options.logger.error(message); + throw new EntityDefinitionUpdateConflict(message); + } + + const shouldRestartTransforms = ( + definition as EntityDefinitionWithState + ).state.components.transforms.some((transform) => transform.running); + + const updatedDefinition = await reinstallEntityDefinition({ + definition, + definitionUpdate, + soClient: this.options.soClient, + esClient: this.options.esClient, + logger: this.options.logger, + }); + + if (shouldRestartTransforms) { + await startTransforms(this.options.esClient, updatedDefinition, this.options.logger); + } + return updatedDefinition; + } + async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) { const [definition] = await findEntityDefinitions({ id, diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts index 8152b93e44ea4..ed0806b798dd6 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts @@ -25,7 +25,7 @@ export type IndexPattern = z.infer; export const IndexPattern = z.string(); export type EngineStatus = z.infer; -export const EngineStatus = z.enum(['installing', 'started', 'stopped']); +export const EngineStatus = z.enum(['installing', 'started', 'stopped', 'updating']); export type EngineStatusEnum = typeof EngineStatus.enum; export const EngineStatusEnum = EngineStatus.enum; diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml index c72d7ae77ffbc..b06f484e4e29a 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml @@ -37,6 +37,7 @@ components: - installing - started - stopped + - updating IndexPattern: type: string diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts new file mode 100644 index 0000000000000..c3492ad88f754 --- /dev/null +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* + * NOTICE: Do not edit this file manually. + * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator. + * + * info: + * title: Apply DataView indices to all installed engines + * version: 2023-10-31 + */ + +import { z } from '@kbn/zod'; + +export type EngineDataviewUpdateResult = z.infer; +export const EngineDataviewUpdateResult = z.object({ + type: z.string(), + changes: z + .object({ + indexPatterns: z.array(z.string()).optional(), + }) + .optional(), +}); + +export type ApplyEntityEngineDataviewIndicesResponse = z.infer< + typeof ApplyEntityEngineDataviewIndicesResponse +>; +export const ApplyEntityEngineDataviewIndicesResponse = z.object({ + success: z.boolean().optional(), + result: z.array(EngineDataviewUpdateResult).optional(), +}); diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml new file mode 100644 index 0000000000000..20afc96cd54e3 --- /dev/null +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/apply_dataview_indices.schema.yaml @@ -0,0 +1,71 @@ +openapi: 3.0.0 + +info: + title: Apply DataView indices to all installed engines + version: '2023-10-31' +paths: + /api/entity_store/engines/apply_dataview_indices: + post: + x-labels: [ess, serverless] + x-codegen-enabled: true + operationId: ApplyEntityEngineDataviewIndices + summary: Apply DataView indices to all installed engines + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + result: + type: array + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + + '207': + description: Partial successful response + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + result: + type: array + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + errors: + type: array + items: + type: string + '500': + description: Error response + content: + application/json: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number +components: + schemas: + EngineDataviewUpdateResult: + type: object + properties: + type: + type: string + changes: + type: object + properties: + indexPatterns: + type: array + items: + type: string + required: + - type diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts index af84f273b3153..b21308de36f18 100644 --- a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts +++ b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/engine/index.ts @@ -12,3 +12,4 @@ export * from './list.gen'; export * from './start.gen'; export * from './stats.gen'; export * from './stop.gen'; +export * from './apply_dataview_indices.gen'; diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 288a08fdb8afb..19fbc38072c14 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -243,6 +243,7 @@ import type { InternalUploadAssetCriticalityRecordsResponse, UploadAssetCriticalityRecordsResponse, } from './entity_analytics/asset_criticality/upload_asset_criticality_csv.gen'; +import type { ApplyEntityEngineDataviewIndicesResponse } from './entity_analytics/entity_store/engine/apply_dataview_indices.gen'; import type { DeleteEntityEngineRequestQueryInput, DeleteEntityEngineRequestParamsInput, @@ -397,6 +398,18 @@ after 30 days. It also deletes other artifacts specific to the migration impleme }) .catch(catchAxiosErrorFormatAndThrow); } + async applyEntityEngineDataviewIndices() { + this.log.info(`${new Date().toISOString()} Calling API ApplyEntityEngineDataviewIndices`); + return this.kbnClient + .request({ + path: '/api/entity_store/engines/apply_dataview_indices', + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31', + }, + method: 'POST', + }) + .catch(catchAxiosErrorFormatAndThrow); + } async assetCriticalityGetPrivileges() { this.log.info(`${new Date().toISOString()} Calling API AssetCriticalityGetPrivileges`); return this.kbnClient diff --git a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index 99b96aed4052a..730ea240fe7b7 100644 --- a/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -452,6 +452,54 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json: + schema: + type: object + properties: + result: + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -720,6 +768,20 @@ components: $ref: '#/components/schemas/AssetCriticalityLevel' required: - criticality_level + EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type EngineDescriptor: type: object properties: @@ -743,6 +805,7 @@ components: - installing - started - stopped + - updating type: string Entity: oneOf: diff --git a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml index 22915fbc3be7b..2522f3cb192ae 100644 --- a/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml @@ -452,6 +452,54 @@ paths: summary: Stop an Entity Engine tags: - Security Entity Analytics API + /api/entity_store/engines/apply_dataview_indices: + post: + operationId: ApplyEntityEngineDataviewIndices + responses: + '200': + content: + application/json: + schema: + type: object + properties: + result: + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + type: array + success: + type: boolean + description: Successful response + '207': + content: + application/json: + schema: + type: object + properties: + errors: + items: + type: string + type: array + result: + items: + $ref: '#/components/schemas/EngineDataviewUpdateResult' + type: array + success: + type: boolean + description: Partial successful response + '500': + content: + application/json: + schema: + type: object + properties: + body: + type: string + statusCode: + type: number + description: Error response + summary: Apply DataView indices to all installed engines + tags: + - Security Entity Analytics API /api/entity_store/entities/list: get: description: List entities records, paging, sorting and filtering as needed. @@ -720,6 +768,20 @@ components: $ref: '#/components/schemas/AssetCriticalityLevel' required: - criticality_level + EngineDataviewUpdateResult: + type: object + properties: + changes: + type: object + properties: + indexPatterns: + items: + type: string + type: array + type: + type: string + required: + - type EngineDescriptor: type: object properties: @@ -743,6 +805,7 @@ components: - installing - started - stopped + - updating type: string Entity: oneOf: diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts index 4e262b79fffcc..db0f48877a73c 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/constants.ts @@ -6,13 +6,11 @@ */ import type { EngineStatus } from '../../../../common/api/entity_analytics/entity_store/common.gen'; -import { DEFAULT_INDEX_PATTERN } from '../../../../common/constants'; /** * Default index pattern for entity store * This is the same as the default index pattern for the SIEM app but might diverge in the future */ -export const ENTITY_STORE_DEFAULT_SOURCE_INDICES = DEFAULT_INDEX_PATTERN; export const DEFAULT_LOOKBACK_PERIOD = '24h'; @@ -22,6 +20,7 @@ export const ENGINE_STATUS: Record, EngineStatus> = { INSTALLING: 'installing', STARTED: 'started', STOPPED: 'stopped', + UPDATING: 'updating', }; export const MAX_SEARCH_RESPONSE_SIZE = 10_000; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index d6a012b539019..4156ea1dbd4ea 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -13,6 +13,8 @@ import { import { EntityStoreDataClient } from './entity_store_data_client'; import type { SortOrder } from '@elastic/elasticsearch/lib/api/types'; import type { EntityType } from '../../../../common/api/entity_analytics/entity_store/common.gen'; +import type { DataViewsService } from '@kbn/data-views-plugin/common'; +import type { AppClient } from '../../..'; describe('EntityStoreDataClient', () => { const mockSavedObjectClient = savedObjectsClientMock.create(); @@ -24,6 +26,8 @@ describe('EntityStoreDataClient', () => { namespace: 'default', soClient: mockSavedObjectClient, kibanaVersion: '9.0.0', + dataViewsService: {} as DataViewsService, + appClient: {} as AppClient, }); const defaultSearchParams = { diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 73a3ad113df38..a71be61781e00 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -14,6 +14,10 @@ import type { import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import type { SortOrder } from '@elastic/elasticsearch/lib/api/types'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; +import type { DataViewsService } from '@kbn/data-views-plugin/common'; +import { isEqual } from 'lodash/fp'; +import type { EngineDataviewUpdateResult } from '../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen'; +import type { AppClient } from '../../..'; import type { Entity } from '../../../../common/api/entity_analytics/entity_store/entities/common.gen'; import type { InitEntityEngineRequestBody, @@ -24,7 +28,6 @@ import type { InspectQuery, } from '../../../../common/api/entity_analytics/entity_store/common.gen'; import { EngineDescriptorClient } from './saved_object/engine_descriptor'; -import { buildEntityDefinitionId, getEntitiesIndexName } from './utils'; import { ENGINE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants'; import { AssetCriticalityEcsMigrationClient } from '../asset_criticality/asset_criticality_migration_client'; import { getUnitedEntityDefinition } from './united_entity_definitions'; @@ -44,6 +47,13 @@ import { deleteFieldRetentionEnrichPolicy, } from './elasticsearch_assets'; import { RiskScoreDataClient } from '../risk_score/risk_score_data_client'; +import { + buildEntityDefinitionId, + buildIndexPatterns, + getEntitiesIndexName, + isPromiseFulfilled, + isPromiseRejected, +} from './utils'; interface EntityStoreClientOpts { logger: Logger; @@ -53,6 +63,8 @@ interface EntityStoreClientOpts { taskManager?: TaskManagerStartContract; auditLogger?: AuditLogger; kibanaVersion: string; + dataViewsService: DataViewsService; + appClient: AppClient; } interface SearchEntitiesParams { @@ -108,7 +120,7 @@ export class EntityStoreDataClient { throw new Error('Task Manager is not available'); } - const { logger, esClient, namespace, taskManager } = this.options; + const { logger, esClient, namespace, taskManager, appClient, dataViewsService } = this.options; await this.riskScoreDataClient.createRiskScoreLatestIndex(); @@ -132,9 +144,11 @@ export class EntityStoreDataClient { indexPattern, }); logger.debug(`Initialized engine for ${entityType}`); + const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); // first create the entity definition without starting it // so that the index template is created which we can add a component template to const unitedDefinition = getUnitedEntityDefinition({ + indexPatterns, entityType, namespace, fieldHistoryLength, @@ -221,7 +235,6 @@ export class EntityStoreDataClient { public async start(entityType: EntityType, options?: { force: boolean }) { const descriptor = await this.engineClient.get(entityType); - if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) { throw new Error( `In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}` @@ -273,9 +286,11 @@ export class EntityStoreDataClient { taskManager: TaskManagerStartContract, deleteData: boolean ) { - const { namespace, logger, esClient } = this.options; + const { namespace, logger, esClient, appClient, dataViewsService } = this.options; const descriptor = await this.engineClient.maybeGet(entityType); + const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); const unitedDefinition = getUnitedEntityDefinition({ + indexPatterns, entityType, namespace: this.options.namespace, fieldHistoryLength: descriptor?.fieldHistoryLength ?? 10, @@ -368,4 +383,83 @@ export class EntityStoreDataClient { return { records, total, inspect }; } + + public async applyDataViewIndices(): Promise<{ + successes: EngineDataviewUpdateResult[]; + errors: Error[]; + }> { + const { logger } = this.options; + logger.info( + `In namespace ${this.options.namespace}: Applying data view indices to the entity store` + ); + + const { engines } = await this.engineClient.list(); + + const updateDefinitionPromises: Array> = await engines.map( + async (engine) => { + const originalStatus = engine.status; + const id = buildEntityDefinitionId(engine.type, this.options.namespace); + const definition = await this.getExistingEntityDefinition(engine.type); + + if ( + originalStatus === ENGINE_STATUS.INSTALLING || + originalStatus === ENGINE_STATUS.UPDATING + ) { + throw new Error( + `Error updating entity store: There is an changes already in progress for engine ${id}` + ); + } + + const indexPatterns = await buildIndexPatterns( + this.options.namespace, + this.options.appClient, + this.options.dataViewsService + ); + + // Skip update if index patterns are the same + if (isEqual(definition.indexPatterns, indexPatterns)) { + return { type: engine.type, changes: {} }; + } + + // Update savedObject status + await this.engineClient.update(engine.type, ENGINE_STATUS.UPDATING); + + try { + // Update entity manager definition + await this.entityClient.updateEntityDefinition({ + id, + definitionUpdate: { + ...definition, + indexPatterns, + }, + }); + + // Restore the savedObject status and set the new index pattern + await this.engineClient.update(engine.type, originalStatus); + + return { type: engine.type, changes: { indexPatterns } }; + } catch (error) { + // Rollback the engine initial status when the update fails + await this.engineClient.update(engine.type, originalStatus); + + throw error; + } + } + ); + + const updatedDefinitions = await Promise.allSettled(updateDefinitionPromises); + + const updateErrors = updatedDefinitions + .filter(isPromiseRejected) + .map((result) => result.reason); + + const updateSuccesses = updatedDefinitions + .filter(isPromiseFulfilled) + .map((result) => result.value); + + return { + successes: updateSuccesses, + errors: updateErrors, + }; + } } diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts new file mode 100644 index 0000000000000..72cd02f273cad --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/apply_dataview_indices.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildSiemResponse } from '@kbn/lists-plugin/server/routes/utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import type { ApplyEntityEngineDataviewIndicesResponse } from '../../../../../common/api/entity_analytics/entity_store/engine/apply_dataview_indices.gen'; +import { API_VERSIONS, APP_ID } from '../../../../../common/constants'; +import type { EntityAnalyticsRoutesDeps } from '../../types'; + +export const applyDataViewIndicesEntityEngineRoute = ( + router: EntityAnalyticsRoutesDeps['router'], + logger: Logger +) => { + router.versioned + .post({ + access: 'public', + path: '/api/entity_store/engines/apply_dataview_indices', + options: { + tags: ['access:securitySolution', `access:${APP_ID}-entity-analytics`], + }, + }) + .addVersion( + { + version: API_VERSIONS.public.v1, + validate: { + request: {}, + }, + }, + + async ( + context, + _, + response + ): Promise> => { + const siemResponse = buildSiemResponse(response); + + try { + const secSol = await context.securitySolution; + const { errors, successes } = await secSol + .getEntityStoreDataClient() + .applyDataViewIndices(); + + const errorMessages = errors.map((e) => e.message); + + if (successes.length === 0 && errors.length > 0) { + return siemResponse.error({ + statusCode: 500, + body: `Error in ApplyEntityEngineDataViewIndices. Errors: [${errorMessages.join( + ', ' + )}]`, + }); + } + + if (errors.length === 0) { + return response.ok({ + body: { + success: true, + result: successes, + }, + }); + } else { + return response.multiStatus({ + body: { + success: false, + errors: errorMessages, + result: successes, + }, + }); + } + } catch (e) { + logger.error('Error in ApplyEntityEngineDataViewIndices:', e); + const error = transformError(e); + return siemResponse.error({ + statusCode: error.statusCode, + body: error.message, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts index fc61ee1f72b11..20b6d92d8f0ff 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/routes/register_entity_store_routes.ts @@ -6,6 +6,7 @@ */ import type { EntityAnalyticsRoutesDeps } from '../../types'; +import { applyDataViewIndicesEntityEngineRoute } from './apply_dataview_indices'; import { deleteEntityEngineRoute } from './delete'; import { listEntitiesRoute } from './entities/list'; import { getEntityEngineRoute } from './get'; @@ -27,4 +28,5 @@ export const registerEntityStoreRoutes = ({ getEntityEngineRoute(router, logger); listEntityEnginesRoute(router, logger); listEntitiesRoute(router, logger); + applyDataViewIndicesEntityEngineRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts index c3a4ee547df28..2657917d45a78 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.test.ts @@ -8,11 +8,13 @@ import { getUnitedEntityDefinition } from './get_united_definition'; describe('getUnitedEntityDefinition', () => { + const indexPatterns = ['test*']; describe('host', () => { const unitedDefinition = getUnitedEntityDefinition({ entityType: 'host', namespace: 'test', fieldHistoryLength: 10, + indexPatterns, }); it('mapping', () => { @@ -151,17 +153,7 @@ describe('getUnitedEntityDefinition', () => { }, ], "indexPatterns": Array [ - "apm-*-transaction*", - "auditbeat-*", - "endgame-*", - "filebeat-*", - "logs-*", - "packetbeat-*", - "traces-apm*", - "winlogbeat-*", - "-*elastic-cloud-logs-*", - ".asset-criticality.asset-criticality-test", - "risk-score.risk-score-latest-test", + "test*", ], "latest": Object { "lookbackPeriod": "24h", @@ -286,6 +278,7 @@ describe('getUnitedEntityDefinition', () => { entityType: 'user', namespace: 'test', fieldHistoryLength: 10, + indexPatterns, }); it('mapping', () => { @@ -416,17 +409,7 @@ describe('getUnitedEntityDefinition', () => { }, ], "indexPatterns": Array [ - "apm-*-transaction*", - "auditbeat-*", - "endgame-*", - "filebeat-*", - "logs-*", - "packetbeat-*", - "traces-apm*", - "winlogbeat-*", - "-*elastic-cloud-logs-*", - ".asset-criticality.asset-criticality-test", - "risk-score.risk-score-latest-test", + "test*", ], "latest": Object { "lookbackPeriod": "24h", diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts index 21214f1bf95fb..6699e160634fd 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/get_united_definition.ts @@ -24,10 +24,16 @@ interface Options { entityType: EntityType; namespace: string; fieldHistoryLength: number; + indexPatterns: string[]; } export const getUnitedEntityDefinition = memoize( - ({ entityType, namespace, fieldHistoryLength }: Options): UnitedEntityDefinition => { + ({ + entityType, + namespace, + fieldHistoryLength, + indexPatterns, + }: Options): UnitedEntityDefinition => { const unitedDefinition = unitedDefinitionBuilders[entityType](fieldHistoryLength); unitedDefinition.fields.push( @@ -40,6 +46,7 @@ export const getUnitedEntityDefinition = memoize( return new UnitedEntityDefinition({ ...unitedDefinition, namespace, + indexPatterns, }); }, ({ entityType, namespace, fieldHistoryLength }: Options) => diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts index e70929101a0b5..c5315c5dca2b0 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts @@ -7,13 +7,7 @@ import { entityDefinitionSchema, type EntityDefinition } from '@kbn/entities-schema'; import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen'; -import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine'; -import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality'; -import { - DEFAULT_INTERVAL, - DEFAULT_LOOKBACK_PERIOD, - ENTITY_STORE_DEFAULT_SOURCE_INDICES, -} from '../constants'; +import { DEFAULT_INTERVAL, DEFAULT_LOOKBACK_PERIOD } from '../constants'; import { buildEntityDefinitionId, getIdentityFieldForEntityType } from '../utils'; import type { FieldRetentionDefinition, @@ -25,6 +19,7 @@ import { BASE_ENTITY_INDEX_MAPPING } from './constants'; export class UnitedEntityDefinition { version: string; entityType: EntityType; + indexPatterns: string[]; fields: UnitedDefinitionField[]; namespace: string; entityManagerDefinition: EntityDefinition; @@ -34,11 +29,13 @@ export class UnitedEntityDefinition { constructor(opts: { version: string; entityType: EntityType; + indexPatterns: string[]; fields: UnitedDefinitionField[]; namespace: string; }) { this.version = opts.version; this.entityType = opts.entityType; + this.indexPatterns = opts.indexPatterns; this.fields = opts.fields; this.namespace = opts.namespace; this.entityManagerDefinition = this.toEntityManagerDefinition(); @@ -47,7 +44,7 @@ export class UnitedEntityDefinition { } private toEntityManagerDefinition(): EntityDefinition { - const { entityType, namespace } = this; + const { entityType, namespace, indexPatterns } = this; const identityField = getIdentityFieldForEntityType(this.entityType); const metadata = this.fields .filter((field) => field.definition) @@ -57,11 +54,7 @@ export class UnitedEntityDefinition { id: buildEntityDefinitionId(entityType, namespace), name: `Security '${entityType}' Entity Store Definition`, type: entityType, - indexPatterns: [ - ...ENTITY_STORE_DEFAULT_SOURCE_INDICES, - getAssetCriticalityIndex(namespace), - getRiskScoreLatestIndex(namespace), - ], + indexPatterns, identityFields: [identityField], displayNameTemplate: `{{${identityField}}}`, metadata, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts index 5274db63f8a02..8fe21317f4ad8 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/utils/entity_utils.ts @@ -10,6 +10,10 @@ import { ENTITY_SCHEMA_VERSION_V1, entitiesIndexPattern, } from '@kbn/entities-schema'; +import type { DataViewsService, DataView } from '@kbn/data-views-plugin/common'; +import type { AppClient } from '../../../../types'; +import { getRiskScoreLatestIndex } from '../../../../../common/entity_analytics/risk_engine'; +import { getAssetCriticalityIndex } from '../../../../../common/entity_analytics/asset_criticality'; import type { EntityType } from '../../../../../common/api/entity_analytics/entity_store/common.gen'; import { entityEngineDescriptorTypeName } from '../saved_object'; @@ -19,6 +23,44 @@ export const getIdentityFieldForEntityType = (entityType: EntityType) => { return 'user.name'; }; +export const buildIndexPatterns = async ( + space: string, + appClient: AppClient, + dataViewsService: DataViewsService +) => { + const { alertsIndex, securitySolutionDataViewIndices } = await getSecuritySolutionIndices( + appClient, + dataViewsService + ); + return [ + ...securitySolutionDataViewIndices.filter((item) => item !== alertsIndex), + getAssetCriticalityIndex(space), + getRiskScoreLatestIndex(space), + ]; +}; + +const getSecuritySolutionIndices = async ( + appClient: AppClient, + dataViewsService: DataViewsService +) => { + const securitySolutionDataViewId = appClient.getSourcererDataViewId(); + let dataView: DataView; + try { + dataView = await dataViewsService.get(securitySolutionDataViewId); + } catch (e) { + if (e.isBoom && e.output.statusCode === 404) { + throw new Error(`Data view not found '${securitySolutionDataViewId}'`); + } + throw e; + } + + const dataViewIndexPattern = dataView.getIndexPattern(); + return { + securitySolutionDataViewIndices: dataViewIndexPattern.split(','), + alertsIndex: appClient.getAlertsIndex(), + }; +}; + export const getByEntityTypeQuery = (entityType: EntityType) => { return `${entityEngineDescriptorTypeName}.attributes.type: ${entityType}`; }; @@ -33,3 +75,11 @@ export const getEntitiesIndexName = (entityType: EntityType, namespace: string) export const buildEntityDefinitionId = (entityType: EntityType, space: string) => { return `security_${entityType}_${space}`; }; + +export const isPromiseFulfilled = ( + result: PromiseSettledResult +): result is PromiseFulfilledResult => result.status === 'fulfilled'; + +export const isPromiseRejected = ( + result: PromiseSettledResult +): result is PromiseRejectedResult => result.status === 'rejected'; diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 6aa1389c137e6..0782fa25c71eb 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -74,6 +74,12 @@ export class RequestContextFactory implements IRequestContextFactory { const licensing = await context.licensing; const actionsClient = await startPlugins.actions.getActionsClientWithRequest(request); + const dataViewsService = await startPlugins.dataViews.dataViewsServiceFactory( + coreContext.savedObjects.client, + coreContext.elasticsearch.client.asInternalUser, + request + ); + const getSpaceId = (): string => startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID; @@ -84,6 +90,7 @@ export class RequestContextFactory implements IRequestContextFactory { kibanaBranch: options.kibanaBranch, buildFlavor: options.buildFlavor, }); + const getAppClient = () => appClientFactory.create(request); const getAuditLogger = () => security?.audit.asScoped(request); @@ -109,7 +116,7 @@ export class RequestContextFactory implements IRequestContextFactory { getFrameworkRequest: () => frameworkRequest, - getAppClient: () => appClientFactory.create(request), + getAppClient, getSpaceId, @@ -197,6 +204,8 @@ export class RequestContextFactory implements IRequestContextFactory { const soClient = coreContext.savedObjects.client; return new EntityStoreDataClient({ namespace: getSpaceId(), + dataViewsService, + appClient: getAppClient(), esClient, logger, soClient, diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index c110ce8676edb..7e1e532806a6c 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -155,6 +155,13 @@ after 30 days. It also deletes other artifacts specific to the migration impleme .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + applyEntityEngineDataviewIndices(kibanaSpace: string = 'default') { + return supertest + .post(routeWithNamespace('/api/entity_store/engines/apply_dataview_indices', kibanaSpace)) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, assetCriticalityGetPrivileges(kibanaSpace: string = 'default') { return supertest .get(routeWithNamespace('/internal/asset_criticality/privileges', kibanaSpace)) diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts index 5053882bcb971..d6963c28b2f73 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/engine.ts @@ -8,8 +8,10 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../ftr_provider_context'; import { EntityStoreUtils, elasticAssetCheckerFactory } from '../../utils'; +import { dataViewRouteHelpersFactory } from '../../utils/data_view'; export default ({ getService }: FtrProviderContext) => { const api = getService('securitySolutionApi'); + const supertest = getService('supertest'); const { expectTransformExists, expectTransformNotFound, @@ -24,8 +26,15 @@ export default ({ getService }: FtrProviderContext) => { const utils = EntityStoreUtils(getService); // TODO: unskip once permissions issue is resolved describe.skip('@ess @serverless @skipInServerlessMKI Entity Store Engine APIs', () => { + const dataView = dataViewRouteHelpersFactory(supertest); + before(async () => { await utils.cleanEngines(); + await dataView.create('security-solution'); + }); + + after(async () => { + await dataView.delete('security-solution'); }); describe('init', () => { @@ -75,9 +84,9 @@ export default ({ getService }: FtrProviderContext) => { expect(getResponse.body).to.eql({ status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', + indexPattern: '', filter: '', + fieldHistoryLength: 10, }); }); @@ -91,9 +100,9 @@ export default ({ getService }: FtrProviderContext) => { expect(getResponse.body).to.eql({ status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', + indexPattern: '', filter: '', + fieldHistoryLength: 10, }); }); }); @@ -109,16 +118,16 @@ export default ({ getService }: FtrProviderContext) => { { status: 'started', type: 'host', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', + indexPattern: '', filter: '', + fieldHistoryLength: 10, }, { status: 'started', type: 'user', - indexPattern: - 'apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-*', + indexPattern: '', filter: '', + fieldHistoryLength: 10, }, ]); }); @@ -200,5 +209,47 @@ export default ({ getService }: FtrProviderContext) => { await expectIngestPipelineNotFound(`ea_default_user_entity_store-latest@platform`); }); }); + + describe('apply_dataview_indices', () => { + before(async () => { + await utils.initEntityEngineForEntityType('host'); + }); + + after(async () => { + await utils.cleanEngines(); + }); + + afterEach(async () => { + await dataView.delete('security-solution'); + await dataView.create('security-solution'); + }); + + it("should not update the index patten when it didn't change", async () => { + const response = await api.applyEntityEngineDataviewIndices(); + + expect(response.body).to.eql({ success: true, result: [{ type: 'host', changes: {} }] }); + }); + + it('should update the index pattern when the data view changes', async () => { + await dataView.updateIndexPattern('security-solution', 'test-*'); + const response = await api.applyEntityEngineDataviewIndices(); + + expect(response.body).to.eql({ + success: true, + result: [ + { + type: 'host', + changes: { + indexPatterns: [ + 'test-*', + '.asset-criticality.asset-criticality-default', + 'risk-score.risk-score-latest-default', + ], + }, + }, + ], + }); + }); + }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts new file mode 100644 index 0000000000000..4eba56d3a757b --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/data_view.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type SuperTest from 'supertest'; + +export const dataViewRouteHelpersFactory = ( + supertest: SuperTest.Agent, + namespace: string = 'default' +) => ({ + create: (name: string) => { + return supertest + .post(`/api/data_views/data_view`) + .set('kbn-xsrf', 'foo') + .send({ + data_view: { + title: `logs-*`, + timeFieldName: '@timestamp', + name: `${name}-${namespace}`, + id: `${name}-${namespace}`, + }, + }) + .expect(200); + }, + delete: (name: string) => { + return supertest + .delete(`/api/data_views/data_view/${name}-${namespace}`) + .set('kbn-xsrf', 'foo') + .expect(200); + }, + updateIndexPattern: (name: string, indexPattern: string) => { + return supertest + .post(`/api/data_views/data_view/${name}-${namespace}`) + .set('kbn-xsrf', 'foo') + .send({ + data_view: { + title: indexPattern, + }, + }) + .expect(200); + }, +}); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 9dc1807d7263c..3ac171de1d4fd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -37,7 +37,7 @@ export const EntityStoreUtils = ( } }; - const initEntityEngineForEntityType = async (entityType: EntityType) => { + const initEntityEngineForEntityType = (entityType: EntityType) => { log.info(`Initializing engine for entity type ${entityType} in namespace ${namespace}`); return api .initEntityEngine(