From 8b0ea6e04ae7b4367f66a06e292f218faa42e0cd Mon Sep 17 00:00:00 2001 From: Tiago Vila Verde Date: Thu, 31 Oct 2024 14:29:24 +0100 Subject: [PATCH] [8.16] [Entity Analytics] [Entity Store] Add audit logs (#196847) (#198482) # Backport This will backport the following commits from `main` to `8.16`: - [Entity Analytics] [Entity Store] Add audit logs (#196847) (6c6ae68d) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../entity_store/auditing/actions.ts | 17 ++ .../entity_store/auditing/resources.ts | 18 ++ .../entity_store/entity_store_data_client.ts | 180 +++++++++++++----- .../entity_analytics/utils/entity_store.ts | 3 + 4 files changed, 171 insertions(+), 47 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts create mode 100644 x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts new file mode 100644 index 0000000000000..63d594a9711a3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/actions.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EntityEngineActions = { + INIT: 'init', + START: 'start', + STOP: 'stop', + CREATE: 'create', + DELETE: 'delete', + EXECUTE: 'execute', +} as const; + +export type EntityEngineActions = (typeof EntityEngineActions)[keyof typeof EntityEngineActions]; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts new file mode 100644 index 0000000000000..67d33fb42dc93 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/auditing/resources.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const EntityStoreResource = { + ENTITY_ENGINE: 'entity_engine', + ENTITY_DEFINITION: 'entity_definition', + ENTITY_INDEX: 'entity_index', + INDEX_COMPONENT_TEMPLATE: 'index_component_template', + PLATFORM_PIPELINE: 'platform_pipeline', + FIELD_RETENTION_ENRICH_POLICY: 'field_retention_enrich_policy', + FIELD_RETENTION_ENRICH_POLICY_TASK: 'field_retention_enrich_policy_task', +} as const; + +export type EntityStoreResource = (typeof EntityStoreResource)[keyof typeof EntityStoreResource]; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index 3afb9c6f7e115..ed6a5f2850e7f 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -10,6 +10,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract, AuditLogger, + AuditEvent, AnalyticsServiceSetup, IScopedClusterClient, } from '@kbn/core/server'; @@ -56,12 +57,15 @@ import { isPromiseRejected, } from './utils'; +import { EntityEngineActions } from './auditing/actions'; +import { EntityStoreResource } from './auditing/resources'; +import { AUDIT_CATEGORY, AUDIT_OUTCOME, AUDIT_TYPE } from '../audit'; +import type { EntityRecord, EntityStoreConfig } from './types'; import { ENTITY_ENGINE_INITIALIZATION_EVENT, ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT, } from '../../telemetry/event_based/events'; -import type { EntityRecord, EntityStoreConfig } from './types'; import { CRITICALITY_VALUES } from '../asset_criticality/constants'; interface EntityStoreClientOpts { @@ -133,7 +137,7 @@ export class EntityStoreDataClient { throw new Error('Task Manager is not available'); } - const { logger, config } = this.options; + const { config } = this.options; await this.riskScoreDataClient.createRiskScoreLatestIndex(); @@ -145,18 +149,20 @@ export class EntityStoreDataClient { 'Asset criticality data migration is required before initializing entity store. If this error persists, please restart Kibana.' ); } - logger.info( - `[Entity Store] In namespace ${this.options.namespace}: Initializing entity store for ${entityType}` - ); + this.log('info', entityType, `Initializing entity store`); + this.audit( + EntityEngineActions.INIT, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Initializing entity engine' + ); const descriptor = await this.engineClient.init(entityType, { filter, fieldHistoryLength, indexPattern, }); - logger.debug(`[Entity Store] Initialized saved object for ${entityType}`); - // first create the entity definition without starting it - // so that the index template is created which we can add a component template to + this.log('debug', entityType, `Initialized engine saved object`); this.asyncSetup( entityType, @@ -166,11 +172,9 @@ export class EntityStoreDataClient { filter, config, pipelineDebugMode - ).catch((error) => { - logger.error( - `[Entity Store] There was an error during async setup of the Entity Store: ${error.message}` - ); - }); + ).catch((e) => + this.log('error', entityType, `Error during async setup of entity store: ${e.message}`) + ); return descriptor; } @@ -198,9 +202,6 @@ export class EntityStoreDataClient { }); const { entityManagerDefinition } = unitedDefinition; - const debugLog = (message: string) => - logger.debug(`[Entity Engine] [${entityType}] ${message}`); - try { // clean up any existing entity store await this.delete(entityType, taskManager, { deleteData: false, deleteEngine: false }); @@ -216,7 +217,7 @@ export class EntityStoreDataClient { }, installOnly: true, }); - debugLog(`Created entity definition`); + this.log(`debug`, entityType, `Created entity definition`); // the index must be in place with the correct mapping before the enrich policy is created // this is because the enrich policy will fail if the index does not exist with the correct fields @@ -224,14 +225,14 @@ export class EntityStoreDataClient { unitedDefinition, esClient: this.esClient, }); - debugLog(`Created entity index component template`); + this.log(`debug`, entityType, `Created entity index component template`); await createEntityIndex({ entityType, esClient: this.esClient, namespace, logger, }); - debugLog(`Created entity index`); + this.log(`debug`, entityType, `Created entity index`); // we must create and execute the enrich policy before the pipeline is created // this is because the pipeline will fail if the enrich index does not exist @@ -239,24 +240,24 @@ export class EntityStoreDataClient { unitedDefinition, esClient: this.esClient, }); - debugLog(`Created field retention enrich policy`); + this.log(`debug`, entityType, `Created field retention enrich policy`); + await executeFieldRetentionEnrichPolicy({ unitedDefinition, esClient: this.esClient, logger, }); - debugLog(`Executed field retention enrich policy`); + this.log(`debug`, entityType, `Executed field retention enrich policy`); await createPlatformPipeline({ debugMode: pipelineDebugMode, unitedDefinition, logger, esClient: this.esClient, }); - debugLog(`Created @platform pipeline`); + this.log(`debug`, entityType, `Created @platform pipeline`); // finally start the entity definition now that everything is in place const updated = await this.start(entityType, { force: true }); - debugLog(`Started entity definition`); // the task will execute the enrich policy on a schedule await startEntityStoreFieldRetentionEnrichTask({ @@ -264,7 +265,8 @@ export class EntityStoreDataClient { logger, taskManager, }); - debugLog(`Entity store initialized`); + this.log(`debug`, entityType, `Started entity store field retention enrich task`); + this.log(`info`, entityType, `Entity store initialized`); const setupEndTime = moment().utc().toISOString(); const duration = moment(setupEndTime).diff(moment(setupStartTime), 'seconds'); @@ -274,10 +276,13 @@ export class EntityStoreDataClient { return updated; } catch (err) { - this.options.logger.error( - `[Entity Store] Error initializing entity store for ${entityType}: ${err.message}` + this.audit( + EntityEngineActions.INIT, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Failed to initialize entity engine resources', + err ); - this.options.telemetry?.reportEvent(ENTITY_ENGINE_RESOURCE_INIT_FAILURE_EVENT.eventType, { error: err.message, }); @@ -312,41 +317,54 @@ export class EntityStoreDataClient { } public async start(entityType: EntityType, options?: { force: boolean }) { + const { namespace } = this.options; const descriptor = await this.engineClient.get(entityType); if (!options?.force && descriptor.status !== ENGINE_STATUS.STOPPED) { throw new Error( - `In namespace ${this.options.namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}` + `In namespace ${namespace}: Cannot start Entity engine for ${entityType} when current status is: ${descriptor.status}` ); } - this.options.logger.info( - `In namespace ${this.options.namespace}: Starting entity store for ${entityType}` - ); + this.log('info', entityType, `Starting entity store`); // startEntityDefinition requires more fields than the engine descriptor // provides so we need to fetch the full entity definition const fullEntityDefinition = await this.getExistingEntityDefinition(entityType); + this.audit( + EntityEngineActions.START, + EntityStoreResource.ENTITY_DEFINITION, + entityType, + 'Starting entity definition' + ); await this.entityClient.startEntityDefinition(fullEntityDefinition); + this.log('debug', entityType, `Started entity definition`); return this.engineClient.updateStatus(entityType, ENGINE_STATUS.STARTED); } public async stop(entityType: EntityType) { + const { namespace } = this.options; const descriptor = await this.engineClient.get(entityType); if (descriptor.status !== ENGINE_STATUS.STARTED) { throw new Error( - `In namespace ${this.options.namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}` + `In namespace ${namespace}: Cannot stop Entity engine for ${entityType} when current status is: ${descriptor.status}` ); } - this.options.logger.info( - `In namespace ${this.options.namespace}: Stopping entity store for ${entityType}` - ); + this.log('info', entityType, `Stopping entity store`); + // stopEntityDefinition requires more fields than the engine descriptor // provides so we need to fetch the full entity definition const fullEntityDefinition = await this.getExistingEntityDefinition(entityType); + this.audit( + EntityEngineActions.STOP, + EntityStoreResource.ENTITY_DEFINITION, + entityType, + 'Stopping entity definition' + ); await this.entityClient.stopEntityDefinition(fullEntityDefinition); + this.log('debug', entityType, `Stopped entity definition`); return this.engineClient.updateStatus(entityType, ENGINE_STATUS.STOPPED); } @@ -380,32 +398,46 @@ export class EntityStoreDataClient { frequency: `${config.frequency.asSeconds()}s`, }); const { entityManagerDefinition } = unitedDefinition; - logger.info( - `[Entity Store] In namespace ${namespace}: Deleting entity store for ${entityType}` + + this.log('info', entityType, `Deleting entity store`); + this.audit( + EntityEngineActions.DELETE, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Deleting entity engine' ); + try { - try { - await this.entityClient.deleteEntityDefinition({ + await this.entityClient + .deleteEntityDefinition({ id: entityManagerDefinition.id, deleteData, - }); - } catch (e) { - logger.warn(`Error deleting entity definition for ${entityType}: ${e.message}`); - } + }) + // Swallowing the error as it is expected to fail if no entity definition exists + .catch((e) => + this.log(`warn`, entityType, `Error deleting entity definition: ${e.message}`) + ); + this.log('debug', entityType, `Deleted entity definition`); + await deleteEntityIndexComponentTemplate({ unitedDefinition, esClient: this.esClient, }); + this.log('debug', entityType, `Deleted entity index component template`); + await deletePlatformPipeline({ unitedDefinition, logger, esClient: this.esClient, }); + this.log('debug', entityType, `Deleted platform pipeline`); + await deleteFieldRetentionEnrichPolicy({ unitedDefinition, logger, esClient: this.esClient, }); + this.log('debug', entityType, `Deleted field retention enrich policy`); if (deleteData) { await deleteEntityIndex({ @@ -414,6 +446,7 @@ export class EntityStoreDataClient { namespace, logger, }); + this.log('debug', entityType, `Deleted entity index`); } if (descriptor && deleteEngine) { @@ -427,14 +460,23 @@ export class EntityStoreDataClient { logger, taskManager, }); + this.log('debug', entityType, `Deleted entity store field retention enrich task`); } logger.info(`[Entity Store] In namespace ${namespace}: Deleted store for ${entityType}`); return { deleted: true }; - } catch (e) { - logger.error(`Error deleting entity store for ${entityType}: ${e.message}`); - // TODO: should we set the engine status to error here? - throw e; + } catch (err) { + this.log(`error`, entityType, `Error deleting entity store: ${err.message}`); + + this.audit( + EntityEngineActions.DELETE, + EntityStoreResource.ENTITY_ENGINE, + entityType, + 'Failed to delete entity engine', + err + ); + + throw err; } } @@ -562,4 +604,48 @@ export class EntityStoreDataClient { errors: updateErrors, }; } + + private log( + level: Exclude, + entityType: EntityType, + msg: string + ) { + this.options.logger[level]( + `[Entity Engine] [entity.${entityType}] [namespace: ${this.options.namespace}] ${msg}` + ); + } + + private audit( + action: EntityEngineActions, + resource: EntityStoreResource, + entityType: EntityType, + msg: string, + error?: Error + ) { + // NOTE: Excluding errors, all auditing events are currently WRITE events, meaning the outcome is always UNKNOWN. + // This may change in the future, depending on the audit action. + const outcome = error ? AUDIT_OUTCOME.FAILURE : AUDIT_OUTCOME.UNKNOWN; + + const type = + action === EntityEngineActions.CREATE + ? AUDIT_TYPE.CREATION + : EntityEngineActions.DELETE + ? AUDIT_TYPE.DELETION + : AUDIT_TYPE.CHANGE; + + const category = AUDIT_CATEGORY.DATABASE; + + const message = error ? `${msg}: ${error.message}` : msg; + const event: AuditEvent = { + message: `[Entity Engine] [entity.${entityType}] ${message}`, + event: { + action: `${action}_${entityType}_${resource}`, + category, + outcome, + type, + }, + }; + + return this.options.auditLogger?.log(event); + } } diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts index 029103425af68..7ee32e20640d6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/utils/entity_store.ts @@ -82,6 +82,9 @@ export const EntityStoreUtils = ( if (body.engines.every((engine: any) => engine.status === 'started')) { return true; } + if (body.engines.some((engine: any) => engine.status === 'error')) { + throw new Error(`Engines not started: ${JSON.stringify(body)}`); + } return false; } );