diff --git a/x-pack/plugins/event_log/generated/mappings.json b/x-pack/plugins/event_log/generated/mappings.json index cbb59cc3204c0..aba23eef79e3f 100644 --- a/x-pack/plugins/event_log/generated/mappings.json +++ b/x-pack/plugins/event_log/generated/mappings.json @@ -267,6 +267,42 @@ } } }, + "alert": { + "properties": { + "rule": { + "properties": { + "execution": { + "properties": { + "uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "status": { + "type": "keyword", + "ignore_above": 1024 + }, + "status_order": { + "type": "long" + }, + "metrics": { + "properties": { + "total_indexing_duration_ms": { + "type": "long" + }, + "total_search_duration_ms": { + "type": "long" + }, + "execution_gap_duration_s": { + "type": "long" + } + } + } + } + } + } + } + } + }, "saved_objects": { "type": "nested", "properties": { @@ -292,6 +328,13 @@ } } }, + "space_ids": { + "type": "keyword", + "ignore_above": 1024, + "meta": { + "isArray": "true" + } + }, "version": { "type": "version" } diff --git a/x-pack/plugins/event_log/generated/schemas.ts b/x-pack/plugins/event_log/generated/schemas.ts index 15dc182dbe653..e73bafd9cb81e 100644 --- a/x-pack/plugins/event_log/generated/schemas.ts +++ b/x-pack/plugins/event_log/generated/schemas.ts @@ -116,6 +116,28 @@ export const EventSchema = schema.maybe( status: ecsString(), }) ), + alert: schema.maybe( + schema.object({ + rule: schema.maybe( + schema.object({ + execution: schema.maybe( + schema.object({ + uuid: ecsString(), + status: ecsString(), + status_order: ecsNumber(), + metrics: schema.maybe( + schema.object({ + total_indexing_duration_ms: ecsNumber(), + total_search_duration_ms: ecsNumber(), + execution_gap_duration_s: ecsNumber(), + }) + ), + }) + ), + }) + ), + }) + ), saved_objects: schema.maybe( schema.arrayOf( schema.object({ @@ -127,6 +149,7 @@ export const EventSchema = schema.maybe( }) ) ), + space_ids: ecsStringMulti(), version: ecsVersion(), }) ), diff --git a/x-pack/plugins/event_log/scripts/mappings.js b/x-pack/plugins/event_log/scripts/mappings.js index d114603052491..231cc225f7c47 100644 --- a/x-pack/plugins/event_log/scripts/mappings.js +++ b/x-pack/plugins/event_log/scripts/mappings.js @@ -49,6 +49,42 @@ exports.EcsCustomPropertyMappings = { }, }, }, + alert: { + properties: { + rule: { + properties: { + execution: { + properties: { + uuid: { + type: 'keyword', + ignore_above: 1024, + }, + status: { + type: 'keyword', + ignore_above: 1024, + }, + status_order: { + type: 'long', + }, + metrics: { + properties: { + total_indexing_duration_ms: { + type: 'long', + }, + total_search_duration_ms: { + type: 'long', + }, + execution_gap_duration_s: { + type: 'long', + }, + }, + }, + }, + }, + }, + }, + }, + }, // array of saved object references, for "linking" via search saved_objects: { type: 'nested', @@ -77,6 +113,10 @@ exports.EcsCustomPropertyMappings = { }, }, }, + space_ids: { + type: 'keyword', + ignore_above: 1024, + }, version: { type: 'version', }, @@ -105,4 +145,10 @@ exports.EcsPropertiesToGenerate = [ /** * These properties can have multiple values (are arrays in the generated event schema). */ -exports.EcsEventLogMultiValuedProperties = ['tags', 'event.category', 'event.type', 'rule.author']; +exports.EcsEventLogMultiValuedProperties = [ + 'tags', + 'event.category', + 'event.type', + 'rule.author', + 'kibana.space_ids', +]; diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 8bb1f4d75e6bc..a76b942e555bc 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -12,15 +12,16 @@ "actions", "alerting", "cases", - "ruleRegistry", "data", "dataEnhanced", "embeddable", + "eventLog", "features", - "taskManager", "inspector", "licensing", "maps", + "ruleRegistry", + "taskManager", "timelines", "triggersActionsUi", "uiActions" diff --git a/x-pack/plugins/security_solution/server/config.ts b/x-pack/plugins/security_solution/server/config.ts index 0850e43b21eda..bc5b43c6d25fd 100644 --- a/x-pack/plugins/security_solution/server/config.ts +++ b/x-pack/plugins/security_solution/server/config.ts @@ -12,6 +12,7 @@ import { getExperimentalAllowedValues, isValidExperimentalValue, } from '../common/experimental_features'; +import { UnderlyingLogClient } from './lib/detection_engine/rule_execution_log/types'; const allowedExperimentalValues = getExperimentalAllowedValues(); @@ -103,6 +104,19 @@ export const configSchema = schema.object({ }, }), + /** + * Rule Execution Log Configuration + */ + ruleExecutionLog: schema.object({ + underlyingClient: schema.oneOf( + [ + schema.literal(UnderlyingLogClient.eventLog), + schema.literal(UnderlyingLogClient.savedObjects), + ], + { defaultValue: UnderlyingLogClient.savedObjects } + ), + }), + /** * Host Endpoint Configuration */ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts index 1ac85f9a27969..2f401d27813ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts @@ -11,6 +11,7 @@ import { serverMock } from './server'; import { requestMock } from './request'; import { responseMock } from './response_factory'; import { ConfigType } from '../../../../config'; +import { UnderlyingLogClient } from '../../rule_execution_log/types'; export { requestMock, requestContextMock, responseMock, serverMock }; @@ -29,6 +30,9 @@ export const createMockConfig = (): ConfigType => ({ alertIgnoreFields: [], prebuiltRulesFromFileSystem: true, prebuiltRulesFromSavedObjects: false, + ruleExecutionLog: { + underlyingClient: UnderlyingLogClient.savedObjects, + }, }); export const mockGetCurrentUser = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts index bc9723e47a9d0..910e1ecaa508f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts @@ -14,7 +14,7 @@ export const ruleExecutionLogClientMock = { update: jest.fn(), delete: jest.fn(), logStatusChange: jest.fn(), - logExecutionMetric: jest.fn(), + logExecutionMetrics: jest.fn(), }), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts new file mode 100644 index 0000000000000..f09eb43bf15f1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const RULE_EXECUTION_LOG_PROVIDER = 'rule-execution.security'; + +export const ALERT_SAVED_OBJECT_TYPE = 'alert'; + +export enum RuleExecutionLogAction { + 'status-change' = 'status-change', + 'execution-metrics' = 'execution-metrics', +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts new file mode 100644 index 0000000000000..6b1a0cd5b18d0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogService } from '../../../../../../event_log/server'; +import { + FindBulkExecutionLogArgs, + FindExecutionLogArgs, + IRuleExecutionLogClient, + LogExecutionMetricsArgs, + LogStatusChangeArgs, + UpdateExecutionLogArgs, +} from '../types'; +import { EventLogClient } from './event_log_client'; + +export class EventLogAdapter implements IRuleExecutionLogClient { + private eventLogClient: EventLogClient; + + constructor(eventLogService: IEventLogService) { + this.eventLogClient = new EventLogClient(eventLogService); + } + + public async find({ ruleId, logsCount = 1, spaceId }: FindExecutionLogArgs) { + return []; // TODO Implement + } + + public async findBulk({ ruleIds, logsCount = 1, spaceId }: FindBulkExecutionLogArgs) { + return {}; // TODO Implement + } + + public async update({ attributes, spaceId, ruleName, ruleType }: UpdateExecutionLogArgs) { + // execution events are immutable, so we just log a status change istead of updating previous + if (attributes.status) { + this.eventLogClient.logStatusChange({ + ruleName, + ruleType, + ruleId: attributes.alertId, + newStatus: attributes.status, + spaceId, + }); + } + } + + public async delete(id: string) { + // execution events are immutable, nothing to do here + } + + public async logExecutionMetrics({ + ruleId, + spaceId, + ruleType, + ruleName, + metrics, + }: LogExecutionMetricsArgs) { + this.eventLogClient.logExecutionMetrics({ + ruleId, + ruleName, + ruleType, + spaceId, + metrics: { + executionGapDuration: metrics.executionGap?.asSeconds(), + totalIndexingDuration: metrics.indexingDurations?.reduce( + (acc, cur) => acc + Number(cur), + 0 + ), + totalSearchDuration: metrics.searchDurations?.reduce((acc, cur) => acc + Number(cur), 0), + }, + }); + } + + public async logStatusChange(args: LogStatusChangeArgs) { + if (args.metrics) { + this.logExecutionMetrics({ + ruleId: args.ruleId, + ruleName: args.ruleName, + ruleType: args.ruleType, + spaceId: args.spaceId, + metrics: args.metrics, + }); + } + + this.eventLogClient.logStatusChange(args); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts new file mode 100644 index 0000000000000..d85c67e422035 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_client.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsUtils } from '../../../../../../../../src/core/server'; +import { + IEventLogger, + IEventLogService, + SAVED_OBJECT_REL_PRIMARY, +} from '../../../../../../event_log/server'; +import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; +import { LogStatusChangeArgs } from '../types'; +import { + RuleExecutionLogAction, + RULE_EXECUTION_LOG_PROVIDER, + ALERT_SAVED_OBJECT_TYPE, +} from './constants'; + +const spaceIdToNamespace = SavedObjectsUtils.namespaceStringToId; + +const statusSeverityDict: Record = { + [RuleExecutionStatus.succeeded]: 0, + [RuleExecutionStatus['going to run']]: 10, + [RuleExecutionStatus.warning]: 20, + [RuleExecutionStatus['partial failure']]: 20, + [RuleExecutionStatus.failed]: 30, +}; + +interface FindExecutionLogArgs { + ruleIds: string[]; + spaceId: string; + logsCount?: number; + statuses?: RuleExecutionStatus[]; +} + +interface LogExecutionMetricsArgs { + ruleId: string; + ruleName: string; + ruleType: string; + spaceId: string; + metrics: EventLogExecutionMetrics; +} + +interface EventLogExecutionMetrics { + totalSearchDuration?: number; + totalIndexingDuration?: number; + executionGapDuration?: number; +} + +interface IExecLogEventLogClient { + find: (args: FindExecutionLogArgs) => Promise<{}>; + logStatusChange: (args: LogStatusChangeArgs) => void; + logExecutionMetrics: (args: LogExecutionMetricsArgs) => void; +} + +export class EventLogClient implements IExecLogEventLogClient { + private sequence = 0; + private eventLogger: IEventLogger; + + constructor(eventLogService: IEventLogService) { + this.eventLogger = eventLogService.getLogger({ + event: { provider: RULE_EXECUTION_LOG_PROVIDER }, + }); + } + + public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) { + return {}; // TODO implement + } + + public logExecutionMetrics({ + ruleId, + ruleName, + ruleType, + metrics, + spaceId, + }: LogExecutionMetricsArgs) { + this.eventLogger.logEvent({ + rule: { + id: ruleId, + name: ruleName, + category: ruleType, + }, + event: { + kind: 'metric', + action: RuleExecutionLogAction['execution-metrics'], + sequence: this.sequence++, + }, + kibana: { + alert: { + rule: { + execution: { + metrics: { + execution_gap_duration_s: metrics.executionGapDuration, + total_search_duration_ms: metrics.totalSearchDuration, + total_indexing_duration_ms: metrics.totalIndexingDuration, + }, + }, + }, + }, + space_ids: [spaceId], + saved_objects: [ + { + rel: SAVED_OBJECT_REL_PRIMARY, + type: ALERT_SAVED_OBJECT_TYPE, + id: ruleId, + namespace: spaceIdToNamespace(spaceId), + }, + ], + }, + }); + } + + public logStatusChange({ + ruleId, + ruleName, + ruleType, + newStatus, + message, + spaceId, + }: LogStatusChangeArgs) { + this.eventLogger.logEvent({ + rule: { + id: ruleId, + name: ruleName, + category: ruleType, + }, + event: { + kind: 'event', + action: RuleExecutionLogAction['status-change'], + sequence: this.sequence++, + }, + message, + kibana: { + alert: { + rule: { + execution: { + status: newStatus, + status_order: statusSeverityDict[newStatus], + }, + }, + }, + space_ids: [spaceId], + saved_objects: [ + { + rel: SAVED_OBJECT_REL_PRIMARY, + type: ALERT_SAVED_OBJECT_TYPE, + id: ruleId, + namespace: spaceIdToNamespace(spaceId), + }, + ], + }, + }); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider.ts new file mode 100644 index 0000000000000..7f28715198da6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogService } from '../../../../../../event_log/server'; +import { RuleExecutionLogAction, RULE_EXECUTION_LOG_PROVIDER } from './constants'; + +export const registerEventLogProvider = (eventLogService: IEventLogService) => { + eventLogService.registerProviderActions( + RULE_EXECUTION_LOG_PROVIDER, + Object.keys(RuleExecutionLogAction) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts index 135cefe2243b2..87a3b00cf4ed3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_execution_log_client.ts @@ -6,34 +6,40 @@ */ import { SavedObjectsClientContract } from '../../../../../../../src/core/server'; -import { RuleRegistryAdapter } from './rule_registry_adapter/rule_registry_adapter'; +import { IEventLogService } from '../../../../../event_log/server'; +import { EventLogAdapter } from './event_log_adapter/event_log_adapter'; import { SavedObjectsAdapter } from './saved_objects_adapter/saved_objects_adapter'; import { - ExecutionMetric, - ExecutionMetricArgs, + LogExecutionMetricsArgs, FindBulkExecutionLogArgs, FindExecutionLogArgs, - IRuleDataPluginService, IRuleExecutionLogClient, LogStatusChangeArgs, UpdateExecutionLogArgs, + UnderlyingLogClient, } from './types'; export interface RuleExecutionLogClientArgs { - ruleDataService: IRuleDataPluginService; savedObjectsClient: SavedObjectsClientContract; + eventLogService: IEventLogService; + underlyingClient: UnderlyingLogClient; } -const RULE_REGISTRY_LOG_ENABLED = false; - export class RuleExecutionLogClient implements IRuleExecutionLogClient { private client: IRuleExecutionLogClient; - constructor({ ruleDataService, savedObjectsClient }: RuleExecutionLogClientArgs) { - if (RULE_REGISTRY_LOG_ENABLED) { - this.client = new RuleRegistryAdapter(ruleDataService); - } else { - this.client = new SavedObjectsAdapter(savedObjectsClient); + constructor({ + savedObjectsClient, + eventLogService, + underlyingClient, + }: RuleExecutionLogClientArgs) { + switch (underlyingClient) { + case UnderlyingLogClient.savedObjects: + this.client = new SavedObjectsAdapter(savedObjectsClient); + break; + case UnderlyingLogClient.eventLog: + this.client = new EventLogAdapter(eventLogService); + break; } } @@ -53,8 +59,8 @@ export class RuleExecutionLogClient implements IRuleExecutionLogClient { return this.client.delete(id); } - public async logExecutionMetric(args: ExecutionMetricArgs) { - return this.client.logExecutionMetric(args); + public async logExecutionMetrics(args: LogExecutionMetricsArgs) { + return this.client.logExecutionMetrics(args); } public async logStatusChange(args: LogStatusChangeArgs) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_adapter.ts deleted file mode 100644 index ab8664ae995bf..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_adapter.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { merge } from 'lodash'; -import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; -import { RuleRegistryLogClient } from './rule_registry_log_client/rule_registry_log_client'; -import { - CreateExecutionLogArgs, - ExecutionMetric, - ExecutionMetricArgs, - FindBulkExecutionLogArgs, - FindExecutionLogArgs, - IRuleDataPluginService, - IRuleExecutionLogClient, - LogStatusChangeArgs, - UpdateExecutionLogArgs, -} from '../types'; - -/** - * @deprecated RuleRegistryAdapter is kept here only as a reference. It will be superseded with EventLog implementation - */ -export class RuleRegistryAdapter implements IRuleExecutionLogClient { - private ruleRegistryClient: RuleRegistryLogClient; - - constructor(ruleDataService: IRuleDataPluginService) { - this.ruleRegistryClient = new RuleRegistryLogClient(ruleDataService); - } - - public async find({ ruleId, logsCount = 1, spaceId }: FindExecutionLogArgs) { - const logs = await this.ruleRegistryClient.find({ - ruleIds: [ruleId], - logsCount, - spaceId, - }); - - return logs[ruleId].map((log) => ({ - id: '', - type: '', - score: 0, - attributes: log, - references: [], - })); - } - - public async findBulk({ ruleIds, logsCount = 1, spaceId }: FindBulkExecutionLogArgs) { - const [statusesById, lastErrorsById] = await Promise.all([ - this.ruleRegistryClient.find({ ruleIds, spaceId }), - this.ruleRegistryClient.find({ - ruleIds, - statuses: [RuleExecutionStatus.failed], - logsCount, - spaceId, - }), - ]); - return merge(statusesById, lastErrorsById); - } - - private async create({ attributes, spaceId }: CreateExecutionLogArgs) { - if (attributes.status) { - await this.ruleRegistryClient.logStatusChange({ - ruleId: attributes.alertId, - newStatus: attributes.status, - spaceId, - }); - } - - if (attributes.bulkCreateTimeDurations) { - await this.ruleRegistryClient.logExecutionMetric({ - ruleId: attributes.alertId, - metric: ExecutionMetric.indexingDurationMax, - value: Math.max(...attributes.bulkCreateTimeDurations.map(Number)), - spaceId, - }); - } - - if (attributes.gap) { - await this.ruleRegistryClient.logExecutionMetric({ - ruleId: attributes.alertId, - metric: ExecutionMetric.executionGap, - value: Number(attributes.gap), - spaceId, - }); - } - } - - public async update({ attributes, spaceId }: UpdateExecutionLogArgs) { - // execution events are immutable, so we just use 'create' here instead of 'update' - await this.create({ attributes, spaceId }); - } - - public async delete(id: string) { - // execution events are immutable, nothing to do here - } - - public async logExecutionMetric(args: ExecutionMetricArgs) { - return this.ruleRegistryClient.logExecutionMetric(args); - } - - public async logStatusChange(args: LogStatusChangeArgs) { - return this.ruleRegistryClient.logStatusChange(args); - } -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/constants.ts deleted file mode 100644 index 8d74c71bf447d..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/constants.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * @deprecated EVENTS_INDEX_PREFIX is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const EVENTS_INDEX_PREFIX = '.kibana_alerts-security.events'; - -/** - * @deprecated MESSAGE is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const MESSAGE = 'message' as const; - -/** - * @deprecated EVENT_SEQUENCE is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const EVENT_SEQUENCE = 'event.sequence' as const; - -/** - * @deprecated EVENT_DURATION is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const EVENT_DURATION = 'event.duration' as const; - -/** - * @deprecated EVENT_END is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const EVENT_END = 'event.end' as const; - -/** - * @deprecated RULE_STATUS is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const RULE_STATUS = 'kibana.rac.detection_engine.rule_status' as const; - -/** - * @deprecated RULE_STATUS_SEVERITY is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const RULE_STATUS_SEVERITY = 'kibana.rac.detection_engine.rule_status_severity' as const; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/parse_rule_execution_log.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/parse_rule_execution_log.ts deleted file mode 100644 index cbc6e570e936f..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/parse_rule_execution_log.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isLeft } from 'fp-ts/lib/Either'; -import { PathReporter } from 'io-ts/lib/PathReporter'; -import { technicalRuleFieldMap } from '../../../../../../../rule_registry/common/assets/field_maps/technical_rule_field_map'; -import { - mergeFieldMaps, - runtimeTypeFromFieldMap, -} from '../../../../../../../rule_registry/common/field_map'; -import { ruleExecutionFieldMap } from './rule_execution_field_map'; - -const ruleExecutionLogRuntimeType = runtimeTypeFromFieldMap( - mergeFieldMaps(technicalRuleFieldMap, ruleExecutionFieldMap) -); - -/** - * @deprecated parseRuleExecutionLog is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const parseRuleExecutionLog = (input: unknown) => { - const validate = ruleExecutionLogRuntimeType.decode(input); - - if (isLeft(validate)) { - throw new Error(PathReporter.report(validate).join('\n')); - } - - return ruleExecutionLogRuntimeType.encode(validate.right); -}; - -/** - * @deprecated RuleExecutionEvent is kept here only as a reference. It will be superseded with EventLog implementation - * - * It's marked as `Partial` because the field map is not yet appropriate for - * execution log events. - */ -export type RuleExecutionEvent = Partial>; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_execution_field_map.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_execution_field_map.ts deleted file mode 100644 index b3c70cd56d9e6..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_execution_field_map.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - EVENT_DURATION, - EVENT_END, - EVENT_SEQUENCE, - MESSAGE, - RULE_STATUS, - RULE_STATUS_SEVERITY, -} from './constants'; - -/** - * @deprecated ruleExecutionFieldMap is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const ruleExecutionFieldMap = { - [MESSAGE]: { type: 'keyword' }, - [EVENT_SEQUENCE]: { type: 'long' }, - [EVENT_END]: { type: 'date' }, - [EVENT_DURATION]: { type: 'long' }, - [RULE_STATUS]: { type: 'keyword' }, - [RULE_STATUS_SEVERITY]: { type: 'integer' }, -} as const; - -/** - * @deprecated RuleExecutionFieldMap is kept here only as a reference. It will be superseded with EventLog implementation - */ -export type RuleExecutionFieldMap = typeof ruleExecutionFieldMap; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts deleted file mode 100644 index 3cd6171b5bbeb..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/rule_registry_log_client.ts +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { - ALERT_RULE_CONSUMER, - ALERT_RULE_TYPE_ID, - EVENT_ACTION, - EVENT_KIND, - SPACE_IDS, - TIMESTAMP, - ALERT_RULE_UUID, -} from '@kbn/rule-data-utils'; -import moment from 'moment'; - -import { mappingFromFieldMap } from '../../../../../../../rule_registry/common/mapping_from_field_map'; -import { Dataset, IRuleDataClient } from '../../../../../../../rule_registry/server'; -import { SERVER_APP_ID } from '../../../../../../common/constants'; -import { RuleExecutionStatus } from '../../../../../../common/detection_engine/schemas/common/schemas'; -import { invariant } from '../../../../../../common/utils/invariant'; -import { IRuleStatusSOAttributes } from '../../../rules/types'; -import { makeFloatString } from '../../../signals/utils'; -import { - ExecutionMetric, - ExecutionMetricArgs, - IRuleDataPluginService, - LogStatusChangeArgs, -} from '../../types'; -import { EVENT_SEQUENCE, MESSAGE, RULE_STATUS, RULE_STATUS_SEVERITY } from './constants'; -import { parseRuleExecutionLog, RuleExecutionEvent } from './parse_rule_execution_log'; -import { ruleExecutionFieldMap } from './rule_execution_field_map'; -import { - getLastEntryAggregation, - getMetricAggregation, - getMetricField, - sortByTimeDesc, -} from './utils'; - -const statusSeverityDict: Record = { - [RuleExecutionStatus.succeeded]: 0, - [RuleExecutionStatus['going to run']]: 10, - [RuleExecutionStatus.warning]: 20, - [RuleExecutionStatus['partial failure']]: 20, - [RuleExecutionStatus.failed]: 30, -}; - -interface FindExecutionLogArgs { - ruleIds: string[]; - spaceId: string; - logsCount?: number; - statuses?: RuleExecutionStatus[]; -} - -interface IRuleRegistryLogClient { - find: (args: FindExecutionLogArgs) => Promise<{ - [ruleId: string]: IRuleStatusSOAttributes[] | undefined; - }>; - create: (event: RuleExecutionEvent) => Promise; - logStatusChange: (args: LogStatusChangeArgs) => Promise; - logExecutionMetric: (args: ExecutionMetricArgs) => Promise; -} - -/** - * @deprecated RuleRegistryLogClient is kept here only as a reference. It will be superseded with EventLog implementation - */ -export class RuleRegistryLogClient implements IRuleRegistryLogClient { - private sequence = 0; - private ruleDataClient: IRuleDataClient; - - constructor(ruleDataService: IRuleDataPluginService) { - this.ruleDataClient = ruleDataService.initializeIndex({ - feature: SERVER_APP_ID, - registrationContext: 'security', - dataset: Dataset.events, - componentTemplateRefs: [], - componentTemplates: [ - { - name: 'mappings', - mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'), - }, - ], - }); - } - - public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) { - if (ruleIds.length === 0) { - return {}; - } - - const filter: estypes.QueryDslQueryContainer[] = [ - { terms: { [ALERT_RULE_UUID]: ruleIds } }, - { terms: { [SPACE_IDS]: [spaceId] } }, - ]; - - if (statuses) { - filter.push({ terms: { [RULE_STATUS]: statuses } }); - } - - const result = await this.ruleDataClient.getReader().search({ - size: 0, - body: { - query: { - bool: { - filter, - }, - }, - aggs: { - rules: { - terms: { - field: ALERT_RULE_UUID, - size: ruleIds.length, - }, - aggs: { - most_recent_logs: { - top_hits: { - sort: sortByTimeDesc, - size: logsCount, - }, - }, - last_failure: getLastEntryAggregation(RuleExecutionStatus.failed), - last_success: getLastEntryAggregation(RuleExecutionStatus.succeeded), - execution_gap: getMetricAggregation(ExecutionMetric.executionGap), - search_duration_max: getMetricAggregation(ExecutionMetric.searchDurationMax), - indexing_duration_max: getMetricAggregation(ExecutionMetric.indexingDurationMax), - indexing_lookback: getMetricAggregation(ExecutionMetric.indexingLookback), - }, - }, - }, - }, - }); - - if (result.hits.total.value === 0) { - return {}; - } - - invariant(result.aggregations, 'Search response should contain aggregations'); - - return Object.fromEntries( - result.aggregations.rules.buckets.map<[ruleId: string, logs: IRuleStatusSOAttributes[]]>( - (bucket) => [ - bucket.key as string, - bucket.most_recent_logs.hits.hits.map((event) => { - const logEntry = parseRuleExecutionLog(event._source); - invariant( - logEntry[ALERT_RULE_UUID] ?? '', - 'Malformed execution log entry: rule.id field not found' - ); - - const lastFailure = bucket.last_failure.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.last_failure.event.hits.hits[0]._source) - : undefined; - - const lastSuccess = bucket.last_success.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.last_success.event.hits.hits[0]._source) - : undefined; - - const lookBack = bucket.indexing_lookback.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.indexing_lookback.event.hits.hits[0]._source) - : undefined; - - const executionGap = bucket.execution_gap.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.execution_gap.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.executionGap) - ] - : undefined; - - const searchDuration = bucket.search_duration_max.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.search_duration_max.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.searchDurationMax) - ] - : undefined; - - const indexingDuration = bucket.indexing_duration_max.event.hits.hits[0] - ? parseRuleExecutionLog(bucket.indexing_duration_max.event.hits.hits[0]._source)[ - getMetricField(ExecutionMetric.indexingDurationMax) - ] - : undefined; - - const alertId = logEntry[ALERT_RULE_UUID] ?? ''; - const statusDate = logEntry[TIMESTAMP]; - const lastFailureAt = lastFailure?.[TIMESTAMP]; - const lastFailureMessage = lastFailure?.[MESSAGE]; - const lastSuccessAt = lastSuccess?.[TIMESTAMP]; - const lastSuccessMessage = lastSuccess?.[MESSAGE]; - const status = (logEntry[RULE_STATUS] as RuleExecutionStatus) || null; - const lastLookBackDate = lookBack?.[getMetricField(ExecutionMetric.indexingLookback)]; - const gap = executionGap ? moment.duration(executionGap).humanize() : null; - const bulkCreateTimeDurations = indexingDuration - ? [makeFloatString(indexingDuration)] - : null; - const searchAfterTimeDurations = searchDuration - ? [makeFloatString(searchDuration)] - : null; - - return { - alertId, - statusDate, - lastFailureAt, - lastFailureMessage, - lastSuccessAt, - lastSuccessMessage, - status, - lastLookBackDate, - gap, - bulkCreateTimeDurations, - searchAfterTimeDurations, - }; - }), - ] - ) - ); - } - - public async logExecutionMetric({ - ruleId, - namespace, - metric, - value, - spaceId, - }: ExecutionMetricArgs) { - await this.create( - { - [SPACE_IDS]: [spaceId], - [EVENT_ACTION]: metric, - [EVENT_KIND]: 'metric', - [getMetricField(metric)]: value, - [ALERT_RULE_UUID]: ruleId ?? '', - [TIMESTAMP]: new Date().toISOString(), - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_RULE_TYPE_ID]: SERVER_APP_ID, - }, - namespace - ); - } - - public async logStatusChange({ - ruleId, - newStatus, - namespace, - message, - spaceId, - }: LogStatusChangeArgs) { - await this.create( - { - [SPACE_IDS]: [spaceId], - [EVENT_ACTION]: 'status-change', - [EVENT_KIND]: 'event', - [EVENT_SEQUENCE]: this.sequence++, - [MESSAGE]: message, - [ALERT_RULE_UUID]: ruleId ?? '', - [RULE_STATUS_SEVERITY]: statusSeverityDict[newStatus], - [RULE_STATUS]: newStatus, - [TIMESTAMP]: new Date().toISOString(), - [ALERT_RULE_CONSUMER]: SERVER_APP_ID, - [ALERT_RULE_TYPE_ID]: SERVER_APP_ID, - }, - namespace - ); - } - - public async create(event: RuleExecutionEvent, namespace?: string) { - await this.ruleDataClient.getWriter({ namespace }).bulk({ - body: [{ index: {} }, event], - }); - } -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/utils.ts deleted file mode 100644 index 713cf73890e7f..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_adapter/rule_registry_log_client/utils.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SearchSort } from '@elastic/elasticsearch/api/types'; -import { EVENT_ACTION, TIMESTAMP } from '@kbn/rule-data-utils'; -import { RuleExecutionStatus } from '../../../../../../common/detection_engine/schemas/common/schemas'; -import { ExecutionMetric } from '../../types'; -import { RULE_STATUS, EVENT_SEQUENCE, EVENT_DURATION, EVENT_END } from './constants'; - -const METRIC_FIELDS = { - [ExecutionMetric.executionGap]: EVENT_DURATION, - [ExecutionMetric.searchDurationMax]: EVENT_DURATION, - [ExecutionMetric.indexingDurationMax]: EVENT_DURATION, - [ExecutionMetric.indexingLookback]: EVENT_END, -}; - -/** - * Returns ECS field in which metric value is stored - * @deprecated getMetricField is kept here only as a reference. It will be superseded with EventLog implementation - * - * @param metric - execution metric - * @returns ECS field - */ -export const getMetricField = (metric: T) => METRIC_FIELDS[metric]; - -/** - * @deprecated sortByTimeDesc is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const sortByTimeDesc: SearchSort = [{ [TIMESTAMP]: 'desc' }, { [EVENT_SEQUENCE]: 'desc' }]; - -/** - * Builds aggregation to retrieve the most recent metric value - * @deprecated getMetricAggregation is kept here only as a reference. It will be superseded with EventLog implementation - * - * @param metric - execution metric - * @returns aggregation - */ -export const getMetricAggregation = (metric: ExecutionMetric) => ({ - filter: { - term: { [EVENT_ACTION]: metric }, - }, - aggs: { - event: { - top_hits: { - size: 1, - sort: sortByTimeDesc, - _source: [TIMESTAMP, getMetricField(metric)], - }, - }, - }, -}); - -/** - * Builds aggregation to retrieve the most recent log entry with the given status - * @deprecated getLastEntryAggregation is kept here only as a reference. It will be superseded with EventLog implementation - * - * @param status - rule execution status - * @returns aggregation - */ -export const getLastEntryAggregation = (status: RuleExecutionStatus) => ({ - filter: { - term: { [RULE_STATUS]: status }, - }, - aggs: { - event: { - top_hits: { - sort: sortByTimeDesc, - size: 1, - }, - }, - }, -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts index 27329ebf8f90c..ca806bd58e369 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts @@ -14,12 +14,11 @@ import { ruleStatusSavedObjectsClientFactory, } from './rule_status_saved_objects_client'; import { - ExecutionMetric, - ExecutionMetricArgs, + LogExecutionMetricsArgs, FindBulkExecutionLogArgs, FindExecutionLogArgs, IRuleExecutionLogClient, - LegacyMetrics, + ExecutionMetrics, LogStatusChangeArgs, UpdateExecutionLogArgs, } from '../types'; @@ -28,14 +27,16 @@ import { assertUnreachable } from '../../../../../common'; // 1st is mutable status, followed by 5 most recent failures export const MAX_RULE_STATUSES = 6; -const METRIC_FIELDS = { - [ExecutionMetric.executionGap]: 'gap', - [ExecutionMetric.searchDurationMax]: 'searchAfterTimeDurations', - [ExecutionMetric.indexingDurationMax]: 'bulkCreateTimeDurations', - [ExecutionMetric.indexingLookback]: 'lastLookBackDate', -} as const; - -const getMetricField = (metric: T) => METRIC_FIELDS[metric]; +const convertMetricFields = ( + metrics: ExecutionMetrics +): Pick< + IRuleStatusSOAttributes, + 'gap' | 'searchAfterTimeDurations' | 'bulkCreateTimeDurations' +> => ({ + gap: metrics.executionGap?.humanize(), + searchAfterTimeDurations: metrics.searchDurations, + bulkCreateTimeDurations: metrics.indexingDurations, +}); export class SavedObjectsAdapter implements IRuleExecutionLogClient { private ruleStatusClient: RuleStatusSavedObjectsClient; @@ -66,16 +67,12 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { await this.ruleStatusClient.delete(id); } - public async logExecutionMetric({ - ruleId, - metric, - value, - }: ExecutionMetricArgs) { + public async logExecutionMetrics({ ruleId, metrics }: LogExecutionMetricsArgs) { const [currentStatus] = await this.getOrCreateRuleStatuses(ruleId); await this.ruleStatusClient.update(currentStatus.id, { ...currentStatus.attributes, - [getMetricField(metric)]: value, + ...convertMetricFields(metrics), }); } @@ -158,11 +155,11 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient { const buildRuleStatusAttributes: ( status: RuleExecutionStatus, message?: string, - metrics?: LegacyMetrics + metrics?: ExecutionMetrics ) => Partial = (status, message, metrics = {}) => { const now = new Date().toISOString(); const baseAttributes: Partial = { - ...metrics, + ...convertMetricFields(metrics), status: status === RuleExecutionStatus.warning ? RuleExecutionStatus['partial failure'] : status, statusDate: now, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts index 9c66032f681de..e38f974ddee2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/types.ts @@ -5,28 +5,16 @@ * 2.0. */ -import { PublicMethodsOf } from '@kbn/utility-types'; +import { Duration } from 'moment'; import { SavedObjectsFindResult } from '../../../../../../../src/core/server'; -import { RuleDataPluginService } from '../../../../../rule_registry/server'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { IRuleStatusSOAttributes } from '../rules/types'; -export enum ExecutionMetric { - 'executionGap' = 'executionGap', - 'searchDurationMax' = 'searchDurationMax', - 'indexingDurationMax' = 'indexingDurationMax', - 'indexingLookback' = 'indexingLookback', +export enum UnderlyingLogClient { + 'savedObjects' = 'savedObjects', + 'eventLog' = 'eventLog', } -export type IRuleDataPluginService = PublicMethodsOf; - -export type ExecutionMetricValue = { - [ExecutionMetric.executionGap]: number; - [ExecutionMetric.searchDurationMax]: number; - [ExecutionMetric.indexingDurationMax]: number; - [ExecutionMetric.indexingLookback]: Date; -}[T]; - export interface FindExecutionLogArgs { ruleId: string; spaceId: string; @@ -39,29 +27,34 @@ export interface FindBulkExecutionLogArgs { logsCount?: number; } -/** - * @deprecated LegacyMetrics are only kept here for backward compatibility - * and should be replaced by ExecutionMetric in the future - */ -export interface LegacyMetrics { - searchAfterTimeDurations?: string[]; - bulkCreateTimeDurations?: string[]; +export interface ExecutionMetrics { + searchDurations?: string[]; + indexingDurations?: string[]; + /** + * @deprecated lastLookBackDate is logged only by SavedObjectsAdapter and should be removed in the future + */ lastLookBackDate?: string; - gap?: string; + executionGap?: Duration; } export interface LogStatusChangeArgs { ruleId: string; + ruleName: string; + ruleType: string; spaceId: string; newStatus: RuleExecutionStatus; - namespace?: string; message?: string; - metrics?: LegacyMetrics; + /** + * @deprecated Use RuleExecutionLogClient.logExecutionMetrics to write metrics instead + */ + metrics?: ExecutionMetrics; } export interface UpdateExecutionLogArgs { id: string; attributes: IRuleStatusSOAttributes; + ruleName: string; + ruleType: string; spaceId: string; } @@ -70,12 +63,12 @@ export interface CreateExecutionLogArgs { spaceId: string; } -export interface ExecutionMetricArgs { +export interface LogExecutionMetricsArgs { ruleId: string; + ruleName: string; + ruleType: string; spaceId: string; - namespace?: string; - metric: T; - value: ExecutionMetricValue; + metrics: ExecutionMetrics; } export interface FindBulkExecutionLogResponse { @@ -90,5 +83,5 @@ export interface IRuleExecutionLogClient { update: (args: UpdateExecutionLogArgs) => Promise; delete: (id: string) => Promise; logStatusChange: (args: LogStatusChangeArgs) => Promise; - logExecutionMetric: (args: ExecutionMetricArgs) => Promise; + logExecutionMetrics: (args: LogExecutionMetricsArgs) => Promise; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts index bcab8a0af5ffb..c6f818f04fc5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts @@ -14,6 +14,7 @@ import { mlPluginServerMock } from '../../../../../../ml/server/mocks'; import type { IRuleDataClient } from '../../../../../../rule_registry/server'; import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks'; +import { eventLogServiceMock } from '../../../../../../event_log/server/mocks'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server'; import { ConfigType } from '../../../../config'; import { AlertAttributes } from '../../signals/types'; @@ -55,6 +56,7 @@ export const createRuleTypeMocks = ( references: [], attributes: { actions: [], + alertTypeId: 'siem.signals', enabled: true, name: 'mock rule', tags: [], @@ -89,7 +91,7 @@ export const createRuleTypeMocks = ( ruleDataClient: ruleRegistryMocks.createRuleDataClient( '.alerts-security.alerts' ) as IRuleDataClient, - ruleDataService: ruleRegistryMocks.createRuleDataPluginService(), + eventLogService: eventLogServiceMock.create(), }, services, scheduleActions, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts index df15d4b2c0112..9ea36abe997c3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts @@ -40,8 +40,9 @@ import { scheduleThrottledNotificationActions } from '../notifications/schedule_ /* eslint-disable complexity */ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = - ({ lists, logger, mergeStrategy, ignoreFields, ruleDataClient, ruleDataService }) => + ({ lists, logger, config, ruleDataClient, eventLogService }) => (type) => { + const { alertIgnoreFields: ignoreFields, alertMergeStrategy: mergeStrategy } = config; const persistenceRuleType = createPersistenceRuleTypeFactory({ ruleDataClient, logger }); return persistenceRuleType({ ...type, @@ -65,13 +66,15 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = const ruleStatusClient = new RuleExecutionLogClient({ savedObjectsClient, - ruleDataService, + eventLogService, + underlyingClient: config.ruleExecutionLog.underlyingClient, }); const ruleSO = await savedObjectsClient.get('alert', alertId); const { actions, name, + alertTypeId, schedule: { interval }, } = ruleSO.attributes; const refresh = actions.length ? 'wait_for' : false; @@ -87,9 +90,14 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = logger.debug(buildRuleMessage(`interval: ${interval}`)); let wroteWarningStatus = false; - await ruleStatusClient.logStatusChange({ + const basicLogArguments = { spaceId, ruleId: alertId, + ruleName: name, + ruleType: alertTypeId, + }; + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, newStatus: RuleExecutionStatus['going to run'], }); @@ -125,8 +133,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = tryCatch( () => hasReadIndexPrivileges({ - spaceId, - ruleId: alertId, + ...basicLogArguments, privileges, logger, buildRuleMessage, @@ -138,8 +145,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = tryCatch( () => hasTimestampFields({ - spaceId, - ruleId: alertId, + ...basicLogArguments, wroteStatus: wroteStatus as boolean, timestampField: hasTimestampOverride ? (timestampOverride as string) @@ -179,11 +185,10 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = logger.warn(gapMessage); hasError = true; await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message: gapMessage, - metrics: { gap: gapString }, + metrics: { executionGap: remainingGap }, }); } @@ -262,8 +267,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = if (result.warningMessages.length) { const warningMessage = buildRuleMessage(result.warningMessages.join()); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus['partial failure'], message: warningMessage, }); @@ -327,13 +331,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = if (!hasError && !wroteWarningStatus && !result.warning) { await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.succeeded, message: 'succeeded', metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookbackDate?.toISOString(), }, }); @@ -356,13 +359,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ); logger.error(errorMessage); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message: errorMessage, metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookbackDate?.toISOString(), }, }); @@ -376,13 +378,12 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = logger.error(message); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message, metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookbackDate?.toISOString(), }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.test.ts index 868419179c76b..43860d396ac5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.test.ts @@ -12,6 +12,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe import { createEqlAlertType } from './create_eql_alert_type'; import { createRuleTypeMocks } from '../__mocks__/rule_type'; import { getEqlRuleParams } from '../../schemas/rule_schemas.mock'; +import { createMockConfig } from '../../routes/__mocks__'; jest.mock('../../rule_execution_log/rule_execution_log_client'); @@ -26,10 +27,9 @@ describe('Event correlation alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - ignoreFields: [], - mergeStrategy: 'allFields', + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); dependencies.alerting.registerType(eqlAlertType); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 5893c6fdc86c2..9324b469bf644 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createEqlAlertType = (createOptions: CreateRuleOptions) => { - const { - experimentalFeatures, - lists, - logger, - ignoreFields, - mergeStrategy, - ruleDataClient, - version, - ruleDataService, - } = createOptions; + const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } = + createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, - ignoreFields, - mergeStrategy, + config, ruleDataClient, - ruleDataService, + eventLogService, }); return createSecurityRuleType({ id: EQL_RULE_TYPE_ID, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts index 6daafbfae40f2..a7accc4ae8a0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert_group_from_sequence.test.ts @@ -44,6 +44,7 @@ describe('buildAlert', () => { const ruleSO = { attributes: { actions: [], + alertTypeId: 'siem.signals', createdAt: new Date().toISOString(), createdBy: 'gandalf', params: getQueryRuleParams(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts index fe836c872dcad..3db4f5686abdc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts @@ -16,6 +16,7 @@ import { createIndicatorMatchAlertType } from './create_indicator_match_alert_ty import { sampleDocNoSortId } from '../../signals/__mocks__/es_results'; import { CountResponse } from 'kibana/server'; import { RuleParams } from '../../schemas/rule_schemas'; +import { createMockConfig } from '../../routes/__mocks__'; jest.mock('../utils/get_list_client', () => ({ getListClient: jest.fn().mockReturnValue({ @@ -56,10 +57,9 @@ describe('Indicator Match Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - ignoreFields: [], - mergeStrategy: 'allFields', + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); @@ -97,10 +97,9 @@ describe('Indicator Match Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); @@ -136,10 +135,9 @@ describe('Indicator Match Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts index e2d5da1def707..c30fdd7d99c2a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts @@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) => { - const { - experimentalFeatures, - lists, - logger, - mergeStrategy, - ignoreFields, - ruleDataClient, - version, - ruleDataService, - } = createOptions; + const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } = + createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, - mergeStrategy, - ignoreFields, + config, ruleDataClient, - ruleDataService, + eventLogService, }); return createSecurityRuleType({ id: INDICATOR_RULE_TYPE_ID, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts index 23cd2e94aedf8..bffc20c3df1e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts @@ -14,6 +14,7 @@ import { createRuleTypeMocks } from '../__mocks__/rule_type'; import { createMlAlertType } from './create_ml_alert_type'; import { RuleParams } from '../../schemas/rule_schemas'; +import { createMockConfig } from '../../routes/__mocks__'; jest.mock('../../signals/bulk_create_ml_signals'); @@ -97,11 +98,10 @@ describe('Machine Learning Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ml: mlMock, ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts index ec2f5dd104646..ac2d3f14831a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts @@ -14,15 +14,13 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createMlAlertType = (createOptions: CreateRuleOptions) => { - const { lists, logger, mergeStrategy, ignoreFields, ml, ruleDataClient, ruleDataService } = - createOptions; + const { lists, logger, config, ml, ruleDataClient, eventLogService } = createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, - mergeStrategy, - ignoreFields, + config, ruleDataClient, - ruleDataService, + eventLogService, }); return createSecurityRuleType({ id: ML_RULE_TYPE_ID, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index e45d8440386fe..4fdeac8047b1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -14,6 +14,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe import { sampleDocNoSortId } from '../../signals/__mocks__/es_results'; import { createQueryAlertType } from './create_query_alert_type'; import { createRuleTypeMocks } from '../__mocks__/rule_type'; +import { createMockConfig } from '../../routes/__mocks__'; jest.mock('../utils/get_list_client', () => ({ getListClient: jest.fn().mockReturnValue({ @@ -31,10 +32,9 @@ describe('Custom Query Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); @@ -79,10 +79,9 @@ describe('Custom Query Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts index d5af7a4c8b5a4..469c237112dcb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts @@ -14,23 +14,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createQueryAlertType = (createOptions: CreateRuleOptions) => { - const { - experimentalFeatures, - lists, - logger, - mergeStrategy, - ignoreFields, - ruleDataClient, - version, - ruleDataService, - } = createOptions; + const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } = + createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, - mergeStrategy, - ignoreFields, + config, ruleDataClient, - ruleDataService, + eventLogService, }); return createSecurityRuleType({ id: QUERY_RULE_TYPE_ID, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts index 74435cb300472..aff57dbdf3cd4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.test.ts @@ -9,6 +9,7 @@ import { allowedExperimentalValues } from '../../../../../common/experimental_fe import { createThresholdAlertType } from './create_threshold_alert_type'; import { createRuleTypeMocks } from '../__mocks__/rule_type'; import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; +import { createMockConfig } from '../../routes/__mocks__'; jest.mock('../../rule_execution_log/rule_execution_log_client'); @@ -20,10 +21,9 @@ describe('Threshold Alerts', () => { experimentalFeatures: allowedExperimentalValues, lists: dependencies.lists, logger: dependencies.logger, - mergeStrategy: 'allFields', - ignoreFields: [], + config: createMockConfig(), ruleDataClient: dependencies.ruleDataClient, - ruleDataService: dependencies.ruleDataService, + eventLogService: dependencies.eventLogService, version: '1.0.0', }); dependencies.alerting.registerType(thresholdAlertTpe); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts index a503cf5aedbea..789e4525c58ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts @@ -16,23 +16,14 @@ import { createSecurityRuleTypeFactory } from '../create_security_rule_type_fact import { CreateRuleOptions } from '../types'; export const createThresholdAlertType = (createOptions: CreateRuleOptions) => { - const { - experimentalFeatures, - lists, - logger, - mergeStrategy, - ignoreFields, - ruleDataClient, - version, - ruleDataService, - } = createOptions; + const { experimentalFeatures, lists, logger, config, ruleDataClient, version, eventLogService } = + createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ lists, logger, - mergeStrategy, - ignoreFields, + config, ruleDataClient, - ruleDataService, + eventLogService, }); return createSecurityRuleType({ id: THRESHOLD_RULE_TYPE_ID, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 6280a50d4981c..c94339da03b93 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -30,12 +30,12 @@ import { import { BaseHit } from '../../../../common/detection_engine/types'; import { ConfigType } from '../../../config'; import { SetupPlugins } from '../../../plugin'; -import { IRuleDataPluginService } from '../rule_execution_log/types'; import { RuleParams } from '../schemas/rule_schemas'; import { BuildRuleMessage } from '../signals/rule_messages'; import { AlertAttributes, BulkCreate, WrapHits, WrapSequences } from '../signals/types'; import { AlertsFieldMap, RulesFieldMap } from './field_maps'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; +import { IEventLogService } from '../../../../../event_log/server'; export interface SecurityAlertTypeReturnValue { bulkCreateTimes: string[]; @@ -98,10 +98,9 @@ type SecurityAlertTypeWithExecutor< export type CreateSecurityRuleTypeFactory = (options: { lists: SetupPlugins['lists']; logger: Logger; - mergeStrategy: ConfigType['alertMergeStrategy']; - ignoreFields: ConfigType['alertIgnoreFields']; + config: ConfigType; ruleDataClient: IRuleDataClient; - ruleDataService: IRuleDataPluginService; + eventLogService: IEventLogService; }) => < TParams extends RuleParams & { index?: string[] | undefined }, TAlertInstanceContext extends AlertInstanceContext, @@ -127,10 +126,9 @@ export interface CreateRuleOptions { experimentalFeatures: ExperimentalFeatures; lists: SetupPlugins['lists']; logger: Logger; - mergeStrategy: ConfigType['alertMergeStrategy']; - ignoreFields: ConfigType['alertIgnoreFields']; + config: ConfigType; ml?: SetupPlugins['ml']; ruleDataClient: IRuleDataClient; version: string; - ruleDataService: IRuleDataPluginService; + eventLogService: IEventLogService; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts index c6a5c00380242..2f3d05e0c9586 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/enable_rule.ts @@ -44,6 +44,8 @@ export const enableRule = async ({ const currentStatusToDisable = ruleCurrentStatus[0]; await ruleStatusClient.update({ id: currentStatusToDisable.id, + ruleName: rule.name, + ruleType: rule.alertTypeId, attributes: { ...currentStatusToDisable.attributes, status: RuleExecutionStatus['going to run'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 850eee3993b60..207ea497c7e8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -33,6 +33,7 @@ export const sampleRuleSO = (params: T): SavedObject { updated_at: '2020-03-27T22:55:59.577Z', attributes: { actions: [], + alertTypeId: 'siem.signals', enabled: true, name: 'rule-name', tags: ['some fake tag 1', 'some fake tag 2'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 5766390099e29..11145405dcc99 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -30,6 +30,7 @@ describe('threshold_executor', () => { updated_at: '2020-03-27T22:55:59.577Z', attributes: { actions: [], + alertTypeId: 'siem.signals', enabled: true, name: 'rule-name', tags: ['some fake tag 1', 'some fake tag 2'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 2696d6981083e..c2923b566175e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -32,10 +32,11 @@ import { mlExecutor } from './executors/ml'; import { getMlRuleParams, getQueryRuleParams } from '../schemas/rule_schemas.mock'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { allowedExperimentalValues } from '../../../../common/experimental_features'; -import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks'; import { scheduleNotificationActions } from '../notifications/schedule_notification_actions'; import { ruleExecutionLogClientMock } from '../rule_execution_log/__mocks__/rule_execution_log_client'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; +import { eventLogServiceMock } from '../../../../../event_log/server/mocks'; +import { createMockConfig } from '../routes/__mocks__'; jest.mock('./utils', () => { const original = jest.requireActual('./utils'); @@ -124,12 +125,12 @@ describe('signal_rule_alert_type', () => { let alert: ReturnType; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleDataService: ReturnType; + let eventLogService: ReturnType; beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleDataService = ruleRegistryMocks.createRuleDataPluginService(); + eventLogService = eventLogServiceMock.create(); (getListsClient as jest.Mock).mockReturnValue({ listClient: getListClientMock(), exceptionsClient: getExceptionListClientMock(), @@ -194,9 +195,8 @@ describe('signal_rule_alert_type', () => { version, ml: mlMock, lists: listMock.createSetup(), - mergeStrategy: 'missingFields', - ignoreFields: [], - ruleDataService, + config: createMockConfig(), + eventLogService, }); mockRuleExecutionLogClient.logStatusChange.mockClear(); @@ -217,11 +217,18 @@ describe('signal_rule_alert_type', () => { payload.previousStartedAt = moment().subtract(100, 'm').toDate(); await alert.executor(payload); expect(logger.warn).toHaveBeenCalled(); - expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenLastCalledWith( + expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + newStatus: RuleExecutionStatus['going to run'], + }) + ); + expect(mockRuleExecutionLogClient.logStatusChange).toHaveBeenNthCalledWith( + 2, expect.objectContaining({ newStatus: RuleExecutionStatus.failed, metrics: { - gap: 'an hour', + executionGap: expect.any(Object), }, }) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 9a6c099ed1760..1e3a8a513c4a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -70,9 +70,9 @@ import { ConfigType } from '../../../config'; import { ExperimentalFeatures } from '../../../../common/experimental_features'; import { injectReferences, extractReferences } from './saved_object_references'; import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log_client'; -import { IRuleDataPluginService } from '../rule_execution_log/types'; import { RuleExecutionStatus } from '../../../../common/detection_engine/schemas/common/schemas'; import { scheduleThrottledNotificationActions } from '../notifications/schedule_throttle_notification_actions'; +import { IEventLogService } from '../../../../../event_log/server'; export const signalRulesAlertType = ({ logger, @@ -81,9 +81,8 @@ export const signalRulesAlertType = ({ version, ml, lists, - mergeStrategy, - ignoreFields, - ruleDataService, + config, + eventLogService, }: { logger: Logger; eventsTelemetry: TelemetryEventsSender | undefined; @@ -91,10 +90,10 @@ export const signalRulesAlertType = ({ version: string; ml: SetupPlugins['ml']; lists: SetupPlugins['lists'] | undefined; - mergeStrategy: ConfigType['alertMergeStrategy']; - ignoreFields: ConfigType['alertIgnoreFields']; - ruleDataService: IRuleDataPluginService; + config: ConfigType; + eventLogService: IEventLogService; }): SignalRuleAlertTypeDefinition => { + const { alertMergeStrategy: mergeStrategy, alertIgnoreFields: ignoreFields } = config; return { id: SIGNALS_ID, name: 'SIEM signal', @@ -138,14 +137,16 @@ export const signalRulesAlertType = ({ let hasError: boolean = false; let result = createSearchAfterReturnType(); const ruleStatusClient = new RuleExecutionLogClient({ - ruleDataService, + eventLogService, savedObjectsClient: services.savedObjectsClient, + underlyingClient: config.ruleExecutionLog.underlyingClient, }); const savedObject = await services.savedObjectsClient.get('alert', alertId); const { actions, name, + alertTypeId, schedule: { interval }, } = savedObject.attributes; const refresh = actions.length ? 'wait_for' : false; @@ -159,10 +160,16 @@ export const signalRulesAlertType = ({ logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); logger.debug(buildRuleMessage(`interval: ${interval}`)); let wroteWarningStatus = false; - await ruleStatusClient.logStatusChange({ + const basicLogArguments = { + spaceId, ruleId: alertId, + ruleName: name, + ruleType: alertTypeId, + }; + + await ruleStatusClient.logStatusChange({ + ...basicLogArguments, newStatus: RuleExecutionStatus['going to run'], - spaceId, }); // check if rule has permissions to access given index pattern @@ -194,8 +201,7 @@ export const signalRulesAlertType = ({ tryCatch( () => hasReadIndexPrivileges({ - spaceId, - ruleId: alertId, + ...basicLogArguments, privileges, logger, buildRuleMessage, @@ -207,13 +213,11 @@ export const signalRulesAlertType = ({ tryCatch( () => hasTimestampFields({ - spaceId, - ruleId: alertId, + ...basicLogArguments, wroteStatus: wroteStatus as boolean, timestampField: hasTimestampOverride ? (timestampOverride as string) : '@timestamp', - ruleName: name, timestampFieldCapsResponse: timestampFieldCaps, inputIndices, ruleStatusClient, @@ -247,11 +251,10 @@ export const signalRulesAlertType = ({ logger.warn(gapMessage); hasError = true; await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message: gapMessage, - metrics: { gap: gapString }, + metrics: { executionGap: remainingGap }, }); } try { @@ -383,8 +386,7 @@ export const signalRulesAlertType = ({ if (result.warningMessages.length) { const warningMessage = buildRuleMessage(result.warningMessages.join()); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus['partial failure'], message: warningMessage, }); @@ -445,13 +447,12 @@ export const signalRulesAlertType = ({ ); if (!hasError && !wroteWarningStatus && !result.warning) { await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.succeeded, message: 'succeeded', metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookBackDate?.toISOString(), }, }); @@ -474,13 +475,12 @@ export const signalRulesAlertType = ({ ); logger.error(errorMessage); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message: errorMessage, metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookBackDate?.toISOString(), }, }); @@ -494,13 +494,12 @@ export const signalRulesAlertType = ({ logger.error(message); await ruleStatusClient.logStatusChange({ - spaceId, - ruleId: alertId, + ...basicLogArguments, newStatus: RuleExecutionStatus.failed, message, metrics: { - bulkCreateTimeDurations: result.bulkCreateTimes, - searchAfterTimeDurations: result.searchAfterTimes, + indexingDurations: result.bulkCreateTimes, + searchDurations: result.searchAfterTimes, lastLookBackDate: result.lastLookBackDate?.toISOString(), }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index c1e7e23c3b161..82b4a46f482b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -255,6 +255,7 @@ export interface SignalHit { export interface AlertAttributes { actions: RuleAlertAction[]; + alertTypeId: string; enabled: boolean; name: string; tags: string[]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index c3e95d6d196ca..7d2eafa46d382 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -789,6 +789,7 @@ describe('utils', () => { inputIndices: ['myfa*'], ruleStatusClient, ruleId: 'ruleId', + ruleType: 'ruleType', spaceId: 'default', logger: mockLogger, buildRuleMessage, @@ -832,6 +833,7 @@ describe('utils', () => { inputIndices: ['myfa*'], ruleStatusClient, ruleId: 'ruleId', + ruleType: 'ruleType', spaceId: 'default', logger: mockLogger, buildRuleMessage, @@ -861,6 +863,7 @@ describe('utils', () => { inputIndices: ['logs-endpoint.alerts-*'], ruleStatusClient, ruleId: 'ruleId', + ruleType: 'ruleType', spaceId: 'default', logger: mockLogger, buildRuleMessage, @@ -890,6 +893,7 @@ describe('utils', () => { inputIndices: ['logs-endpoint.alerts-*'], ruleStatusClient, ruleId: 'ruleId', + ruleType: 'ruleType', spaceId: 'default', logger: mockLogger, buildRuleMessage, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 5993dd626729f..2aefc7ea0bd64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -99,9 +99,20 @@ export const hasReadIndexPrivileges = async (args: { buildRuleMessage: BuildRuleMessage; ruleStatusClient: IRuleExecutionLogClient; ruleId: string; + ruleName: string; + ruleType: string; spaceId: string; }): Promise => { - const { privileges, logger, buildRuleMessage, ruleStatusClient, ruleId, spaceId } = args; + const { + privileges, + logger, + buildRuleMessage, + ruleStatusClient, + ruleId, + ruleName, + ruleType, + spaceId, + } = args; const indexNames = Object.keys(privileges.index); const [indexesWithReadPrivileges, indexesWithNoReadPrivileges] = partition( @@ -119,6 +130,8 @@ export const hasReadIndexPrivileges = async (args: { await ruleStatusClient.logStatusChange({ message: errorString, ruleId, + ruleName, + ruleType, spaceId, newStatus: RuleExecutionStatus['partial failure'], }); @@ -136,6 +149,8 @@ export const hasReadIndexPrivileges = async (args: { await ruleStatusClient.logStatusChange({ message: errorString, ruleId, + ruleName, + ruleType, spaceId, newStatus: RuleExecutionStatus['partial failure'], }); @@ -156,6 +171,7 @@ export const hasTimestampFields = async (args: { ruleStatusClient: IRuleExecutionLogClient; ruleId: string; spaceId: string; + ruleType: string; logger: Logger; buildRuleMessage: BuildRuleMessage; }): Promise => { @@ -167,6 +183,7 @@ export const hasTimestampFields = async (args: { inputIndices, ruleStatusClient, ruleId, + ruleType, spaceId, logger, buildRuleMessage, @@ -184,6 +201,8 @@ export const hasTimestampFields = async (args: { await ruleStatusClient.logStatusChange({ message: errorString.trimEnd(), ruleId, + ruleName, + ruleType, spaceId, newStatus: RuleExecutionStatus['partial failure'], }); @@ -210,6 +229,8 @@ export const hasTimestampFields = async (args: { await ruleStatusClient.logStatusChange({ message: errorString, ruleId, + ruleName, + ruleType, spaceId, newStatus: RuleExecutionStatus['partial failure'], }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 9c4d739e0f434..bffcc823d047e 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -109,11 +109,14 @@ import { ctiFieldMap } from './lib/detection_engine/rule_types/field_maps/cti'; import { legacyRulesNotificationAlertType } from './lib/detection_engine/notifications/legacy_rules_notification_alert_type'; // eslint-disable-next-line no-restricted-imports import { legacyIsNotificationAlertExecutor } from './lib/detection_engine/notifications/legacy_types'; +import { IEventLogClientService, IEventLogService } from '../../event_log/server'; +import { registerEventLogProvider } from './lib/detection_engine/rule_execution_log/event_log_adapter/register_event_log_provider'; export interface SetupPlugins { alerting: AlertingSetup; data: DataPluginSetup; encryptedSavedObjects?: EncryptedSavedObjectsSetup; + eventLog: IEventLogService; features: FeaturesSetup; lists?: ListPluginSetup; ml?: MlSetup; @@ -121,20 +124,21 @@ export interface SetupPlugins { security?: SecuritySetup; spaces?: SpacesSetup; taskManager?: TaskManagerSetupContract; - usageCollection?: UsageCollectionSetup; telemetry?: TelemetryPluginSetup; + usageCollection?: UsageCollectionSetup; } export interface StartPlugins { alerting: AlertPluginStartContract; + cases?: CasesPluginStartContract; data: DataPluginStart; + eventLog: IEventLogClientService; fleet?: FleetStartContract; licensing: LicensingPluginStart; ruleRegistry: RuleRegistryPluginStartContract; + security: SecurityPluginStart; taskManager?: TaskManagerStartContract; telemetry?: TelemetryPluginStart; - security: SecurityPluginStart; - cases?: CasesPluginStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -202,6 +206,9 @@ export class Plugin implements IPlugin(); core.http.registerRouteHandlerContext( APP_ID, @@ -210,8 +217,9 @@ export class Plugin implements IPlugin plugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID, getExecutionLogClient: () => new RuleExecutionLogClient({ - ruleDataService: plugins.ruleRegistry.ruleDataService, savedObjectsClient: context.core.savedObjects.client, + eventLogService, + underlyingClient: config.ruleExecutionLog.underlyingClient, }), }) ); @@ -262,11 +270,10 @@ export class Plugin implements IPlugin