Implement writing rule execution events to event_log (#112286)

x-pack/plugins/event_log/generated/mappings.json

@@ -267,6 +267,42 @@
                         }
                     }
                 },
+                "alert": {
+                    "properties": {
+                        "rule": {
+                            "properties": {
+                                "execution": {
+                                    "properties": {
+                                        "uuid": {
+                                            "type": "keyword",
+                                            "ignore_above": 1024
+                                        },
+                                        "status": {
+                                            "type": "keyword",
+                                            "ignore_above": 1024
+                                        },
+                                        "status_order": {
+                                            "type": "long"
+                                        },
+                                        "metrics": {
+                                            "properties": {
+                                                "total_indexing_duration_ms": {
+                                                    "type": "long"
+                                                },
+                                                "total_search_duration_ms": {
+                                                    "type": "long"
+                                                },
+                                                "execution_gap_duration_s": {
+                                                    "type": "long"
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
                 "saved_objects": {
                     "type": "nested",
                     "properties": {
@@ -292,6 +328,13 @@
                         }
                     }
                 },
+                "space_ids": {
+                    "type": "keyword",
+                    "ignore_above": 1024,
+                    "meta": {
+                        "isArray": "true"
+                    }
+                },
                 "version": {
                     "type": "version"
                 }

x-pack/plugins/event_log/generated/schemas.ts

@@ -116,6 +116,28 @@ export const EventSchema = schema.maybe(
             status: ecsString(),
           })
         ),
+        alert: schema.maybe(
+          schema.object({
+            rule: schema.maybe(
+              schema.object({
+                execution: schema.maybe(
+                  schema.object({
+                    uuid: ecsString(),
+                    status: ecsString(),
+                    status_order: ecsNumber(),
+                    metrics: schema.maybe(
+                      schema.object({
+                        total_indexing_duration_ms: ecsNumber(),
+                        total_search_duration_ms: ecsNumber(),
+                        execution_gap_duration_s: ecsNumber(),
+                      })
+                    ),
+                  })
+                ),
+              })
+            ),
+          })
+        ),
         saved_objects: schema.maybe(
           schema.arrayOf(
             schema.object({
@@ -127,6 +149,7 @@ export const EventSchema = schema.maybe(
             })
           )
         ),
+        space_ids: ecsStringMulti(),
         version: ecsVersion(),
       })
     ),

x-pack/plugins/event_log/scripts/mappings.js

@@ -49,6 +49,42 @@ exports.EcsCustomPropertyMappings = {
           },
         },
       },
+      alert: {
+        properties: {
+          rule: {
+            properties: {
+              execution: {
+                properties: {
+                  uuid: {
+                    type: 'keyword',
+                    ignore_above: 1024,
+                  },
+                  status: {
+                    type: 'keyword',
+                    ignore_above: 1024,
+                  },
+                  status_order: {
+                    type: 'long',
+                  },
+                  metrics: {
+                    properties: {
+                      total_indexing_duration_ms: {
+                        type: 'long',
+                      },
+                      total_search_duration_ms: {
+                        type: 'long',
+                      },
+                      execution_gap_duration_s: {
+                        type: 'long',
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
       // array of saved object references, for "linking" via search
       saved_objects: {
         type: 'nested',
@@ -77,6 +113,10 @@ exports.EcsCustomPropertyMappings = {
           },
         },
       },
+      space_ids: {
+        type: 'keyword',
+        ignore_above: 1024,
+      },
       version: {
         type: 'version',
       },
@@ -105,4 +145,10 @@ exports.EcsPropertiesToGenerate = [
 /**
  * These properties can have multiple values (are arrays in the generated event schema).
  */
-exports.EcsEventLogMultiValuedProperties = ['tags', 'event.category', 'event.type', 'rule.author'];
+exports.EcsEventLogMultiValuedProperties = [
+  'tags',
+  'event.category',
+  'event.type',
+  'rule.author',
+  'kibana.space_ids',
+];

x-pack/plugins/security_solution/kibana.json

@@ -12,15 +12,16 @@
     "actions",
     "alerting",
     "cases",
-    "ruleRegistry",
     "data",
     "dataEnhanced",
     "embeddable",
+    "eventLog",
     "features",
-    "taskManager",
     "inspector",
     "licensing",
     "maps",
+    "ruleRegistry",
+    "taskManager",
     "timelines",
     "triggersActionsUi",
     "uiActions"

x-pack/plugins/security_solution/server/config.ts

@@ -12,6 +12,7 @@ import {
   getExperimentalAllowedValues,
   isValidExperimentalValue,
 } from '../common/experimental_features';
+import { UnderlyingLogClient } from './lib/detection_engine/rule_execution_log/types';
 
 const allowedExperimentalValues = getExperimentalAllowedValues();
 
@@ -103,6 +104,19 @@ export const configSchema = schema.object({
     },
   }),
 
+  /**
+   * Rule Execution Log Configuration
+   */
+  ruleExecutionLog: schema.object({
+    underlyingClient: schema.oneOf(
+      [
+        schema.literal(UnderlyingLogClient.eventLog),
+        schema.literal(UnderlyingLogClient.savedObjects),
+      ],
+      { defaultValue: UnderlyingLogClient.savedObjects }
+    ),
+  }),
+
   /**
    * Host Endpoint Configuration
    */

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts

@@ -11,6 +11,7 @@ import { serverMock } from './server';
 import { requestMock } from './request';
 import { responseMock } from './response_factory';
 import { ConfigType } from '../../../../config';
+import { UnderlyingLogClient } from '../../rule_execution_log/types';
 
 export { requestMock, requestContextMock, responseMock, serverMock };
 
@@ -29,6 +30,9 @@ export const createMockConfig = (): ConfigType => ({
   alertIgnoreFields: [],
   prebuiltRulesFromFileSystem: true,
   prebuiltRulesFromSavedObjects: false,
+  ruleExecutionLog: {
+    underlyingClient: UnderlyingLogClient.savedObjects,
+  },
 });
 
 export const mockGetCurrentUser = {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/__mocks__/rule_execution_log_client.ts

@@ -14,7 +14,7 @@ export const ruleExecutionLogClientMock = {
     update: jest.fn(),
     delete: jest.fn(),
     logStatusChange: jest.fn(),
-    logExecutionMetric: jest.fn(),
+    logExecutionMetrics: jest.fn(),
   }),
 };
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/constants.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const RULE_EXECUTION_LOG_PROVIDER = 'rule-execution.security';
+
+export const ALERT_SAVED_OBJECT_TYPE = 'alert';
+
+export enum RuleExecutionLogAction {
+  'status-change' = 'status-change',
+  'execution-metrics' = 'execution-metrics',
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/event_log_adapter/event_log_adapter.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IEventLogService } from '../../../../../../event_log/server';
+import {
+  FindBulkExecutionLogArgs,
+  FindExecutionLogArgs,
+  IRuleExecutionLogClient,
+  LogExecutionMetricsArgs,
+  LogStatusChangeArgs,
+  UpdateExecutionLogArgs,
+} from '../types';
+import { EventLogClient } from './event_log_client';
+
+export class EventLogAdapter implements IRuleExecutionLogClient {
+  private eventLogClient: EventLogClient;
+
+  constructor(eventLogService: IEventLogService) {
+    this.eventLogClient = new EventLogClient(eventLogService);
+  }
+
+  public async find({ ruleId, logsCount = 1, spaceId }: FindExecutionLogArgs) {
+    return []; // TODO Implement
+  }
+
+  public async findBulk({ ruleIds, logsCount = 1, spaceId }: FindBulkExecutionLogArgs) {
+    return {}; // TODO Implement
+  }
+
+  public async update({ attributes, spaceId, ruleName, ruleType }: UpdateExecutionLogArgs) {
+    // execution events are immutable, so we just log a status change istead of updating previous
+    if (attributes.status) {
+      this.eventLogClient.logStatusChange({
+        ruleName,
+        ruleType,
+        ruleId: attributes.alertId,
+        newStatus: attributes.status,
+        spaceId,
+      });
+    }
+  }
+
+  public async delete(id: string) {
+    // execution events are immutable, nothing to do here
+  }
+
+  public async logExecutionMetrics({
+    ruleId,
+    spaceId,
+    ruleType,
+    ruleName,
+    metrics,
+  }: LogExecutionMetricsArgs) {
+    this.eventLogClient.logExecutionMetrics({
+      ruleId,
+      ruleName,
+      ruleType,
+      spaceId,
+      metrics: {
+        executionGapDuration: metrics.executionGap?.asSeconds(),
+        totalIndexingDuration: metrics.indexingDurations?.reduce(
+          (acc, cur) => acc + Number(cur),
+          0
+        ),
+        totalSearchDuration: metrics.searchDurations?.reduce((acc, cur) => acc + Number(cur), 0),
+      },
+    });
+  }
+
+  public async logStatusChange(args: LogStatusChangeArgs) {
+    if (args.metrics) {
+      this.logExecutionMetrics({
+        ruleId: args.ruleId,
+        ruleName: args.ruleName,
+        ruleType: args.ruleType,
+        spaceId: args.spaceId,
+        metrics: args.metrics,
+      });
+    }
+
+    this.eventLogClient.logStatusChange(args);
+  }
+}