[Security Solution] Improve rule execution logging (#166070)

**Relates to:** #126063 ## Summary This PR extends rule event log's filter and improves log messages. ## Details We have Rule execution log feature hidden by a feature flag and disabled, it's shown on a rule details page when enabled. <img width="1782" alt="image" src="https://github.com/elastic/kibana/assets/3775283/71565d96-13aa-4275-b870-22118ac90335"> The feature is close to a releasable state but some tasks had to be addressed to make it usable. This PR addresses the following tasks to make rule execution log feature releasable - Adds search bar to search for specific messages <img width="1529" alt="image" src="https://github.com/elastic/kibana/assets/3775283/4bd198de-60e8-4511-a96d-4d68ec53a7f2"> - Adds a date range filter by default set to show logs for last 24 hours <img width="1529" alt="image" src="https://github.com/elastic/kibana/assets/3775283/b9d7e658-a19a-402a-a039-28d225000952"> - Improves error, warning and debug messages - Returns rule metrics in a message as a serialized JSON <img width="1526" alt="image" src="https://github.com/elastic/kibana/assets/3775283/7d9501b9-4a12-4d31-be99-6ce3c04b2b97"> - Adds `execution_id` to the response <img width="1522" alt="image" src="https://github.com/elastic/kibana/assets/3775283/92d1291e-0605-456c-abca-8c6fd329ade2"> ### Tasks to address later - [ ] Further improve logging messages. We have either error, warning or debug messages. In fact info or trace levels aren't used but it can give useful info. - [ ] Add an OpenAPI spec for the rule execution log endpoint
elastic · Sep 19, 2023 · 6a96906 · 6a96906
1 parent a7e196d
commit 6a96906
Show file tree

Hide file tree

Showing 35 changed files with 398 additions and 132 deletions.
diff --git a/...curity_solution/common/api/detection_engine/rule_monitoring/model/execution_event.mock.ts b/...curity_solution/common/api/detection_engine/rule_monitoring/model/execution_event.mock.ts
@@ -18,8 +18,9 @@ const getMessageEvent = (props: Partial<RuleExecutionEvent> = {}): RuleExecution
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
     level: LogLevel.debug,
+    execution_id: 'execution-id-1',
     message: 'Some message',
-    // Overriden values
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     type: RuleExecutionEventType.message,
@@ -31,8 +32,9 @@ const getRunningStatusChange = (props: Partial<RuleExecutionEvent> = {}): RuleEx
     // Default values
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
+    execution_id: 'execution-id-1',
     message: 'Rule changed status to "running"',
-    // Overriden values
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     level: LogLevel.info,
@@ -47,8 +49,9 @@ const getPartialFailureStatusChange = (
     // Default values
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
+    execution_id: 'execution-id-1',
     message: 'Rule changed status to "partial failure". Unknown error',
-    // Overriden values
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     level: LogLevel.warn,
@@ -61,8 +64,9 @@ const getFailedStatusChange = (props: Partial<RuleExecutionEvent> = {}): RuleExe
     // Default values
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
+    execution_id: 'execution-id-1',
     message: 'Rule changed status to "failed". Unknown error',
-    // Overriden values
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     level: LogLevel.error,
@@ -75,8 +79,9 @@ const getSucceededStatusChange = (props: Partial<RuleExecutionEvent> = {}): Rule
     // Default values
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
+    execution_id: 'execution-id-1',
     message: 'Rule changed status to "succeeded". Rule executed successfully',
-    // Overriden values
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     level: LogLevel.info,
@@ -89,8 +94,9 @@ const getExecutionMetricsEvent = (props: Partial<RuleExecutionEvent> = {}): Rule
     // Default values
     timestamp: DEFAULT_TIMESTAMP,
     sequence: DEFAULT_SEQUENCE_NUMBER,
-    message: '',
-    // Overriden values
+    execution_id: 'execution-id-1',
+    message: JSON.stringify({ some_metric_ms: 10 }),
+    // Overridden values
     ...props,
     // Mandatory values for this type of event
     level: LogLevel.debug,

diff --git a/...ns/security_solution/common/api/detection_engine/rule_monitoring/model/execution_event.ts b/...ns/security_solution/common/api/detection_engine/rule_monitoring/model/execution_event.ts
@@ -6,7 +6,7 @@
  */
 
 import * as t from 'io-ts';
-import { enumeration, IsoDateString } from '@kbn/securitysolution-io-ts-types';
+import { enumeration, IsoDateString, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
 import { enumFromString } from '../../../../utils/enum_from_string';
 import { TLogLevel } from './log_level';
 
@@ -56,5 +56,6 @@ export const RuleExecutionEvent = t.type({
   sequence: t.number,
   level: TLogLevel,
   type: TRuleExecutionEventType,
+  execution_id: NonEmptyString,
   message: t.string,
 });
diff --git a/...nitoring/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts b/...nitoring/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts
@@ -8,7 +8,7 @@
 import * as t from 'io-ts';
 
 import { DefaultPerPage, DefaultPage } from '@kbn/securitysolution-io-ts-alerting-types';
-import { defaultCsvArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
+import { defaultCsvArray, IsoDateString, NonEmptyString } from '@kbn/securitysolution-io-ts-types';
 
 import { DefaultSortOrderDesc, PaginationResult } from '../../../model';
 import { RuleExecutionEvent, TRuleExecutionEventType, TLogLevel } from '../../model';
@@ -32,13 +32,20 @@ export type GetRuleExecutionEventsRequestQuery = t.TypeOf<
   typeof GetRuleExecutionEventsRequestQuery
 >;
 export const GetRuleExecutionEventsRequestQuery = t.exact(
-  t.type({
-    event_types: defaultCsvArray(TRuleExecutionEventType),
-    log_levels: defaultCsvArray(TLogLevel),
-    sort_order: DefaultSortOrderDesc, // defaults to 'desc'
-    page: DefaultPage, // defaults to 1
-    per_page: DefaultPerPage, // defaults to 20
-  })
+  t.intersection([
+    t.partial({
+      search_term: NonEmptyString,
+      event_types: defaultCsvArray(TRuleExecutionEventType),
+      log_levels: defaultCsvArray(TLogLevel),
+      date_start: IsoDateString,
+      date_end: IsoDateString,
+    }),
+    t.type({
+      sort_order: DefaultSortOrderDesc, // defaults to 'desc'
+      page: DefaultPage, // defaults to 1
+      per_page: DefaultPerPage, // defaults to 20
+    }),
+  ])
 );
 
 /**

diff --git a/...ins/security_solution/public/detection_engine/rule_monitoring/api/__mocks__/api_client.ts b/...ins/security_solution/public/detection_engine/rule_monitoring/api/__mocks__/api_client.ts
@@ -32,6 +32,7 @@ export const api: jest.Mocked<IRuleMonitoringApiClient> = {
           sequence: 0,
           level: LogLevel.info,
           type: RuleExecutionEventType['status-change'],
+          execution_id: 'execution-id-1',
           message: 'Rule changed status to "succeeded". Rule execution completed without errors',
         },
       ],

diff --git a/.../plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.test.ts b/.../plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.test.ts
@@ -72,26 +72,59 @@ describe('Rule Monitoring API Client', () => {
       );
     });
 
-    it('calls API correctly with filter and pagination options', async () => {
+    const ISO_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
+
+    it.each([
+      [
+        'search term filter',
+        { searchTerm: 'something to search' },
+        { search_term: 'something to search' },
+      ],
+      [
+        'event types filter',
+        { eventTypes: [RuleExecutionEventType.message] },
+        { event_types: 'message' },
+      ],
+      [
+        'log level filter',
+        { logLevels: [LogLevel.warn, LogLevel.error] },
+        { log_levels: 'warn,error' },
+      ],
+      [
+        'start date filter in relative format',
+        { dateRange: { start: 'now-1d/d' } },
+        { date_start: expect.stringMatching(ISO_PATTERN) },
+      ],
+      [
+        'end date filter',
+        { dateRange: { end: 'now-3d/d' } },
+        { date_end: expect.stringMatching(ISO_PATTERN) },
+      ],
+      [
+        'date range filter in relative format',
+        { dateRange: { start: new Date().toISOString(), end: new Date().toISOString() } },
+        {
+          date_start: expect.stringMatching(ISO_PATTERN),
+          date_end: expect.stringMatching(ISO_PATTERN),
+        },
+      ],
+      [
+        'pagination',
+        { sortOrder: 'asc', page: 42, perPage: 146 } as const,
+        { sort_order: 'asc', page: 42, per_page: 146 },
+      ],
+    ])('calls API correctly with %s', async (_, params, expectedParams) => {
       await api.fetchRuleExecutionEvents({
         ruleId: '42',
-        eventTypes: [RuleExecutionEventType.message],
-        logLevels: [LogLevel.warn, LogLevel.error],
-        sortOrder: 'asc',
-        page: 42,
-        perPage: 146,
+        ...params,
       });
 
       expect(fetchMock).toHaveBeenCalledWith(
         '/internal/detection_engine/rules/42/execution/events',
         expect.objectContaining({
           method: 'GET',
           query: {
-            event_types: 'message',
-            log_levels: 'warn,error',
-            sort_order: 'asc',
-            page: 42,
-            per_page: 146,
+            ...expectedParams,
           },
         })
       );

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { omitBy, isUndefined } from 'lodash';
 import dateMath from '@kbn/datemath';
 
 import { KibanaServices } from '../../../common/lib/kibana';
@@ -36,20 +37,38 @@ export const api: IRuleMonitoringApiClient = {
   fetchRuleExecutionEvents: (
     args: FetchRuleExecutionEventsArgs
   ): Promise<GetRuleExecutionEventsResponse> => {
-    const { ruleId, eventTypes, logLevels, sortOrder, page, perPage, signal } = args;
+    const {
+      ruleId,
+      searchTerm,
+      eventTypes,
+      logLevels,
+      dateRange,
+      sortOrder,
+      page,
+      perPage,
+      signal,
+    } = args;
 
     const url = getRuleExecutionEventsUrl(ruleId);
+    const startDate = dateMath.parse(dateRange?.start ?? '')?.toISOString();
+    const endDate = dateMath.parse(dateRange?.end ?? '', { roundUp: true })?.toISOString();
 
     return http().fetch<GetRuleExecutionEventsResponse>(url, {
       method: 'GET',
       version: '1',
-      query: {
-        event_types: eventTypes?.join(','),
-        log_levels: logLevels?.join(','),
-        sort_order: sortOrder,
-        page,
-        per_page: perPage,
-      },
+      query: omitBy(
+        {
+          search_term: searchTerm?.length ? searchTerm : undefined,
+          event_types: eventTypes?.length ? eventTypes.join(',') : undefined,
+          log_levels: logLevels?.length ? logLevels.join(',') : undefined,
+          date_start: startDate,
+          date_end: endDate,
+          sort_order: sortOrder,
+          page,
+          per_page: perPage,
+        },
+        isUndefined
+      ),
       signal,
     });
   },

diff --git a/...ins/security_solution/public/detection_engine/rule_monitoring/api/api_client_interface.ts b/...ins/security_solution/public/detection_engine/rule_monitoring/api/api_client_interface.ts
@@ -46,12 +46,22 @@ export interface RuleMonitoringApiCallArgs {
   signal?: AbortSignal;
 }
 
+export interface DateRange {
+  start?: string;
+  end?: string;
+}
+
 export interface FetchRuleExecutionEventsArgs extends RuleMonitoringApiCallArgs {
   /**
    * Saved Object ID of the rule (`rule.id`, not static `rule.rule_id`).
    */
   ruleId: string;
 
+  /**
+   * Filter by event message. If set, result will include events matching the search term.
+   */
+  searchTerm?: string;
+
   /**
    * Filter by event type. If set, result will include only events matching any of these.
    */
@@ -62,6 +72,11 @@ export interface FetchRuleExecutionEventsArgs extends RuleMonitoringApiCallArgs
    */
   logLevels?: LogLevel[];
 
+  /**
+   * Filter by date range. If set, result will include only events recorded in the specified date range.
+   */
+  dateRange?: DateRange;
+
   /**
    * What order to sort by (e.g. `asc` or `desc`).
    */

diff --git a/...onitoring/components/execution_events_table/event_message_filter/event_message_filter.tsx b/...onitoring/components/execution_events_table/event_message_filter/event_message_filter.tsx
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ChangeEvent } from 'react';
+import React, { useCallback } from 'react';
+import { EuiFieldSearch } from '@elastic/eui';
+import * as i18n from './translations';
+
+interface EventMessageFilterProps {
+  value: string;
+  onChange: (value: string) => void;
+}
+
+export function EventMessageFilter({ value, onChange }: EventMessageFilterProps): JSX.Element {
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLInputElement>) => onChange(e.target.value),
+    [onChange]
+  );
+
+  return (
+    <EuiFieldSearch
+      aria-label={i18n.SEARCH_BY_EVENT_MESSAGE_ARIA_LABEL}
+      fullWidth
+      incremental={false}
+      placeholder={i18n.SEARCH_BY_EVENT_MESSAGE_PLACEHOLDER}
+      value={value}
+      onChange={handleChange}
+      data-test-subj="ruleEventLogMessageSearchField"
+    />
+  );
+}
diff --git a/...on_engine/rule_monitoring/components/execution_events_table/event_message_filter/index.ts b/...on_engine/rule_monitoring/components/execution_events_table/event_message_filter/index.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './event_message_filter';
diff --git a/...ne/rule_monitoring/components/execution_events_table/event_message_filter/translations.ts b/...ne/rule_monitoring/components/execution_events_table/event_message_filter/translations.ts
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const SEARCH_BY_EVENT_MESSAGE_ARIA_LABEL = i18n.translate(
+  'xpack.securitySolution.detectionEngine.rules.eventLog.searchAriaLabel',
+  {
+    defaultMessage: 'Search by event message',
+  }
+);
+
+export const SEARCH_BY_EVENT_MESSAGE_PLACEHOLDER = i18n.translate(
+  'xpack.securitySolution.detectionEngine.rules.eventLog.searchPlaceholder',
+  {
+    defaultMessage: 'Search by event message',
+  }
+);