Fix bug: query ONLY detection rules

elastic · Oct 3, 2023 · 93a67fb · 93a67fb
1 parent e666f07
commit 93a67fb
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 26 deletions.
diff --git a/...e_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts b/...e_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts
@@ -26,23 +26,33 @@ import {
 import { DEFAULT_PERCENTILES } from '../../../utils/es_aggregations';
 import type { RawData } from '../../../utils/normalization';
 import * as f from '../../../event_log/event_log_fields';
+import {
+  ALERTING_PROVIDER,
+  RULE_EXECUTION_LOG_PROVIDER,
+} from '../../../event_log/event_log_constants';
 
 export type RuleExecutionStatsAggregationLevel = 'whole-interval' | 'histogram';
 
 export const getRuleExecutionStatsAggregation = (
-  aggregationContext: RuleExecutionStatsAggregationLevel
+  aggregationLevel: RuleExecutionStatsAggregationLevel
 ): Record<string, estypes.AggregationsAggregationContainer> => {
   return {
-    totalExecutions: {
-      cardinality: {
-        field: f.RULE_EXECUTION_UUID,
-      },
-    },
     executeEvents: {
       filter: {
-        term: { [f.EVENT_ACTION]: 'execute' },
+        bool: {
+          filter: [
+            { term: { [f.EVENT_PROVIDER]: ALERTING_PROVIDER } },
+            { term: { [f.EVENT_ACTION]: 'execute' } },
+            { term: { [f.EVENT_CATEGORY]: 'siem' } },
+          ],
+        },
       },
       aggs: {
+        totalExecutions: {
+          cardinality: {
+            field: f.RULE_EXECUTION_UUID,
+          },
+        },
         executionDurationMs: {
           percentiles: {
             field: f.RULE_EXECUTION_TOTAL_DURATION_MS,
@@ -63,11 +73,8 @@ export const getRuleExecutionStatsAggregation = (
       filter: {
         bool: {
           filter: [
-            {
-              term: {
-                [f.EVENT_ACTION]: RuleExecutionEventType['status-change'],
-              },
-            },
+            { term: { [f.EVENT_PROVIDER]: RULE_EXECUTION_LOG_PROVIDER } },
+            { term: { [f.EVENT_ACTION]: RuleExecutionEventType['status-change'] } },
           ],
           must_not: [
             {
@@ -91,7 +98,12 @@ export const getRuleExecutionStatsAggregation = (
     },
     executionMetricsEvents: {
       filter: {
-        term: { [f.EVENT_ACTION]: RuleExecutionEventType['execution-metrics'] },
+        bool: {
+          filter: [
+            { term: { [f.EVENT_PROVIDER]: RULE_EXECUTION_LOG_PROVIDER } },
+            { term: { [f.EVENT_ACTION]: RuleExecutionEventType['execution-metrics'] } },
+          ],
+        },
       },
       aggs: {
         gaps: {
@@ -126,10 +138,17 @@ export const getRuleExecutionStatsAggregation = (
     },
     messageContainingEvents: {
       filter: {
-        terms: {
-          [f.EVENT_ACTION]: [
-            RuleExecutionEventType['status-change'],
-            RuleExecutionEventType.message,
+        bool: {
+          filter: [
+            { term: { [f.EVENT_PROVIDER]: RULE_EXECUTION_LOG_PROVIDER } },
+            {
+              terms: {
+                [f.EVENT_ACTION]: [
+                  RuleExecutionEventType['status-change'],
+                  RuleExecutionEventType.message,
+                ],
+              },
+            },
           ],
         },
       },
@@ -139,7 +158,7 @@ export const getRuleExecutionStatsAggregation = (
             field: f.LOG_LEVEL,
           },
         },
-        ...(aggregationContext === 'whole-interval'
+        ...(aggregationLevel === 'whole-interval'
           ? {
               errors: {
                 filter: {
@@ -180,12 +199,12 @@ export const normalizeRuleExecutionStatsAggregationResult = (
   aggregations: Record<string, RawData>,
   aggregationLevel: RuleExecutionStatsAggregationLevel
 ): HealthOverviewStats => {
-  const totalExecutions = aggregations.totalExecutions || {};
   const executeEvents = aggregations.executeEvents || {};
   const statusChangeEvents = aggregations.statusChangeEvents || {};
   const executionMetricsEvents = aggregations.executionMetricsEvents || {};
   const messageContainingEvents = aggregations.messageContainingEvents || {};
 
+  const totalExecutions = executeEvents.totalExecutions || {};
   const executionDurationMs = executeEvents.executionDurationMs || {};
   const scheduleDelayNs = executeEvents.scheduleDelayNs || {};
   const executionsByStatus = statusChangeEvents.executionsByStatus || {};

diff --git a/...engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts b/...engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts
@@ -65,20 +65,21 @@ export const createEventLogHealthClient = (
   ruleSpacesClient: IRuleSpacesClient,
   logger: Logger
 ): IEventLogHealthClient => {
+  const EVENT_PROVIDERS = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER];
+  const EVENT_PROVIDERS_FILTER = `${f.EVENT_PROVIDER}: (${kqlOr(EVENT_PROVIDERS)})`;
+
   async function aggregateEventsForRules(
     ruleIds: string[],
     interval: HealthInterval,
     aggs: Record<string, estypes.AggregationsAggregationContainer>
   ) {
     const soType = RULE_SAVED_OBJECT_TYPE;
     const soIds = ruleIds;
-    const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER];
-    const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`;
 
     const result = await eventLog.aggregateEventsBySavedObjectIds(soType, soIds, {
       start: interval.from,
       end: interval.to,
-      filter: kqlFilter,
+      filter: EVENT_PROVIDERS_FILTER,
       aggs,
     });
 
@@ -92,8 +93,6 @@ export const createEventLogHealthClient = (
   ) {
     const soType = RULE_SAVED_OBJECT_TYPE;
     const authFilter = {} as KueryNode;
-    const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER];
-    const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`;
 
     // The `aggregateEventsWithAuthFilter` method accepts "namespace ids" instead of "space ids".
     // If you have two Kibana spaces with ids ['default', 'space-x'],
@@ -106,7 +105,7 @@ export const createEventLogHealthClient = (
       {
         start: interval.from,
         end: interval.to,
-        filter: kqlFilter,
+        filter: EVENT_PROVIDERS_FILTER,
         aggs,
       },
       namespaces

diff --git a/...ib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/filters.ts b/...ib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/filters.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DETECTION_RULES_FILTER = 'alert.attributes.consumer: "siem"';
diff --git a/.../rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts b/.../rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts
@@ -16,6 +16,7 @@ import type {
   SpaceHealthSnapshot,
 } from '../../../../../../../common/api/detection_engine/rule_monitoring';
 import { RULE_SAVED_OBJECT_TYPE } from '../../event_log/event_log_constants';
+import { DETECTION_RULES_FILTER } from './filters';
 import {
   getClusterHealthAggregation,
   normalizeClusterHealthAggregationResult,
@@ -69,7 +70,12 @@ export const createRuleObjectsHealthClient = (
 
     async calculateSpaceHealth(args: SpaceHealthParameters): Promise<SpaceHealth> {
       const aggs = getSpaceHealthAggregation();
-      const aggregations = await rulesClient.aggregate({ aggs });
+      const aggregations = await rulesClient.aggregate({
+        options: {
+          filter: DETECTION_RULES_FILTER, // make sure to query only detection rules
+        },
+        aggs,
+      });
 
       return {
         state_at_the_moment: normalizeSpaceHealthAggregationResult(aggregations),
@@ -86,6 +92,7 @@ export const createRuleObjectsHealthClient = (
       const aggs = getClusterHealthAggregation();
       const response = await internalSavedObjectsClient.find<unknown, Record<string, unknown>>({
         type: RULE_SAVED_OBJECT_TYPE, // query rules
+        filter: DETECTION_RULES_FILTER, // make sure to query only detection rules
         namespaces: ['*'], // aggregate rules in all Kibana spaces
         perPage: 0, // don't return rules in the response, we only need aggs
         aggs,

diff --git a/...on_engine/rule_monitoring/logic/detection_engine_health/rule_spaces/rule_spaces_client.ts b/...on_engine/rule_monitoring/logic/detection_engine_health/rule_spaces/rule_spaces_client.ts
@@ -7,6 +7,7 @@
 
 import type { SavedObjectsClientContract, Logger } from '@kbn/core/server';
 import { RULE_SAVED_OBJECT_TYPE } from '../../event_log/event_log_constants';
+import { DETECTION_RULES_FILTER } from '../rule_objects/filters';
 import { getSpacesAggregation, normalizeSpacesAggregation } from './aggregations/spaces';
 
 /**
@@ -38,6 +39,7 @@ export const createRuleSpacesClient = (
       const aggs = getSpacesAggregation();
       const response = await internalSavedObjectsClient.find<unknown, Record<string, unknown>>({
         type: RULE_SAVED_OBJECT_TYPE, // query rules
+        filter: DETECTION_RULES_FILTER, // make sure to query only detection rules
         namespaces: ['*'], // aggregate rules in all Kibana spaces
         perPage: 0, // don't return rules in the response, we only need aggs
         aggs,

diff --git a/..._solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts b/..._solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts
@@ -13,6 +13,7 @@ export const TIMESTAMP = `@timestamp` as const;
 export const MESSAGE = 'message' as const;
 export const EVENT_PROVIDER = 'event.provider' as const;
 export const EVENT_ACTION = 'event.action' as const;
+export const EVENT_CATEGORY = 'event.category' as const;
 export const EVENT_SEQUENCE = 'event.sequence' as const;
 
 export const LOG_LEVEL = 'log.level' as const;