Suppression/im investigate ftr failures

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import sortBy from 'lodash/sortBy';
 import dateMath from '@elastic/datemath';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { RuleExecutorOptions } from '@kbn/alerting-plugin/server';
@@ -17,12 +18,14 @@ import {
   ALERT_START,
   ALERT_SUPPRESSION_DOCS_COUNT,
   ALERT_SUPPRESSION_END,
+  ALERT_SUPPRESSION_START,
   ALERT_UUID,
   ALERT_WORKFLOW_STATUS,
   TIMESTAMP,
   VERSION,
 } from '@kbn/rule-data-utils';
 import { mapKeys, snakeCase } from 'lodash/fp';
+import type { IRuleDataClient } from '..';
 import { getCommonAlertFields } from './get_common_alert_fields';
 import { CreatePersistenceRuleTypeWrapper } from './persistence_types';
 import { errorAggregator } from './utils';
@@ -63,6 +66,115 @@ const mapAlertsToBulkCreate = <T>(alerts: Array<{ _id: string; _source: T }>) =>
   return alerts.flatMap((alert) => [{ create: { _id: alert._id } }, alert._source]);
 };
 
+const filterDuplicateAlerts = async <T extends { _id: string }>({
+  alerts,
+  spaceId,
+  ruleDataClient,
+}: {
+  alerts: T[];
+  spaceId: string;
+  ruleDataClient: IRuleDataClient;
+}) => {
+  const CHUNK_SIZE = 10000;
+  const alertChunks = chunk(alerts, CHUNK_SIZE);
+  const filteredAlerts: typeof alerts = [];
+
+  for (const alertChunk of alertChunks) {
+    const request: estypes.SearchRequest = {
+      body: {
+        query: {
+          ids: {
+            values: alertChunk.map((alert) => alert._id),
+          },
+        },
+        aggs: {
+          uuids: {
+            terms: {
+              field: ALERT_UUID,
+              size: CHUNK_SIZE,
+            },
+          },
+        },
+        size: 0,
+      },
+    };
+    const response = await ruleDataClient.getReader({ namespace: spaceId }).search(request);
+    const uuidsMap: Record<string, boolean> = {};
+    const aggs = response.aggregations as
+      | Record<estypes.AggregateName, { buckets: Array<{ key: string }> }>
+      | undefined;
+    if (aggs != null) {
+      aggs.uuids.buckets.forEach((bucket) => (uuidsMap[bucket.key] = true));
+      const newAlerts = alertChunk.filter((alert) => !uuidsMap[alert._id]);
+      filteredAlerts.push(...newAlerts);
+    } else {
+      filteredAlerts.push(...alertChunk);
+    }
+  }
+
+  return filteredAlerts;
+};
+
+/**
+ * suppress alerts by ALERT_INSTANCE_ID in memory
+ */
+const suppressAlertsInMemory = <
+  T extends {
+    [ALERT_SUPPRESSION_DOCS_COUNT]: number;
+    [ALERT_INSTANCE_ID]: string;
+    [ALERT_SUPPRESSION_START]: Date;
+    [ALERT_SUPPRESSION_END]: Date;
+  },
+  A extends {
+    _id: string;
+    _source: T;
+  }
+>(
+  alerts: A[]
+): {
+  alertCandidates: A[];
+  suppressedAlerts: A[];
+} => {
+  const idsMap: Record<string, { count: number; suppressionEnd: Date }> = {};
+  const suppressedAlerts: A[] = [];
+
+  const filteredAlerts = sortBy(alerts, (alert) => alert._source[ALERT_SUPPRESSION_START]).filter(
+    (alert) => {
+      const instanceId = alert._source[ALERT_INSTANCE_ID];
+      const suppressionDocsCount = alert._source[ALERT_SUPPRESSION_DOCS_COUNT];
+      const suppressionEnd = alert._source[ALERT_SUPPRESSION_END];
+
+      if (instanceId && idsMap[instanceId] != null) {
+        idsMap[instanceId].count += suppressionDocsCount + 1;
+        // store the max value of suppression end boundary
+        if (suppressionEnd > idsMap[instanceId].suppressionEnd) {
+          idsMap[instanceId].suppressionEnd = suppressionEnd;
+        }
+        suppressedAlerts.push(alert);
+        return false;
+      } else {
+        idsMap[instanceId] = { count: suppressionDocsCount, suppressionEnd };
+        return true;
+      }
+    },
+    []
+  );
+
+  const alertCandidates = filteredAlerts.map((alert) => {
+    const instanceId = alert._source[ALERT_INSTANCE_ID];
+    if (instanceId) {
+      alert._source[ALERT_SUPPRESSION_DOCS_COUNT] = idsMap[instanceId].count;
+      alert._source[ALERT_SUPPRESSION_END] = idsMap[instanceId].suppressionEnd;
+    }
+    return alert;
+  });
+
+  return {
+    alertCandidates,
+    suppressedAlerts,
+  };
+};
+
 export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper =
   ({ logger, ruleDataClient, formatAlert }) =>
   (type) => {
@@ -91,44 +203,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts();
 
               if (writeAlerts && numAlerts) {
-                const CHUNK_SIZE = 10000;
-                const alertChunks = chunk(alerts, CHUNK_SIZE);
-                const filteredAlerts: typeof alerts = [];
-
-                for (const alertChunk of alertChunks) {
-                  const request: estypes.SearchRequest = {
-                    body: {
-                      query: {
-                        ids: {
-                          values: alertChunk.map((alert) => alert._id),
-                        },
-                      },
-                      aggs: {
-                        uuids: {
-                          terms: {
-                            field: ALERT_UUID,
-                            size: CHUNK_SIZE,
-                          },
-                        },
-                      },
-                      size: 0,
-                    },
-                  };
-                  const response = await ruleDataClient
-                    .getReader({ namespace: options.spaceId })
-                    .search(request);
-                  const uuidsMap: Record<string, boolean> = {};
-                  const aggs = response.aggregations as
-                    | Record<estypes.AggregateName, { buckets: Array<{ key: string }> }>
-                    | undefined;
-                  if (aggs != null) {
-                    aggs.uuids.buckets.forEach((bucket) => (uuidsMap[bucket.key] = true));
-                    const newAlerts = alertChunk.filter((alert) => !uuidsMap[alert._id]);
-                    filteredAlerts.push(...newAlerts);
-                  } else {
-                    filteredAlerts.push(...alertChunk);
-                  }
-                }
+                const filteredAlerts: typeof alerts = await filterDuplicateAlerts({
+                  alerts,
+                  ruleDataClient,
+                  spaceId: options.spaceId,
+                });
 
                 if (filteredAlerts.length === 0) {
                   return { createdAlerts: [], errors: {}, alertsWereTruncated: false };
@@ -238,13 +317,29 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 const suppressionWindowStart = dateMath.parse(suppressionWindow, {
                   forceNow: currentTimeOverride,
                 });
+
                 if (!suppressionWindowStart) {
                   throw new Error('Failed to parse suppression window');
                 }
 
+                const filteredDuplicates = await filterDuplicateAlerts({
+                  alerts,
+                  ruleDataClient,
+                  spaceId: options.spaceId,
+                });
+
+                const {
+                  alertCandidates: filteredAlerts,
+                  suppressedAlerts: suppressedInMemoryAlerts,
+                } = suppressAlertsInMemory(filteredDuplicates);
+
+                if (filteredAlerts.length === 0) {
+                  return { createdAlerts: [], errors: {}, suppressedAlerts: [] };
+                }
+
                 const suppressionAlertSearchRequest = {
                   body: {
-                    size: alerts.length,
+                    size: filteredAlerts.length,
                     query: {
                       bool: {
                         filter: [
@@ -257,7 +352,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                           },
                           {
                             terms: {
-                              [ALERT_INSTANCE_ID]: alerts.map(
+                              [ALERT_INSTANCE_ID]: filteredAlerts.map(
                                 (alert) => alert._source['kibana.alert.instance.id']
                               ),
                             },
@@ -299,38 +394,47 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 const existingAlertsByInstanceId = response.hits.hits.reduce<
                   Record<string, estypes.SearchHit<AlertWithSuppressionFields870<{}>>>
                 >((acc, hit) => {
-                  acc[hit._source['kibana.alert.instance.id']] = hit;
+                  acc[hit._source[ALERT_INSTANCE_ID]] = hit;
                   return acc;
                 }, {});
 
                 const [duplicateAlerts, newAlerts] = partition(
-                  alerts,
-                  (alert) =>
-                    existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']] != null
+                  filteredAlerts,
+                  (alert) => existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]] != null
                 );
 
                 const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => {
                   const existingAlert =
-                    existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']];
+                    existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]];
                   const existingDocsCount =
                     existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0;
-                  return [
-                    {
-                      update: {
-                        _id: existingAlert._id,
-                        _index: existingAlert._index,
-                        require_alias: false,
+
+                  // do not count alerts that already were suppressed
+                  if (
+                    existingAlert._source?.[ALERT_SUPPRESSION_END] &&
+                    existingAlert._source?.[ALERT_SUPPRESSION_END] <=
+                      alert._source[ALERT_SUPPRESSION_END]
+                  ) {
+                    return [];
+                  } else {
+                    return [
+                      {
+                        update: {
+                          _id: existingAlert._id,
+                          _index: existingAlert._index,
+                          require_alias: false,
+                        },
                       },
-                    },
-                    {
-                      doc: {
-                        [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
-                        [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END],
-                        [ALERT_SUPPRESSION_DOCS_COUNT]:
-                          existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1,
+                      {
+                        doc: {
+                          [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
+                          [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END],
+                          [ALERT_SUPPRESSION_DOCS_COUNT]:
+                            existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1,
+                        },
                       },
-                    },
-                  ];
+                    ];
+                  }
                 });
 
                 let enrichedAlerts = newAlerts;
@@ -358,7 +462,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 });
 
                 if (bulkResponse == null) {
-                  return { createdAlerts: [], errors: {} };
+                  return { createdAlerts: [], errors: {}, suppressedAlerts: [] };
                 }
 
                 const createdAlerts = augmentedAlerts
@@ -402,11 +506,12 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
 
                 return {
                   createdAlerts,
+                  suppressedAlerts: [...duplicateAlerts, ...suppressedInMemoryAlerts],
                   errors: errorAggregator(bulkResponse.body, [409]),
                 };
               } else {
                 logger.debug('Writing is disabled.');
-                return { createdAlerts: [], errors: {} };
+                return { createdAlerts: [], errors: {}, suppressedAlerts: [] };
               }
             },
           },

x-pack/plugins/rule_registry/server/utils/persistence_types.ts

@@ -52,7 +52,12 @@ export type SuppressedAlertService = <T extends SuppressionFieldsLatest>(
     params: { spaceId: string }
   ) => Promise<Array<{ _id: string; _source: T }>>,
   currentTimeOverride?: Date
-) => Promise<Omit<PersistenceAlertServiceResult<T>, 'alertsWereTruncated'>>;
+) => Promise<SuppressedAlertServiceResult<T>>;
+
+export interface SuppressedAlertServiceResult<T>
+  extends Omit<PersistenceAlertServiceResult<T>, 'alertsWereTruncated'> {
+  suppressedAlerts: Array<{ _id: string; _source: T }>;
+}
 
 export interface PersistenceAlertServiceResult<T> {
   createdAlerts: Array<AlertWithCommonFieldsLatest<T> & { _id: string; _index: string }>;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/bulk_create_factory.ts

@@ -27,6 +27,7 @@ export interface GenericBulkCreateResponse<T extends BaseFieldsLatest> {
   createdItems: Array<AlertWithCommonFieldsLatest<T> & { _id: string; _index: string }>;
   errors: string[];
   alertsWereTruncated: boolean;
+  suppressedAlertsCount?: number;
 }
 
 export const bulkCreateFactory =

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -12,13 +12,15 @@ import { SERVER_APP_ID } from '../../../../../common/constants';
 
 import { ThreatRuleParams } from '../../rule_schema';
 import { indicatorMatchExecutor } from './indicator_match';
-import type { CreateRuleOptions, SecurityAlertType } from '../types';
+import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types';
 import { validateIndexPatterns } from '../utils';
+import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts';
+import type { BuildReasonMessage } from '../utils/reason_formatters';
 
 export const createIndicatorMatchAlertType = (
   createOptions: CreateRuleOptions
 ): SecurityAlertType<ThreatRuleParams, {}, {}, 'default'> => {
-  const { eventsTelemetry, version } = createOptions;
+  const { eventsTelemetry, version, licensing } = createOptions;
   return {
     id: INDICATOR_RULE_TYPE_ID,
     name: 'Indicator Match Rule',
@@ -74,8 +76,26 @@ export const createIndicatorMatchAlertType = (
           inputIndexFields,
         },
         services,
+        spaceId,
         state,
       } = execOptions;
+      const runOpts = execOptions.runOpts;
+
+      const wrapSuppressedHits = (
+        events: SignalSourceHit[],
+        buildReasonMessage: BuildReasonMessage
+      ) =>
+        wrapSuppressedAlerts({
+          events,
+          spaceId,
+          completeRule,
+          mergeStrategy: runOpts.mergeStrategy,
+          indicesToQuery: runOpts.inputIndex,
+          buildReasonMessage,
+          alertTimestampOverride: runOpts.alertTimestampOverride,
+          ruleExecutionLogger,
+          publicBaseUrl: runOpts.publicBaseUrl,
+        });
 
       const result = await indicatorMatchExecutor({
         inputIndex,
@@ -95,6 +115,9 @@ export const createIndicatorMatchAlertType = (
         exceptionFilter,
         unprocessedExceptions,
         inputIndexFields,
+        wrapSuppressedHits,
+        runOpts,
+        licensing,
       });
       return { ...result, state };
     },