[Security Solution][Alerts] Alert suppression time window

packages/kbn-rule-data-utils/src/technical_field_names.ts

@@ -32,6 +32,7 @@ const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as
 const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const;
 const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const;
 const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const;
+const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected_at` as const;
 const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const;
 const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const;
 const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const;
@@ -50,6 +51,7 @@ const ALERT_SUPPRESSION_VALUE = `${ALERT_SUPPRESSION_TERMS}.value` as const;
 const ALERT_SUPPRESSION_START = `${ALERT_SUPPRESSION_META}.start` as const;
 const ALERT_SUPPRESSION_END = `${ALERT_SUPPRESSION_META}.end` as const;
 const ALERT_SUPPRESSION_DOCS_COUNT = `${ALERT_SUPPRESSION_META}.docs_count` as const;
+const ALERT_SUPPRESSION_GROUP_ID = `${ALERT_SUPPRESSION_META}.group_id` as const;
 
 // Fields pertaining to the cases associated with the alert
 const ALERT_CASE_IDS = `${ALERT_NAMESPACE}.case_ids` as const;
@@ -128,6 +130,7 @@ const fields = {
   ALERT_EVALUATION_VALUE,
   ALERT_FLAPPING,
   ALERT_INSTANCE_ID,
+  ALERT_LAST_DETECTED,
   ALERT_RULE_CONSUMER,
   ALERT_RULE_PRODUCER,
   ALERT_REASON,
@@ -184,6 +187,7 @@ const fields = {
   ALERT_SUPPRESSION_START,
   ALERT_SUPPRESSION_END,
   ALERT_SUPPRESSION_DOCS_COUNT,
+  ALERT_SUPPRESSION_GROUP_ID,
   SPACE_IDS,
   VERSION,
 };
@@ -197,6 +201,7 @@ export {
   ALERT_EVALUATION_VALUE,
   ALERT_FLAPPING,
   ALERT_INSTANCE_ID,
+  ALERT_LAST_DETECTED,
   ALERT_NAMESPACE,
   ALERT_RULE_NAMESPACE,
   ALERT_RULE_CONSUMER,
@@ -260,6 +265,7 @@ export {
   ALERT_SUPPRESSION_START,
   ALERT_SUPPRESSION_END,
   ALERT_SUPPRESSION_DOCS_COUNT,
+  ALERT_SUPPRESSION_GROUP_ID,
   TAGS,
   TIMESTAMP,
   SPACE_IDS,

x-pack/plugins/rule_registry/common/schemas/8.7.0/index.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ALERT_SUPPRESSION_TERMS,
+  ALERT_SUPPRESSION_START,
+  ALERT_SUPPRESSION_END,
+  ALERT_SUPPRESSION_DOCS_COUNT,
+  ALERT_INSTANCE_ID,
+} from '@kbn/rule-data-utils';
+
+/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.0.0.
+Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.0.0.
+
+If you are adding new fields for a new release of Kibana, create a new sibling folder to this one
+for the version to be released and add the field(s) to the schema in that folder.
+
+Then, update `../index.ts` to import from the new folder that has the latest schemas, add the
+new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas.
+*/
+
+export interface SuppressionFields {
+  [ALERT_SUPPRESSION_TERMS]: Array<{ field: string; value: string | number | null }>;
+  [ALERT_SUPPRESSION_START]: Date;
+  [ALERT_SUPPRESSION_END]: Date;
+  [ALERT_SUPPRESSION_DOCS_COUNT]: number;
+  [ALERT_INSTANCE_ID]: string;
+}

x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts

@@ -106,20 +106,18 @@ export class RuleDataClient implements IRuleDataClient {
     };
 
     return {
-      search: async <TSearchRequest extends ESSearchRequest>(
+      search: async <
+        TSearchRequest extends ESSearchRequest,
+        TAlertDoc = Partial<ParsedTechnicalFields & ParsedExperimentalFields>
+      >(
         request: TSearchRequest
-      ): Promise<
-        ESSearchResponse<Partial<ParsedTechnicalFields & ParsedExperimentalFields>, TSearchRequest>
-      > => {
+      ): Promise<ESSearchResponse<TAlertDoc, TSearchRequest>> => {
         try {
           const clusterClient = await waitUntilReady();
           return (await clusterClient.search({
             ...request,
             index: indexPattern,
-          })) as unknown as ESSearchResponse<
-            Partial<ParsedTechnicalFields & ParsedExperimentalFields>,
-            TSearchRequest
-          >;
+          })) as unknown as ESSearchResponse<TAlertDoc, TSearchRequest>;
         } catch (err) {
           this.options.logger.error(`Error performing search in RuleDataClient - ${err.message}`);
           throw err;

x-pack/plugins/rule_registry/server/rule_data_client/types.ts

@@ -23,11 +23,12 @@ export interface IRuleDataClient {
 }
 
 export interface IRuleDataReader {
-  search<TSearchRequest extends ESSearchRequest>(
+  search<
+    TSearchRequest extends ESSearchRequest,
+    TAlertDoc = Partial<ParsedTechnicalFields & ParsedExperimentalFields>
+  >(
     request: TSearchRequest
-  ): Promise<
-    ESSearchResponse<Partial<ParsedTechnicalFields & ParsedExperimentalFields>, TSearchRequest>
-  >;
+  ): Promise<ESSearchResponse<TAlertDoc, TSearchRequest>>;
 
   getDynamicIndexPattern(target?: string): Promise<{
     title: string;

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.mock.ts

@@ -11,6 +11,7 @@ import { PersistenceServices } from './persistence_types';
 export const createPersistenceServicesMock = (): jest.Mocked<PersistenceServices> => {
   return {
     alertWithPersistence: jest.fn(),
+    alertWithSuppression: jest.fn(),
   };
 };
 

x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts

@@ -5,13 +5,23 @@
  * 2.0.
  */
 
+import dateMath from '@elastic/datemath';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { chunk } from 'lodash';
-import { ALERT_UUID, VERSION } from '@kbn/rule-data-utils';
+import { chunk, partition } from 'lodash';
+import {
+  ALERT_INSTANCE_ID,
+  ALERT_LAST_DETECTED,
+  ALERT_SUPPRESSION_DOCS_COUNT,
+  ALERT_SUPPRESSION_END,
+  ALERT_UUID,
+  TIMESTAMP,
+  VERSION,
+} from '@kbn/rule-data-utils';
 import { getCommonAlertFields } from './get_common_alert_fields';
 import { CreatePersistenceRuleTypeWrapper } from './persistence_types';
 import { errorAggregator } from './utils';
 import { createGetSummarizedAlertsFn } from './create_get_summarized_alerts_fn';
+import { SuppressionFields } from '../../common/schemas/8.7.0';
 
 export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper =
   ({ logger, ruleDataClient }) =>
@@ -94,7 +104,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                       spaceId: options.spaceId,
                     });
                   } catch (e) {
-                    logger.debug('Enrichemnts failed');
+                    logger.debug('Enrichments failed');
                   }
                 }
 
@@ -146,6 +156,171 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 return { createdAlerts: [], errors: {}, alertsWereTruncated: false };
               }
             },
+            alertWithSuppression: async (
+              alerts,
+              suppressionWindow,
+              enrichAlerts,
+              currentTimeOverride
+            ) => {
+              const ruleDataClientWriter = await ruleDataClient.getWriter({
+                namespace: options.spaceId,
+              });
+
+              // Only write alerts if:
+              // - writing is enabled
+              //   AND
+              //   - rule execution has not been cancelled due to timeout
+              //     OR
+              //   - if execution has been cancelled due to timeout, if feature flags are configured to write alerts anyway
+              const writeAlerts =
+                ruleDataClient.isWriteEnabled() && options.services.shouldWriteAlerts();
+
+              if (writeAlerts && alerts.length > 0) {
+                const commonRuleFields = getCommonAlertFields(options);
+
+                const suppressionWindowStart = dateMath.parse(suppressionWindow, {
+                  forceNow: currentTimeOverride,
+                });
+                if (!suppressionWindowStart) {
+                  throw new Error('Failed to parse suppression window');
+                }
+
+                const suppressionAlertSearchRequest = {
+                  body: {
+                    size: alerts.length,
+                    query: {
+                      bool: {
+                        filter: [
+                          {
+                            range: {
+                              [TIMESTAMP]: {
+                                gte: suppressionWindowStart.toISOString(),
+                              },
+                            },
+                          },
+                          {
+                            terms: {
+                              [ALERT_INSTANCE_ID]: alerts.map(
+                                (alert) => alert._source['kibana.alert.instance.id']
+                              ),
+                            },
+                          },
+                        ],
+                      },
+                    },
+                    collapse: {
+                      field: ALERT_INSTANCE_ID,
+                    },
+                    sort: [
+                      {
+                        '@timestamp': {
+                          order: 'desc' as const,
+                        },
+                      },
+                    ],
+                  },
+                };
+
+                const response = await ruleDataClient
+                  .getReader({ namespace: options.spaceId })
+                  .search<typeof suppressionAlertSearchRequest, SuppressionFields>(
+                    suppressionAlertSearchRequest
+                  );
+
+                const existingAlertsByInstanceId = response.hits.hits.reduce<
+                  Record<string, estypes.SearchHit<SuppressionFields>>
+                >((acc, hit) => {
+                  acc[hit._source['kibana.alert.instance.id']] = hit;
+                  return acc;
+                }, {});
+
+                const [duplicateAlerts, newAlerts] = partition(
+                  alerts,
+                  (alert) =>
+                    existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']] != null
+                );
+
+                const duplicateAlertUpdates = duplicateAlerts.flatMap((alert) => {
+                  const existingAlert =
+                    existingAlertsByInstanceId[alert._source['kibana.alert.instance.id']];
+                  const existingDocsCount =
+                    existingAlert._source?.[ALERT_SUPPRESSION_DOCS_COUNT] ?? 0;
+                  return [
+                    {
+                      update: {
+                        _id: existingAlert._id,
+                        _index: existingAlert._index,
+                        require_alias: false,
+                      },
+                    },
+                    {
+                      doc: {
+                        [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
+                        [ALERT_SUPPRESSION_END]: alert._source[ALERT_SUPPRESSION_END],
+                        [ALERT_SUPPRESSION_DOCS_COUNT]:
+                          existingDocsCount + alert._source[ALERT_SUPPRESSION_DOCS_COUNT] + 1,
+                      },
+                    },
+                  ];
+                });
+
+                let enrichedAlerts = newAlerts;
+
+                if (enrichAlerts) {
+                  try {
+                    enrichedAlerts = await enrichAlerts(enrichedAlerts, {
+                      spaceId: options.spaceId,
+                    });
+                  } catch (e) {
+                    logger.debug('Enrichments failed');
+                  }
+                }
+
+                const augmentedAlerts = enrichedAlerts.map((alert) => {
+                  return {
+                    ...alert,
+                    _source: {
+                      [VERSION]: ruleDataClient.kibanaVersion,
+                      [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(),
+                      ...commonRuleFields,
+                      ...alert._source,
+                    },
+                  };
+                });
+
+                const newAlertCreates = augmentedAlerts.flatMap((alert) => [
+                  { create: { _id: alert._id } },
+                  alert._source,
+                ]);
+
+                const bulkResponse = await ruleDataClientWriter.bulk({
+                  body: [...duplicateAlertUpdates, ...newAlertCreates],
+                  refresh: true,
+                });
+
+                if (bulkResponse == null) {
+                  return { createdAlerts: [], errors: {} };
+                }
+
+                return {
+                  createdAlerts: augmentedAlerts
+                    .map((alert, idx) => {
+                      const responseItem =
+                        bulkResponse.body.items[idx + duplicateAlertUpdates.length].create;
+                      return {
+                        _id: responseItem?._id ?? '',
+                        _index: responseItem?._index ?? '',
+                        ...alert._source,
+                      };
+                    })
+                    .filter((_, idx) => bulkResponse.body.items[idx].create?.status === 201),
+                  errors: errorAggregator(bulkResponse.body, [409]),
+                };
+              } else {
+                logger.debug('Writing is disabled.');
+                return { createdAlerts: [], errors: {} };
+              }
+            },
           },
         });
 

x-pack/plugins/rule_registry/server/utils/persistence_types.ts

@@ -18,6 +18,7 @@ import { WithoutReservedActionGroups } from '@kbn/alerting-plugin/common';
 import { IRuleDataClient } from '../rule_data_client';
 import { BulkResponseErrorAggregation } from './utils';
 import { AlertWithCommonFieldsLatest } from '../../common/schemas';
+import { SuppressionFields } from '../../common/schemas/8.7.0';
 
 export type PersistenceAlertService = <T>(
   alerts: Array<{
@@ -40,6 +41,19 @@ export type PersistenceAlertService = <T>(
   >
 ) => Promise<PersistenceAlertServiceResult<T>>;
 
+export type SuppressedAlertService = <T extends SuppressionFields>(
+  alerts: Array<{
+    _id: string;
+    _source: T;
+  }>,
+  suppressionWindow: string,
+  enrichAlerts?: (
+    alerts: Array<{ _id: string; _source: T }>,
+    params: { spaceId: string }
+  ) => Promise<Array<{ _id: string; _source: T }>>,
+  currentTimeOverride?: Date
+) => Promise<Omit<PersistenceAlertServiceResult<T>, 'alertsWereTruncated'>>;
+
 export interface PersistenceAlertServiceResult<T> {
   createdAlerts: Array<AlertWithCommonFieldsLatest<T> & { _id: string; _index: string }>;
   errors: BulkResponseErrorAggregation;
@@ -48,6 +62,7 @@ export interface PersistenceAlertServiceResult<T> {
 
 export interface PersistenceServices {
   alertWithPersistence: PersistenceAlertService;
+  alertWithSuppression: SuppressedAlertService;
 }
 
 export type PersistenceAlertType<