[ResponseOps] Alert creation delay based on user definition (elastic#…

…175851) Resolves elastic#173009 ## Summary This PR: - Changes the field name from `notification_delay` to `alert_delay` - Updates the alerts client and rule registry to index new alert docs on a delay - Updates the framework code to delay the creation of an alert ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### To verify - Use [Dev Tools](http://localhost:5601/app/dev_tools#/console) to create a rule with the `alertDelay` ``` POST kbn:/api/alerting/rule { "params": { "searchType": "esQuery", "timeWindowSize": 5, "timeWindowUnit": "m", "threshold": [ -1 ], "thresholdComparator": ">", "size": 100, "esQuery": """{ "query":{ "match_all" : {} } }""", "aggType": "count", "groupBy": "all", "termSize": 5, "excludeHitsFromPreviousRun": false, "sourceFields": [], "index": [ ".kibana-event-log*" ], "timeField": "@timestamp" }, "consumer": "stackAlerts", "schedule": { "interval": "1m" }, "tags": [], "name": "test", "rule_type_id": ".es-query", "actions": [ { "group": "query matched", "id": "${ACTION_ID}", "params": { "level": "info", "message": """Elasticsearch query rule '{{rule.name}}' is active: - Value: {{context.value}} - Conditions Met: {{context.conditions}} over {{rule.params.timeWindowSize}}{{rule.params.timeWindowUnit}} - Timestamp: {{context.date}} - Link: {{context.link}}""" }, "frequency": { "notify_when": "onActionGroupChange", "throttle": null, "summary": false } } ], "alert_delay": { "active": 3 } } ``` - Verify that the alert will not be created until it has matched the delay threshold. - Verify that the delay does not affect recovered alerts
fkanout · Mar 4, 2024 · b36717e · b36717e
1 parent 6a2bf7d
commit b36717e
Show file tree

Hide file tree

Showing 47 changed files with 1,253 additions and 386 deletions.
diff --git a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts
@@ -20,6 +20,8 @@ const trackedAlertStateRt = t.type({
   // count of consecutive recovered alerts for flapping
   // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: t.number,
+  // count of consecutive active alerts will reset if the alert is recovered
+  activeCount: t.number,
 });
 
 export type TrackedLifecycleAlertState = t.TypeOf<typeof trackedAlertStateRt>;

diff --git a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts
@@ -36,8 +36,7 @@ export const metaSchema = schema.object({
   // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: schema.maybe(schema.number()),
   uuid: schema.maybe(schema.string()),
-  // count of consecutive active alerts
-  // will reset if the alert is recovered or if equal to notificationDelay.active stored in the rule
+  // count of consecutive active alerts will reset if the alert is recovered
   activeCount: schema.maybe(schema.number()),
 });
 

diff --git a/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts
@@ -7,7 +7,7 @@
 
 import { schema } from '@kbn/config-schema';
 import { validateDurationV1, validateHoursV1, validateTimezoneV1 } from '../../../validation';
-import { notifyWhenSchemaV1, notificationDelaySchemaV1 } from '../../../response';
+import { notifyWhenSchemaV1, alertDelaySchemaV1 } from '../../../response';
 import { alertsFilterQuerySchemaV1 } from '../../../../alerts_filter_query';
 
 export const actionFrequencySchema = schema.object({
@@ -68,7 +68,7 @@ export const createBodySchema = schema.object({
   }),
   actions: schema.arrayOf(actionSchema, { defaultValue: [] }),
   notify_when: schema.maybe(schema.nullable(notifyWhenSchemaV1)),
-  notification_delay: schema.maybe(notificationDelaySchemaV1),
+  alert_delay: schema.maybe(alertDelaySchemaV1),
 });
 
 export const createParamsSchema = schema.object({

diff --git a/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts b/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts
@@ -30,7 +30,7 @@ export interface CreateRuleRequestBody<Params extends RuleParamsV1 = never> {
   schedule: CreateBodySchema['schedule'];
   actions: CreateBodySchema['actions'];
   notify_when?: CreateBodySchema['notify_when'];
-  notification_delay?: CreateBodySchema['notification_delay'];
+  alert_delay?: CreateBodySchema['alert_delay'];
 }
 
 export interface CreateRuleResponse<Params extends RuleParamsV1 = never> {

diff --git a/x-pack/plugins/alerting/common/routes/rule/response/index.ts b/x-pack/plugins/alerting/common/routes/rule/response/index.ts
@@ -37,7 +37,7 @@ export {
   ruleSnoozeScheduleSchema as ruleSnoozeScheduleSchemaV1,
   notifyWhenSchema as notifyWhenSchemaV1,
   scheduleIdsSchema as scheduleIdsSchemaV1,
-  notificationDelaySchema as notificationDelaySchemaV1,
+  alertDelaySchema as alertDelaySchemaV1,
 } from './schemas/v1';
 
 export type {

diff --git a/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts
@@ -182,7 +182,7 @@ export const ruleSnoozeScheduleSchema = schema.object({
   skipRecurrences: schema.maybe(schema.arrayOf(schema.string())),
 });
 
-export const notificationDelaySchema = schema.object({
+export const alertDelaySchema = schema.object({
   active: schema.number(),
 });
 
@@ -218,7 +218,7 @@ export const ruleResponseSchema = schema.object({
   revision: schema.number(),
   running: schema.maybe(schema.nullable(schema.boolean())),
   view_in_app_relative_url: schema.maybe(schema.nullable(schema.string())),
-  notification_delay: schema.maybe(notificationDelaySchema),
+  alert_delay: schema.maybe(alertDelaySchema),
 });
 
 export const scheduleIdsSchema = schema.maybe(schema.arrayOf(schema.string()));
diff --git a/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts b/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts
@@ -53,5 +53,5 @@ export interface RuleResponse<Params extends RuleParams = never> {
   revision: RuleResponseSchemaType['revision'];
   running?: RuleResponseSchemaType['running'];
   view_in_app_relative_url?: RuleResponseSchemaType['view_in_app_relative_url'];
-  notification_delay?: RuleResponseSchemaType['notification_delay'];
+  alert_delay?: RuleResponseSchemaType['alert_delay'];
 }
diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts
@@ -141,7 +141,7 @@ export interface MappedParamsProperties {
 
 export type MappedParams = SavedObjectAttributes & MappedParamsProperties;
 
-export interface NotificationDelay {
+export interface AlertDelay extends SavedObjectAttributes {
   active: number;
 }
 
@@ -178,7 +178,7 @@ export interface Rule<Params extends RuleTypeParams = never> {
   revision: number;
   running?: boolean | null;
   viewInAppRelativeUrl?: string;
-  notificationDelay?: NotificationDelay;
+  alertDelay?: AlertDelay;
 }
 
 export interface SanitizedAlertsFilter extends AlertsFilter {
@@ -222,6 +222,7 @@ export type SanitizedRuleConfig = Pick<
   | 'muteAll'
   | 'revision'
   | 'snoozeSchedule'
+  | 'alertDelay'
 > & {
   producer: string;
   ruleTypeId: string;

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts
@@ -308,6 +308,7 @@ describe('Alerts Client', () => {
           flappingSettings: DEFAULT_FLAPPING_SETTINGS,
           notifyOnActionGroupChange: true,
           maintenanceWindowIds: [],
+          alertDelay: 0,
         };
       });
 
@@ -516,6 +517,25 @@ describe('Alerts Client', () => {
           });
         });
 
+        test('should not index new alerts if the activeCount is less than the rule alertDelay', async () => {
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({
+            ...alertsClientParams,
+            rule: { ...alertsClientParams.rule, alertDelay: 3 },
+          });
+
+          await alertsClient.initializeExecution(defaultExecutionOpts);
+
+          // Report 1 new alerts
+          const alertExecutorService = alertsClient.factory();
+          alertExecutorService.create('1').scheduleActions('default');
+
+          alertsClient.processAndLogAlerts(processAndLogAlertsOpts);
+
+          await alertsClient.persistAlerts();
+
+          expect(clusterClient.bulk).not.toHaveBeenCalled();
+        });
+
         test('should update ongoing alerts in existing index', async () => {
           clusterClient.search.mockResolvedValue({
             took: 10,

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts
@@ -344,6 +344,11 @@ export class AlertsClient<
             })
           );
         } else {
+          // skip writing the alert document if the number of consecutive
+          // active alerts is less than the rule alertDelay threshold
+          if (activeAlerts[id].getActiveCount() < this.options.rule.alertDelay) {
+            continue;
+          }
           activeAlertsToIndex.push(
             buildNewAlert<
               AlertData,

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts
@@ -25,6 +25,7 @@ export const alertRuleData: AlertRuleData = {
   revision: 0,
   spaceId: 'default',
   tags: ['rule-', '-tags'],
+  alertDelay: 0,
 };
 
 export const mockAAD = {

diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts
@@ -245,6 +245,7 @@ describe('Legacy Alerts Client', () => {
       flappingSettings: DEFAULT_FLAPPING_SETTINGS,
       notifyOnActionGroupChange: true,
       maintenanceWindowIds: ['window-id1', 'window-id2'],
+      alertDelay: 5,
     });
 
     expect(processAlerts).toHaveBeenCalledWith({
@@ -275,13 +276,15 @@ describe('Legacy Alerts Client', () => {
       },
       true,
       'default',
+      5,
       {},
       {
         '1': new Alert<AlertInstanceContext, AlertInstanceContext>('1', testAlert1),
         '2': new Alert<AlertInstanceContext, AlertInstanceContext>('2', testAlert2),
       },
       {},
-      {}
+      {},
+      null
     );
 
     expect(logAlerts).toHaveBeenCalledWith({

diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts
@@ -140,6 +140,7 @@ export class LegacyAlertsClient<
     notifyOnActionGroupChange,
     flappingSettings,
     maintenanceWindowIds,
+    alertDelay,
   }: ProcessAlertsOpts) {
     const {
       newAlerts: processedAlertsNew,
@@ -168,10 +169,12 @@ export class LegacyAlertsClient<
       flappingSettings,
       notifyOnActionGroupChange,
       this.options.ruleType.defaultActionGroupId,
+      alertDelay,
       processedAlertsNew,
       processedAlertsActive,
       trimmedAlertsRecovered,
-      processedAlertsRecoveredCurrent
+      processedAlertsRecoveredCurrent,
+      this.startedAtString
     );
     alerts.currentRecoveredAlerts = merge(alerts.currentRecoveredAlerts, earlyRecoveredAlerts);
 
@@ -203,11 +206,13 @@ export class LegacyAlertsClient<
     flappingSettings,
     notifyOnActionGroupChange,
     maintenanceWindowIds,
+    alertDelay,
   }: ProcessAndLogAlertsOpts) {
     this.processAlerts({
       notifyOnActionGroupChange,
       flappingSettings,
       maintenanceWindowIds,
+      alertDelay,
     });
 
     this.logAlerts({

diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts b/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts
@@ -61,6 +61,7 @@ describe('formatRule', () => {
           revision: 0,
           spaceId: 'default',
           tags: ['rule-', '-tags'],
+          alertDelay: 0,
         },
         ruleType,
       })

diff --git a/x-pack/plugins/alerting/server/alerts_client/types.ts b/x-pack/plugins/alerting/server/alerts_client/types.ts
@@ -47,6 +47,7 @@ export interface AlertRuleData {
   revision: number;
   spaceId: string;
   tags: string[];
+  alertDelay: number;
 }
 
 export interface AlertRule {
@@ -111,12 +112,14 @@ export interface ProcessAndLogAlertsOpts {
   flappingSettings: RulesSettingsFlappingProperties;
   notifyOnActionGroupChange: boolean;
   maintenanceWindowIds: string[];
+  alertDelay: number;
 }
 
 export interface ProcessAlertsOpts {
   flappingSettings: RulesSettingsFlappingProperties;
   notifyOnActionGroupChange: boolean;
   maintenanceWindowIds: string[];
+  alertDelay: number;
 }
 
 export interface LogAlertsOpts {

diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts
@@ -1475,6 +1475,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -1495,6 +1496,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
             kibanaVersion: '8.8.0',
           });
@@ -1528,6 +1530,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -1576,6 +1579,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -1610,6 +1614,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
             kibanaVersion: '8.8.0',
           });
@@ -1674,6 +1679,7 @@ describe('Alerts Service', () => {
                 revision: 0,
                 spaceId: 'default',
                 tags: ['rule-', '-tags'],
+                alertDelay: 0,
               },
             }),
             alertsService.createAlertsClient({
@@ -1691,6 +1697,7 @@ describe('Alerts Service', () => {
                 revision: 0,
                 spaceId: 'default',
                 tags: ['rule-', '-tags'],
+                alertDelay: 0,
               },
             }),
           ]);
@@ -1725,6 +1732,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
             kibanaVersion: '8.8.0',
           });
@@ -1781,6 +1789,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -1801,6 +1810,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
             kibanaVersion: '8.8.0',
           });
@@ -1865,6 +1875,7 @@ describe('Alerts Service', () => {
                 revision: 0,
                 spaceId: 'default',
                 tags: ['rule-', '-tags'],
+                alertDelay: 0,
               },
             });
           };
@@ -1892,6 +1903,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
             kibanaVersion: '8.8.0',
           });
@@ -1961,6 +1973,7 @@ describe('Alerts Service', () => {
                 revision: 0,
                 spaceId: 'default',
                 tags: ['rule-', '-tags'],
+                alertDelay: 0,
               },
             });
           };
@@ -2026,6 +2039,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -2091,6 +2105,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 
@@ -2154,6 +2169,7 @@ describe('Alerts Service', () => {
               revision: 0,
               spaceId: 'default',
               tags: ['rule-', '-tags'],
+              alertDelay: 0,
             },
           });
 

diff --git a/...lugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts b/...lugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts
@@ -7,11 +7,7 @@
 
 import { schema } from '@kbn/config-schema';
 import { validateDuration } from '../../../validation';
-import {
-  notifyWhenSchema,
-  actionAlertsFilterSchema,
-  notificationDelaySchema,
-} from '../../../schemas';
+import { notifyWhenSchema, actionAlertsFilterSchema, alertDelaySchema } from '../../../schemas';
 
 export const createRuleDataSchema = schema.object({
   name: schema.string(),
@@ -44,5 +40,5 @@ export const createRuleDataSchema = schema.object({
     { defaultValue: [] }
   ),
   notifyWhen: schema.maybe(schema.nullable(notifyWhenSchema)),
-  notificationDelay: schema.maybe(notificationDelaySchema),
+  alertDelay: schema.maybe(alertDelaySchema),
 });