From 5fa4af9c8b0e0ec675ee2b84b5b60207e3043ffa Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Wed, 4 Dec 2024 07:21:49 -0500 Subject: [PATCH] [Security Solution][Detection Engine] Adds support for suppressing EQL sequence alerts (#189725) ## Summary Provide support for suppressing EQL sequence alerts. Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> --- .../common/schemas/8.17.0/index.ts | 32 + .../rule_registry/common/schemas/index.ts | 6 +- .../create_persistence_rule_type_wrapper.ts | 19 +- .../server/utils/persistence_types.ts | 4 + .../common/experimental_features.ts | 11 + .../components/step_define_rule/index.tsx | 27 +- .../components/step_define_rule/schema.tsx | 23 - .../step_define_rule/translations.tsx | 16 - ...e_experimental_feature_fields_transform.ts | 35 +- .../pages/rule_editing/index.tsx | 12 +- .../rule_details/rule_definition_section.tsx | 2 +- .../logic/use_alert_suppression.test.tsx | 38 +- .../logic/use_alert_suppression.tsx | 17 +- .../build_alert_group_from_sequence.test.ts | 40 +- .../eql/build_alert_group_from_sequence.ts | 114 +- .../rule_types/eql/create_eql_alert_type.ts | 31 +- .../rule_types/eql/eql.test.ts | 70 +- .../detection_engine/rule_types/eql/eql.ts | 39 +- .../rule_types/eql/wrap_sequences_factory.ts | 45 +- .../esql/wrap_suppressed_esql_alerts.ts | 2 +- .../wrap_suppressed_new_terms_alerts.ts | 2 +- .../rule_types/query/query.ts | 1 + .../lib/detection_engine/rule_types/types.ts | 6 +- ...bulk_create_suppressed_alerts_in_memory.ts | 118 +- .../utils/bulk_create_with_suppression.ts | 18 +- .../utils/get_is_alert_suppression_active.ts | 6 +- .../utils/robust_field_access.test.ts | 3 + .../utils/suppression_utils.test.ts | 28 +- .../rule_types/utils/suppression_utils.ts | 22 +- .../rule_types/utils/utils.ts | 121 +- .../utils/wrap_suppressed_alerts.ts | 29 +- .../lib/detection_engine/scripts/post_rule.sh | 1 + .../rules/queries/sequence_eql_query.json | 62 + .../sequence_eql_query_no_duration.json | 61 + .../translations/translations/fr-FR.json | 2 - .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - .../config/ess/config.base.ts | 1 + .../eql_alert_suppression.ts | 1515 ++++++++++++++++- .../configs/serverless.config.ts | 3 + .../test/security_solution_cypress/config.ts | 3 + .../rule_creation/eql_rule_suppression.cy.ts | 2 +- .../eql_rule_suppression_sequence.cy.ts | 104 +- 43 files changed, 2371 insertions(+), 324 deletions(-) create mode 100644 x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json diff --git a/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts new file mode 100644 index 0000000000000..cc1d73de3c4ae --- /dev/null +++ b/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALERT_SUPPRESSION_TERMS } from '@kbn/rule-data-utils'; +import { SearchTypes } from '@kbn/data-plugin/common'; +import { AlertWithCommonFields880 } from '../8.8.0'; + +import { SuppressionFields8130 } from '../8.13.0'; + +/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.13.0. +Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.13.0. + +If you are adding new fields for a new release of Kibana, create a new sibling folder to this one +for the version to be released and add the field(s) to the schema in that folder. + +Then, update `../index.ts` to import from the new folder that has the latest schemas, add the +new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas. +*/ + +export interface SuppressionFields8170 + extends Omit { + [ALERT_SUPPRESSION_TERMS]: Array<{ + field: string; + value: SearchTypes | null; + }>; +} + +export type AlertWithSuppressionFields8170 = AlertWithCommonFields880 & SuppressionFields8170; diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts index 5c168a4b899cc..5a94d250392ef 100644 --- a/x-pack/plugins/rule_registry/common/schemas/index.ts +++ b/x-pack/plugins/rule_registry/common/schemas/index.ts @@ -13,11 +13,11 @@ import type { CommonAlertFields880, } from './8.8.0'; -import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0'; +import type { AlertWithSuppressionFields8170, SuppressionFields8170 } from './8.17.0'; export type { - AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest, - SuppressionFields8130 as SuppressionFieldsLatest, + AlertWithSuppressionFields8170 as AlertWithSuppressionFieldsLatest, + SuppressionFields8170 as SuppressionFieldsLatest, CommonAlertFieldName880 as CommonAlertFieldNameLatest, CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest, CommonAlertFields880 as CommonAlertFieldsLatest, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index 9900c889ae73e..892ba70d247ed 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -27,6 +27,7 @@ import { ALERT_RULE_EXECUTION_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; + import type { IRuleDataClient } from '..'; import { getCommonAlertFields } from './get_common_alert_fields'; import { CreatePersistenceRuleTypeWrapper } from './persistence_types'; @@ -471,9 +472,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper }, {}); // filter out alerts that were already suppressed - // alert was suppressed if its suppression ends is older than suppression end of existing alert - // if existing alert was created earlier during the same rule execution - then alerts can be counted as not suppressed yet - // as they are processed for the first against this existing alert + // alert was suppressed if its suppression ends is older + // than suppression end of existing alert + // if existing alert was created earlier during the same + // rule execution - then alerts can be counted as not suppressed yet + // as they are processed for the first time against this existing alert const nonSuppressedAlerts = filteredDuplicates.filter((alert) => { const existingAlert = existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]]; @@ -544,7 +547,15 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ]; }); - let enrichedAlerts = newAlerts; + // we can now augment and enrich + // the sub alerts (if any) the same as we would + // any other newAlert + let enrichedAlerts = newAlerts.some((newAlert) => newAlert.subAlerts != null) + ? newAlerts.flatMap((newAlert) => { + const { subAlerts, ...everything } = newAlert; + return [everything, ...(subAlerts ?? [])]; + }) + : newAlerts; if (enrichAlerts) { try { diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index 328e5185a2b80..f6e1ae5942b37 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -45,6 +45,10 @@ export type SuppressedAlertService = ( alerts: Array<{ _id: string; _source: T; + subAlerts?: Array<{ + _id: string; + _source: T; + }>; }>, suppressionWindow: string, enrichAlerts?: ( diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 3697359365619..095324840fc5c 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -12,6 +12,17 @@ export type ExperimentalFeatures = { [K in keyof typeof allowedExperimentalValue * This object is then used to validate and parse the value entered. */ export const allowedExperimentalValues = Object.freeze({ + /* + * Enables experimental feature flag for eql sequence alert suppression. + * + * Ticket: https://github.com/elastic/security-team/issues/9608 + * Owners: https://github.com/orgs/elastic/teams/security-detection-engine + * Added: on October 1st, 2024 in https://github.com/elastic/kibana/pull/189725 + * Turned: on (TBD) + * Expires: on (TBD) + */ + alertSuppressionForSequenceEqlRuleEnabled: true, + // FIXME:PT delete? excludePoliciesInFilterEnabled: false, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index dd50d83ce7f09..7b056b5969799 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -145,7 +145,6 @@ const RuleTypeEuiFormRow = styled(EuiFormRow).attrs<{ $isVisible: boolean }>(({ }, }))<{ $isVisible: boolean }>``; -// eslint-disable-next-line complexity const StepDefineRuleComponent: FC = ({ dataSourceType, defaultSavedQuery, @@ -172,7 +171,6 @@ const StepDefineRuleComponent: FC = ({ watch: ['ruleType', 'queryBar', 'machineLearningJobId'], }); - const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); const [indexModified, setIndexModified] = useState(false); const [threatIndexModified, setThreatIndexModified] = useState(false); @@ -358,10 +356,9 @@ const StepDefineRuleComponent: FC = ({ * purpose and so are treated as if the field is always selected. */ const areSuppressionFieldsSelected = isThresholdRule || Boolean(alertSuppressionFields?.length); - const areSuppressionFieldsDisabledBySequence = - isEqlRule(ruleType) && - isEqlSequenceQuery(queryBar?.query?.query as string) && - alertSuppressionFields?.length === 0; + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression( + isEqlSequenceQuery(queryBar?.query?.query as string) + ); /** If we don't have ML field information, users can't meaningfully interact with suppression fields */ const areSuppressionFieldsDisabledByMlFields = @@ -369,30 +366,21 @@ const StepDefineRuleComponent: FC = ({ /** Suppression fields are generally disabled if either: * - License is insufficient (i.e. less than platinum) - * - An EQL Sequence is used * - ML Field information is not available */ const areSuppressionFieldsDisabled = - !isAlertSuppressionLicenseValid || - areSuppressionFieldsDisabledBySequence || - areSuppressionFieldsDisabledByMlFields; + !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledByMlFields; const isSuppressionGroupByDisabled = (areSuppressionFieldsDisabled || isEsqlSuppressionLoading) && !areSuppressionFieldsSelected; const suppressionGroupByDisabledText = useMemo(() => { - if (areSuppressionFieldsDisabledBySequence) { - return i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP; - } else if (areSuppressionFieldsDisabledByMlFields) { + if (areSuppressionFieldsDisabledByMlFields) { return i18n.MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL; } else { return alertSuppressionUpsellingMessage; } - }, [ - alertSuppressionUpsellingMessage, - areSuppressionFieldsDisabledByMlFields, - areSuppressionFieldsDisabledBySequence, - ]); + }, [alertSuppressionUpsellingMessage, areSuppressionFieldsDisabledByMlFields]); const suppressionGroupByFields = useMemo(() => { if (isEsqlRule(ruleType)) { @@ -824,13 +812,14 @@ const StepDefineRuleReadOnlyComponent: FC = ({ }) => { const dataForDescription: Partial = getStepDataDataSource(data); const transformFields = useExperimentalFeatureFieldsTransform(); + const fieldsToDisplay = transformFields(dataForDescription); return ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx index f0db9f0342736..323e321eee8d9 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx @@ -16,8 +16,6 @@ import { customValidators, } from '../../../../common/components/threat_match/helpers'; import { - isEqlRule, - isEqlSequenceQuery, isEsqlRule, isNewTermsRule, isThreatMatchRule, @@ -44,7 +42,6 @@ import { THREAT_MATCH_INDEX_HELPER_TEXT, THREAT_MATCH_REQUIRED, THREAT_MATCH_EMPTIES, - EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT, } from './translations'; import { queryRequiredValidatorFactory } from '../../validators/query_required_validator_factory'; import { kueryValidatorFactory } from '../../validators/kuery_validator_factory'; @@ -587,7 +584,6 @@ export const schema: FormSchema = { validator: (...args: Parameters) => { const [{ formData }] = args; const needsValidation = isSuppressionRuleConfiguredWithGroupBy(formData.ruleType); - if (!needsValidation) { return; } @@ -595,25 +591,6 @@ export const schema: FormSchema = { return alertSuppressionFieldsValidatorFactory()(...args); }, }, - { - validator: ( - ...args: Parameters - ): ReturnType> | undefined => { - const [{ formData, value }] = args; - - if (!isEqlRule(formData.ruleType) || !Array.isArray(value) || value.length === 0) { - return; - } - - const query: string = formData.queryBar?.query?.query ?? ''; - - if (isEqlSequenceQuery(query)) { - return { - message: EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT, - }; - } - }, - }, ], }, [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx index 897331d20da73..362f586dc5abd 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx @@ -176,22 +176,6 @@ export const getEnableThresholdSuppressionLabel = (fields: string[] | undefined) ) ); -export const EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText', - { - defaultMessage: 'Suppression is not supported for EQL sequence queries.', - } -); - -export const EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT = i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText', - { - defaultMessage: - '{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Change the EQL query to a non-sequence query, or remove the suppression fields.', - values: { EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP }, - } -); - export const MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.machineLearningSuppressionDisabledLabel', { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index c035fef5af6e4..d896fa676d31d 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -7,6 +7,14 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { isEqlRule, isEqlSequenceQuery } from '../../../../../common/detection_engine/utils'; +import { + ALERT_SUPPRESSION_FIELDS_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME, + ALERT_SUPPRESSION_DURATION_FIELD_NAME, + ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME, +} from '../../../rule_creation/components/alert_suppression_edit'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -14,9 +22,30 @@ import type { DefineStepRule } from '../../../../detections/pages/detection_engi export const useExperimentalFeatureFieldsTransform = >(): (( fields: T ) => T) => { - const transformer = useCallback((fields: T) => { - return fields; - }, []); + const isAlertSuppressionForSequenceEqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForSequenceEqlRuleEnabled' + ); + const transformer = useCallback( + (fields: T) => { + const isSuppressionDisabled = + isEqlRule(fields.ruleType) && + isEqlSequenceQuery(fields.queryBar?.query?.query as string) && + !isAlertSuppressionForSequenceEqlRuleEnabled; + + // reset any alert suppression values hidden behind feature flag + if (isSuppressionDisabled) { + return { + ...fields, + [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: [], + [ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME]: undefined, + [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: undefined, + [ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME]: undefined, + }; + } + return fields; + }, + [isAlertSuppressionForSequenceEqlRuleEnabled] + ); return transformer; }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx index dbc753ca33d37..0a75fa5c8f319 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx @@ -42,6 +42,7 @@ import { useUserData } from '../../../../detections/components/user_info'; import { StepPanel } from '../../../rule_creation/components/step_panel'; import { StepAboutRule } from '../../components/step_about_rule'; import { StepDefineRule } from '../../components/step_define_rule'; +import { useExperimentalFeatureFieldsTransform } from '../../components/step_define_rule/use_experimental_feature_fields_transform'; import { StepScheduleRule } from '../../components/step_schedule_rule'; import { StepRuleActions } from '../../../rule_creation/components/step_rule_actions'; import { formatRule } from '../rule_creation/helpers'; @@ -52,6 +53,7 @@ import { MaxWidthEuiFlexItem, } from '../../../../detections/pages/detection_engine/rules/helpers'; import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations'; +import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; import { RuleStep } from '../../../../detections/pages/detection_engine/rules/types'; import * as i18n from './translations'; import { SecurityPageName } from '../../../../app/types'; @@ -368,11 +370,16 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { const { startTransaction } = useStartTransaction(); + const defineFieldsTransform = useExperimentalFeatureFieldsTransform(); + const saveChanges = useCallback(async () => { startTransaction({ name: SINGLE_RULE_ACTIONS.SAVE }); + const localDefineStepData: DefineStepRule = defineFieldsTransform({ + ...defineStepData, + }); const updatedRule = await updateRule({ ...formatRule( - defineStepData, + localDefineStepData, aboutStepData, scheduleStepData, actionsStepData, @@ -390,8 +397,9 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => { }, [ aboutStepData, actionsStepData, - addSuccess, defineStepData, + defineFieldsTransform, + addSuccess, navigateToApp, rule?.exceptions_list, ruleId, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx index 70f267ac94ba4..3e08f4ce3acc8 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx @@ -843,7 +843,7 @@ export const RuleDefinitionSection = ({ ruleType: rule.type, }); - const { isSuppressionEnabled } = useAlertSuppression(rule.type); + const { isSuppressionEnabled } = useAlertSuppression(); const definitionSectionListItems = prepareDefinitionSectionListItems( rule, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index 27615ef2d489a..7877a86385cde 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -6,37 +6,23 @@ */ import { renderHook } from '@testing-library/react'; -import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import * as useIsExperimentalFeatureEnabledMock from '../../../common/hooks/use_experimental_features'; import { useAlertSuppression } from './use_alert_suppression'; describe('useAlertSuppression', () => { - ( - [ - 'new_terms', - 'threat_match', - 'saved_query', - 'query', - 'threshold', - 'eql', - 'esql', - 'machine_learning', - ] as Type[] - ).forEach((ruleType) => { - it(`should return the isSuppressionEnabled true for ${ruleType} rule type that exists in SUPPRESSIBLE_ALERT_RULES`, () => { - const { result } = renderHook(() => useAlertSuppression(ruleType)); - - expect(result.current.isSuppressionEnabled).toBe(true); - }); + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockReturnValue(false); + it(`should return the isSuppressionEnabled true if query for all rule types is not an eql sequence query`, () => { + const { result } = renderHook(() => useAlertSuppression()); + expect(result.current.isSuppressionEnabled).toBe(true); }); - it('should return false if rule type is undefined', () => { - const { result } = renderHook(() => useAlertSuppression(undefined)); - expect(result.current.isSuppressionEnabled).toBe(false); - }); - - it('should return false if rule type is not a suppressible rule', () => { - const { result } = renderHook(() => useAlertSuppression('OTHER_RULE_TYPE' as Type)); - + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockReturnValue(false); + it('should return isSuppressionEnabled false for eql sequence query when feature flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression(true)); expect(result.current.isSuppressionEnabled).toBe(false); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index 6e1b2a4d6163f..dae422a8e81c5 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -5,21 +5,24 @@ * 2.0. */ import { useCallback } from 'react'; -import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } -export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { +export const useAlertSuppression = (isEqlSequenceQuery = false): UseAlertSuppressionReturn => { + const isAlertSuppressionForSequenceEQLRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForSequenceEqlRuleEnabled' + ); + const isSuppressionEnabledForRuleType = useCallback(() => { - if (!ruleType) { - return false; + if (isEqlSequenceQuery) { + return isAlertSuppressionForSequenceEQLRuleEnabled; } - return isSuppressibleAlertRule(ruleType); - }, [ruleType]); + return true; + }, [isAlertSuppressionForSequenceEQLRuleEnabled, isEqlSequenceQuery]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts index 84349e9142e22..a2b0d4f21fc78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts @@ -43,19 +43,19 @@ describe('buildAlert', () => { sampleDocNoSortId('619389b2-b077-400e-b40b-abde20d675d3'), ], }; - const alertGroup = buildAlertGroupFromSequence( - ruleExecutionLoggerMock, - eqlSequence, + const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({ + ruleExecutionLogger: ruleExecutionLoggerMock, + sequence: eqlSequence, completeRule, - 'allFields', - SPACE_ID, - jest.fn(), - completeRule.ruleParams.index as string[], - undefined, - PUBLIC_BASE_URL - ); - expect(alertGroup.length).toEqual(3); - expect(alertGroup[0]).toEqual( + mergeStrategy: 'allFields', + spaceId: SPACE_ID, + buildReasonMessage: jest.fn(), + indicesToQuery: completeRule.ruleParams.index as string[], + alertTimestampOverride: undefined, + publicBaseUrl: PUBLIC_BASE_URL, + }); + expect(buildingBlocks.length).toEqual(2); + expect(buildingBlocks[0]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -72,10 +72,10 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[0]._source[ALERT_URL]).toContain( + expect(buildingBlocks[0]?._source?.[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/f2db3574eaf8450e3f4d1cf4f416d70b110b035ae0a7a00026242df07f0a6c90?index=.alerts-security.alerts-space' ); - expect(alertGroup[1]).toEqual( + expect(buildingBlocks[1]).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: [ @@ -92,10 +92,10 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[1]._source[ALERT_URL]).toContain( + expect(buildingBlocks[1]._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1dbc416333244efbda833832eb83f13ea5d980a33c2f981ca8d2b35d82a045da?index=.alerts-security.alerts-space' ); - expect(alertGroup[2]).toEqual( + expect(shellAlert).toEqual( expect.objectContaining({ _source: expect.objectContaining({ [ALERT_ANCESTORS]: expect.arrayContaining([ @@ -113,14 +113,14 @@ describe('buildAlert', () => { }, { depth: 1, - id: alertGroup[0]._id, + id: buildingBlocks[0]?._id, index: '', rule: sampleRuleGuid, type: 'signal', }, { depth: 1, - id: alertGroup[1]._id, + id: buildingBlocks[0]._id, index: '', rule: sampleRuleGuid, type: 'signal', @@ -132,10 +132,10 @@ describe('buildAlert', () => { }), }) ); - expect(alertGroup[2]._source[ALERT_URL]).toContain( + expect(shellAlert?._source[ALERT_URL]).toContain( 'http://testkibanabaseurl.com/s/space/app/security/alerts/redirect/1b7d06954e74257140f3bf73f139078483f9658fe829fd806cc307fc0388fb23?index=.alerts-security.alerts-space' ); - const groupIds = alertGroup.map((alert) => alert._source[ALERT_GROUP_ID]); + const groupIds = buildingBlocks.map((alert) => alert?._source?.[ALERT_GROUP_ID]); for (const groupId of groupIds) { expect(groupId).toEqual(groupIds[0]); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts index 185aa1236a234..2e0948b1af4d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts @@ -5,6 +5,13 @@ * 2.0. */ +import type { + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, +} from '@kbn/rule-data-utils'; import { ALERT_URL, ALERT_UUID } from '@kbn/rule-data-utils'; import { intersection as lodashIntersection, isArray } from 'lodash'; @@ -30,6 +37,35 @@ import type { EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; +import type { SuppressionTerm } from '../utils'; + +export interface ExtraFieldsForShellAlert { + [ALERT_INSTANCE_ID]: string; + [ALERT_SUPPRESSION_TERMS]: SuppressionTerm[]; + [ALERT_SUPPRESSION_START]: Date; + [ALERT_SUPPRESSION_END]: Date; + [ALERT_SUPPRESSION_DOCS_COUNT]: number; +} + +export interface BuildAlertGroupFromSequence { + ruleExecutionLogger: IRuleExecutionLogForExecutors; + sequence: EqlSequence; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + spaceId: string | null | undefined; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + applyOverrides?: boolean; + publicBaseUrl?: string; + intendedTimestamp?: Date; +} + +// eql shell alerts can have a subAlerts property +// when suppression is used in EQL sequence queries +export type WrappedEqlShellOptionalSubAlertsType = WrappedFieldsLatest & { + subAlerts?: Array>; +}; /** * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - @@ -38,25 +74,30 @@ import type { * @param sequence The raw ES documents that make up the sequence * @param completeRule object representing the rule that found the sequence */ -export const buildAlertGroupFromSequence = ( - ruleExecutionLogger: IRuleExecutionLogForExecutors, - sequence: EqlSequence, - completeRule: CompleteRule, - mergeStrategy: ConfigType['alertMergeStrategy'], - spaceId: string | null | undefined, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, - publicBaseUrl?: string, - intendedTimestamp?: Date -): Array> => { +export const buildAlertGroupFromSequence = ({ + ruleExecutionLogger, + sequence, + completeRule, + mergeStrategy, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl, + intendedTimestamp, +}: BuildAlertGroupFromSequence): { + shellAlert: WrappedFieldsLatest | undefined; + buildingBlocks: Array>; +} => { const ancestors: Ancestor[] = sequence.events.flatMap((event) => buildAncestors(event)); if (ancestors.some((ancestor) => ancestor?.rule === completeRule.alertId)) { - return []; + return { shellAlert: undefined, buildingBlocks: [] }; } - // The "building block" alerts start out as regular BaseFields. We'll add the group ID and index fields - // after creating the shell alert later on, since that's when the group ID is determined. + // The "building block" alerts start out as regular BaseFields. + // We'll add the group ID and index fields + // after creating the shell alert later on + // since that's when the group ID is determined. let baseAlerts: BaseFieldsLatest[] = []; try { baseAlerts = sequence.events.map((event) => @@ -79,7 +120,7 @@ export const buildAlertGroupFromSequence = ( ); } catch (error) { ruleExecutionLogger.error(error); - return []; + return { shellAlert: undefined, buildingBlocks: [] }; } // The ID of each building block alert depends on all of the other building blocks as well, @@ -99,16 +140,16 @@ export const buildAlertGroupFromSequence = ( // Now that we have an array of building blocks for the events in the sequence, // we can build the signal that links the building blocks together // and also insert the group id (which is also the "shell" signal _id) in each building block - const shellAlert = buildAlertRoot( - wrappedBaseFields, + const shellAlert = buildAlertRoot({ + wrappedBuildingBlocks: wrappedBaseFields, completeRule, spaceId, buildReasonMessage, indicesToQuery, alertTimestampOverride, publicBaseUrl, - intendedTimestamp - ); + intendedTimestamp, + }); const sequenceAlert: WrappedFieldsLatest = { _id: shellAlert[ALERT_UUID], _index: '', @@ -139,19 +180,30 @@ export const buildAlertGroupFromSequence = ( } ); - return [...wrappedBuildingBlocks, sequenceAlert]; + return { shellAlert: sequenceAlert, buildingBlocks: wrappedBuildingBlocks }; }; -export const buildAlertRoot = ( - wrappedBuildingBlocks: Array>, - completeRule: CompleteRule, - spaceId: string | null | undefined, - buildReasonMessage: BuildReasonMessage, - indicesToQuery: string[], - alertTimestampOverride: Date | undefined, - publicBaseUrl?: string, - intendedTimestamp?: Date -): EqlShellFieldsLatest => { +export interface BuildAlertRootParams { + wrappedBuildingBlocks: Array>; + completeRule: CompleteRule; + spaceId: string | null | undefined; + buildReasonMessage: BuildReasonMessage; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + publicBaseUrl?: string; + intendedTimestamp?: Date; +} + +export const buildAlertRoot = ({ + wrappedBuildingBlocks, + completeRule, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl, + intendedTimestamp, +}: BuildAlertRootParams): EqlShellFieldsLatest => { const mergedAlerts = objectArrayIntersection(wrappedBuildingBlocks.map((alert) => alert._source)); const reason = buildReasonMessage({ name: completeRule.ruleConfig.name, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts index 2bdf0d913d156..79bc9c9849da1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts @@ -13,9 +13,10 @@ import { EqlRuleParams } from '../../rule_schema'; import { eqlExecutor } from './eql'; import type { CreateRuleOptions, SecurityAlertType, SignalSourceHit } from '../types'; import { validateIndexPatterns } from '../utils'; -import type { BuildReasonMessage } from '../utils/reason_formatters'; -import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; +import type { SharedParams } from '../utils/utils'; +import { wrapSuppressedAlerts } from '../utils/wrap_suppressed_alerts'; +import type { BuildReasonMessage } from '../utils/reason_formatters'; export const createEqlAlertType = ( createOptions: CreateRuleOptions @@ -86,6 +87,24 @@ export const createEqlAlertType = ( spaceId, } = execOptions; + const isAlertSuppressionActive = await getIsAlertSuppressionActive({ + alertSuppression: completeRule.ruleParams.alertSuppression, + licensing, + }); + + const sharedParams: SharedParams = { + spaceId, + completeRule, + mergeStrategy, + indicesToQuery: inputIndex, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + intendedTimestamp, + }; + const wrapSuppressedHits = ( events: SignalSourceHit[], buildReasonMessage: BuildReasonMessage @@ -104,10 +123,7 @@ export const createEqlAlertType = ( secondaryTimestamp, intendedTimestamp, }); - const isNonSeqAlertSuppressionActive = await getIsAlertSuppressionActive({ - alertSuppression: completeRule.ruleParams.alertSuppression, - licensing, - }); + const { result, loggedRequests } = await eqlExecutor({ completeRule, tuple, @@ -124,9 +140,10 @@ export const createEqlAlertType = ( exceptionFilter, unprocessedExceptions, wrapSuppressedHits, + sharedParams, alertTimestampOverride, alertWithSuppression, - isAlertSuppressionActive: isNonSeqAlertSuppressionActive, + isAlertSuppressionActive, experimentalFeatures, state, scheduleNotificationResponseActionsService, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts index 4f5aa7d322c9e..f1fabcfd0f96b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.test.ts @@ -18,6 +18,7 @@ import { getCompleteRuleMock, getEqlRuleParams } from '../../rule_schema/mocks'; import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; import { eqlExecutor } from './eql'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; +import type { SharedParams } from '../utils/utils'; jest.mock('../../routes/index/get_index_version'); jest.mock('../utils/get_data_tier_filter', () => ({ getDataTierFilter: jest.fn() })); @@ -38,6 +39,21 @@ describe('eql_executor', () => { }; const mockExperimentalFeatures = {} as ExperimentalFeatures; const mockScheduleNotificationResponseActionsService = jest.fn(); + const ruleExecutionLoggerMock = ruleExecutionLogMock.forExecutors.create(); + const SPACE_ID = 'space'; + const PUBLIC_BASE_URL = 'http://testkibanabaseurl.com'; + + const sharedParams: SharedParams = { + ruleExecutionLogger: ruleExecutionLoggerMock, + completeRule: eqlCompleteRule, + mergeStrategy: 'allFields', + spaceId: SPACE_ID, + indicesToQuery: eqlCompleteRule.ruleParams.index as string[], + alertTimestampOverride: undefined, + publicBaseUrl: PUBLIC_BASE_URL, + intendedTimestamp: undefined, + primaryTimestamp: new Date().toISOString(), + }; beforeEach(() => { jest.clearAllMocks(); @@ -69,6 +85,7 @@ describe('eql_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [getExceptionListItemSchemaMock()], wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, @@ -82,56 +99,6 @@ describe('eql_executor', () => { }`, ]); }); - - it('warns when a sequence query is used with alert suppression', async () => { - // mock a sequences response - alertServices.scopedClusterClient.asCurrentUser.eql.search.mockReset().mockResolvedValue({ - hits: { - total: { relation: 'eq', value: 10 }, - sequences: [], - }, - }); - - const ruleWithSequenceAndSuppression = getCompleteRuleMock({ - ...params, - query: 'sequence [any where true] [any where true]', - alertSuppression: { - groupBy: ['event.type'], - duration: { - value: 10, - unit: 'm', - }, - missingFieldsStrategy: 'suppress', - }, - }); - - const { result } = await eqlExecutor({ - inputIndex: DEFAULT_INDEX_PATTERN, - runtimeMappings: {}, - completeRule: ruleWithSequenceAndSuppression, - tuple, - ruleExecutionLogger, - services: alertServices, - version, - bulkCreate: jest.fn(), - wrapHits: jest.fn(), - wrapSequences: jest.fn(), - primaryTimestamp: '@timestamp', - exceptionFilter: undefined, - unprocessedExceptions: [], - wrapSuppressedHits: jest.fn(), - alertTimestampOverride: undefined, - alertWithSuppression: jest.fn(), - isAlertSuppressionActive: true, - experimentalFeatures: mockExperimentalFeatures, - scheduleNotificationResponseActionsService: - mockScheduleNotificationResponseActionsService, - }); - - expect(result.warningMessages).toContain( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - ); - }); }); it('should classify EQL verification exceptions as "user errors" when reporting to the framework', async () => { @@ -155,6 +122,7 @@ describe('eql_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [], wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, @@ -180,6 +148,7 @@ describe('eql_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [], wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: false, @@ -220,6 +189,7 @@ describe('eql_executor', () => { exceptionFilter: undefined, unprocessedExceptions: [], wrapSuppressedHits: jest.fn(), + sharedParams, alertTimestampOverride: undefined, alertWithSuppression: jest.fn(), isAlertSuppressionActive: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts index cd8b76a93d23b..756f220f06d55 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/eql.ts @@ -5,6 +5,7 @@ * 2.0. */ import { performance } from 'perf_hooks'; + import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; import type { @@ -25,9 +26,10 @@ import type { RuleRangeTuple, SearchAfterAndBulkCreateReturnType, SignalSource, - WrapSuppressedHits, CreateRuleOptions, + WrapSuppressedHits, } from '../types'; +import type { SharedParams } from '../utils/utils'; import { addToSearchAfterReturn, createSearchAfterReturnType, @@ -44,11 +46,15 @@ import type { WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; -import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; +import { + bulkCreateSuppressedAlertsInMemory, + bulkCreateSuppressedSequencesInMemory, +} from '../utils/bulk_create_suppressed_alerts_in_memory'; import { getDataTierFilter } from '../utils/get_data_tier_filter'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import { logEqlRequest } from '../utils/logged_requests'; import * as i18n from '../translations'; +import { alertSuppressionTypeGuard } from '../utils/get_is_alert_suppression_active'; interface EqlExecutorParams { inputIndex: string[]; @@ -60,6 +66,7 @@ interface EqlExecutorParams { version: string; bulkCreate: BulkCreate; wrapHits: WrapHits; + sharedParams: SharedParams; wrapSequences: WrapSequences; primaryTimestamp: string; secondaryTimestamp?: string; @@ -90,6 +97,7 @@ export const eqlExecutor = async ({ exceptionFilter, unprocessedExceptions, wrapSuppressedHits, + sharedParams, alertTimestampOverride, alertWithSuppression, isAlertSuppressionActive, @@ -104,6 +112,7 @@ export const eqlExecutor = async ({ const isLoggedRequestsEnabled = state?.isLoggedRequestsEnabled ?? false; const loggedRequests: RulePreviewLoggedRequest[] = []; + // eslint-disable-next-line complexity return withSecuritySpan('eqlExecutor', async () => { const result = createSearchAfterReturnType(); @@ -179,12 +188,28 @@ export const eqlExecutor = async ({ newSignals = wrapHits(events, buildReasonMessageForEqlAlert); } } else if (sequences) { - if (isAlertSuppressionActive) { - result.warningMessages.push( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' - ); + if ( + isAlertSuppressionActive && + experimentalFeatures.alertSuppressionForSequenceEqlRuleEnabled && + alertSuppressionTypeGuard(completeRule.ruleParams.alertSuppression) + ) { + await bulkCreateSuppressedSequencesInMemory({ + sequences, + toReturn: result, + bulkCreate, + services, + buildReasonMessage: buildReasonMessageForEqlAlert, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + sharedParams, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + }); + } else { + newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } - newSignals = wrapSequences(sequences, buildReasonMessageForEqlAlert); } else { throw new Error( 'eql query response should have either `sequences` or `events` but had neither' diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts index 39f9df366627e..d7ce37dd1feb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/wrap_sequences_factory.ts @@ -11,7 +11,8 @@ import type { ConfigType } from '../../../../config'; import type { CompleteRule, RuleParams } from '../../rule_schema'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import type { - BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; @@ -38,21 +39,27 @@ export const wrapSequencesFactory = intendedTimestamp: Date | undefined; }): WrapSequences => (sequences, buildReasonMessage) => - sequences.reduce( - (acc: Array>, sequence) => [ - ...acc, - ...buildAlertGroupFromSequence( - ruleExecutionLogger, - sequence, - completeRule, - mergeStrategy, - spaceId, - buildReasonMessage, - indicesToQuery, - alertTimestampOverride, - publicBaseUrl, - intendedTimestamp - ), - ], - [] - ); + sequences.reduce< + Array< + | WrappedFieldsLatest + | WrappedFieldsLatest + > + >((acc, sequence) => { + const { shellAlert, buildingBlocks } = buildAlertGroupFromSequence({ + ruleExecutionLogger, + sequence, + completeRule, + mergeStrategy, + spaceId, + buildReasonMessage, + indicesToQuery, + alertTimestampOverride, + publicBaseUrl, + intendedTimestamp, + }); + if (shellAlert) { + acc.push(shellAlert, ...buildingBlocks); + return acc; + } + return acc; + }, []); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts index b2d01e4d5ee7a..fe72aa566583a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -61,7 +61,7 @@ export const wrapSuppressedEsqlAlerts = ({ const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: combinedFields, + input: combinedFields, }); const id = generateAlertId({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts index fa3781b2a2e63..dc9a322301126 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/wrap_suppressed_new_terms_alerts.ts @@ -58,7 +58,7 @@ export const wrapSuppressedNewTermsAlerts = ({ const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: event.fields, + input: event.fields, }); const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts index 8c235c5e8f238..106a7589a2826 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts @@ -65,6 +65,7 @@ export const queryExecutor = async ({ const hasPlatinumLicense = license.hasAtLeast('platinum'); const result = + // TODO: replace this with getIsAlertSuppressionActive function ruleParams.alertSuppression?.groupBy != null && hasPlatinumLicense ? await groupAndBulkCreate({ runOpts, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index c1e8165fe3aed..94aa55e6b0dd5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -52,6 +52,8 @@ import type { BuildReasonMessage } from './utils/reason_formatters'; import type { BaseFieldsLatest, DetectionAlert, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../common/api/detection_engine/model/alerts'; import type { @@ -354,7 +356,9 @@ export type WrapSuppressedHits = ( export type WrapSequences = ( sequences: Array>, buildReasonMessage: BuildReasonMessage -) => Array>; +) => Array< + WrappedFieldsLatest | WrappedFieldsLatest +>; export type RuleServices = RuleExecutorServices< AlertInstanceState, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 030cb213d94dd..291b5cfa0d2f6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -7,14 +7,18 @@ import type { SuppressedAlertService } from '@kbn/rule-registry-plugin/server'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; +import type { EqlHitsSequence } from '@elastic/elasticsearch/lib/api/types'; + import type { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType, WrapSuppressedHits, SignalSourceHit, + SignalSource, } from '../types'; import { MAX_SIGNALS_SUPPRESSION_MULTIPLIER } from '../constants'; -import { addToSearchAfterReturn } from './utils'; +import type { SharedParams } from './utils'; +import { addToSearchAfterReturn, buildShellAlertSuppressionTermsAndFields } from './utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; import { DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY } from '../../../../../common/detection_engine/constants'; import { partitionMissingFieldsEvents } from './partition_missing_fields_events'; @@ -25,8 +29,12 @@ import type { ExperimentalFeatures } from '../../../../../common'; import type { BaseFieldsLatest, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; +import { buildAlertGroupFromSequence } from '../eql/build_alert_group_from_sequence'; interface SearchAfterAndBulkCreateSuppressedAlertsParams extends SearchAfterAndBulkCreateParams { wrapSuppressedHits: WrapSuppressedHits; @@ -54,9 +62,31 @@ export interface BulkCreateSuppressedAlertsParams mergeSourceAndFields?: boolean; maxNumberOfAlertsMultiplier?: number; } + +export interface BulkCreateSuppressedSequencesParams + extends Pick< + SearchAfterAndBulkCreateSuppressedAlertsParams, + | 'bulkCreate' + | 'services' + | 'buildReasonMessage' + | 'ruleExecutionLogger' + | 'tuple' + | 'alertSuppression' + | 'alertWithSuppression' + | 'alertTimestampOverride' + > { + sequences: Array>; + buildingBlockAlerts?: Array>; + toReturn: SearchAfterAndBulkCreateReturnType; + experimentalFeatures: ExperimentalFeatures; + maxNumberOfAlertsMultiplier?: number; + sharedParams: SharedParams; + alertSuppression: AlertSuppressionCamel; +} /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. - * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, regular alerts will be created for such events without suppression + * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, + * regular alerts will be created for such events without suppression */ export const bulkCreateSuppressedAlertsInMemory = async ({ enrichedEvents, @@ -112,6 +142,90 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ }); }; +/** + * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. + * If parameter alertSuppression.missingFieldsStrategy configured not to be suppressed, + * regular alerts will be created for such events without suppression + */ +export const bulkCreateSuppressedSequencesInMemory = async ({ + sequences, + toReturn, + bulkCreate, + services, + ruleExecutionLogger, + tuple, + alertSuppression, + buildReasonMessage, + sharedParams, + alertWithSuppression, + alertTimestampOverride, + experimentalFeatures, + maxNumberOfAlertsMultiplier, +}: BulkCreateSuppressedSequencesParams) => { + const suppressOnMissingFields = + (alertSuppression.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === + AlertSuppressionMissingFieldsStrategyEnum.suppress; + + const suppressibleWrappedSequences: Array< + WrappedFieldsLatest & { + subAlerts: Array>; + } + > = []; + const unsuppressibleWrappedDocs: Array> = []; + + sequences.forEach((sequence) => { + const alertGroupFromSequence = buildAlertGroupFromSequence({ + sequence, + applyOverrides: true, + buildReasonMessage, + ...sharedParams, + }); + const shellAlert = alertGroupFromSequence.shellAlert; + const buildingBlocks = alertGroupFromSequence.buildingBlocks; + if (shellAlert) { + if (!suppressOnMissingFields) { + // does the shell alert have all the suppression fields? + const hasEverySuppressionField = alertSuppression.groupBy.every( + (suppressionPath) => + robustGet({ key: suppressionPath, document: shellAlert._source }) != null + ); + if (!hasEverySuppressionField) { + unsuppressibleWrappedDocs.push(shellAlert, ...buildingBlocks); + } else { + const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({ + shellAlert, + buildingBlockAlerts: buildingBlocks, + ...sharedParams, + }); + suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); + } + } else { + const wrappedWithSuppressionTerms = buildShellAlertSuppressionTermsAndFields({ + shellAlert, + buildingBlockAlerts: buildingBlocks, + ...sharedParams, + }); + suppressibleWrappedSequences.push(wrappedWithSuppressionTerms); + } + } + }); + + return executeBulkCreateAlerts({ + suppressibleWrappedDocs: suppressibleWrappedSequences, + unsuppressibleWrappedDocs, + toReturn, + bulkCreate, + services, + ruleExecutionLogger, + tuple, + alertSuppression, + alertWithSuppression, + alertTimestampOverride, + experimentalFeatures, + maxNumberOfAlertsMultiplier, + }); +}; + export interface ExecuteBulkCreateAlertsParams extends Pick< SearchAfterAndBulkCreateSuppressedAlertsParams, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts index 7decfd8294913..6c7228d6b78af 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_with_suppression.ts @@ -53,7 +53,7 @@ export const bulkCreateWithSuppression = async < }: { alertWithSuppression: SuppressedAlertService; ruleExecutionLogger: IRuleExecutionLogForExecutors; - wrappedDocs: Array>; + wrappedDocs: Array & { subAlerts?: Array> }>; services: RuleServices; suppressionWindow: string; alertTimestampOverride: Date | undefined; @@ -97,13 +97,19 @@ export const bulkCreateWithSuppression = async < } }; + const alerts = wrappedDocs.map((doc) => ({ + _id: doc._id, + // `fields` should have already been merged into `doc._source` + _source: doc._source, + subAlerts: + doc?.subAlerts != null + ? doc?.subAlerts?.map((subAlert) => ({ _id: subAlert._id, _source: subAlert._source })) + : undefined, + })); + const { createdAlerts, errors, suppressedAlerts, alertsWereTruncated } = await alertWithSuppression( - wrappedDocs.map((doc) => ({ - _id: doc._id, - // `fields` should have already been merged into `doc._source` - _source: doc._source, - })), + alerts, suppressionWindow, enrichAlertsWrapper, alertTimestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts index 5ab06db3043af..263bd88bc1b0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_is_alert_suppression_active.ts @@ -17,6 +17,10 @@ interface GetIsAlertSuppressionActiveParams { licensing: LicensingPluginSetup; } +export const alertSuppressionTypeGuard = ( + alertSuppression: AlertSuppressionCamel | undefined +): alertSuppression is AlertSuppressionCamel => Boolean(alertSuppression?.groupBy?.length); + /** * checks if alert suppression is active: * - rule should have alert suppression config @@ -32,7 +36,7 @@ export const getIsAlertSuppressionActive = async ({ return false; } - const isAlertSuppressionConfigured = Boolean(alertSuppression?.groupBy?.length); + const isAlertSuppressionConfigured = alertSuppressionTypeGuard(alertSuppression); if (!isAlertSuppressionConfigured) { return false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts index 646068b59eec7..1b4b9e7d49174 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/source_fields_merging/utils/robust_field_access.test.ts @@ -34,6 +34,9 @@ describe('robust field access', () => { it('returns undefined if the key does not exist', () => { expect(robustGet({ key: 'a.b.c', document: { a: { b: 'my-value' } } })).toEqual(undefined); }); + it('returns an array if the key exists', () => { + expect(robustGet({ key: 'a.b', document: { a: { b: ['my-value'] } } })).toEqual(['my-value']); + }); }); describe('set', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts index 745dd08977520..46aa25d08d44a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.test.ts @@ -85,17 +85,37 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.name': 'localhost-1' }, + input: { 'host.name': 'localhost-1' }, }) ).toEqual([{ field: 'host.name', value: 'localhost-1' }]); }); + it('should return suppression terms when using source', () => { + expect( + getSuppressionTerms({ + alertSuppression: { + groupBy: ['host.name'], + }, + input: { host: { name: 'localhost-1' } }, + }) + ).toEqual([{ field: 'host.name', value: 'localhost-1' }]); + }); + it('should return suppression terms when using source and mixed notation', () => { + expect( + getSuppressionTerms({ + alertSuppression: { + groupBy: ['host.something.name'], + }, + input: { 'host.something': { name: 'localhost-1' } }, + }) + ).toEqual([{ field: 'host.something.name', value: 'localhost-1' }]); + }); it('should return suppression terms array when fields do not have matches', () => { expect( getSuppressionTerms({ alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.ip': '127.0.0.1' }, + input: { 'host.ip': '127.0.0.1' }, }) ).toEqual([{ field: 'host.name', value: null }]); }); @@ -105,7 +125,7 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name'], }, - fields: { 'host.name': ['localhost-2', 'localhost-1'] }, + input: { 'host.name': ['localhost-2', 'localhost-1'] }, }) ).toEqual([{ field: 'host.name', value: ['localhost-1', 'localhost-2'] }]); }); @@ -115,7 +135,7 @@ describe('getSuppressionTerms', () => { alertSuppression: { groupBy: ['host.name', 'host.ip'], }, - fields: { 'host.name': ['localhost-1'], 'agent.name': 'test', 'host.ip': '127.0.0.1' }, + input: { 'host.name': ['localhost-1'], 'agent.name': 'test', 'host.ip': '127.0.0.1' }, }) ).toEqual([ { field: 'host.name', value: ['localhost-1'] }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 44febba73e68e..0e066069665dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -5,7 +5,6 @@ * 2.0. */ -import pick from 'lodash/pick'; import get from 'lodash/get'; import sortBy from 'lodash/sortBy'; @@ -17,10 +16,13 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; +import type { ExtraFieldsForShellAlert } from '../eql/build_alert_group_from_sequence'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; +import type { SearchTypes } from '../../../../../common/detection_engine/types'; export interface SuppressionTerm { field: string; - value: string[] | number[] | null; + value: SearchTypes | null; } /** @@ -47,7 +49,7 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp ); - const suppressionFields = { + const suppressionFields: ExtraFieldsForShellAlert = { [ALERT_INSTANCE_ID]: instanceId, [ALERT_SUPPRESSION_TERMS]: suppressionTerms, [ALERT_SUPPRESSION_START]: suppressionTime, @@ -59,23 +61,21 @@ export const getSuppressionAlertFields = ({ }; /** + * generates values from a source event for the fields provided in the alertSuppression object + * @param alertSuppression {@link AlertSuppressionCamel} options defining how to suppress alerts + * @param input source data from either the _source of "fields" property on the event * returns an array of {@link SuppressionTerm}s by retrieving the appropriate field values based on the provided alertSuppression configuration */ export const getSuppressionTerms = ({ alertSuppression, - fields, + input, }: { - fields: Record | undefined; alertSuppression: AlertSuppressionCamel | undefined; + input: Record | undefined; }): SuppressionTerm[] => { const suppressedBy = alertSuppression?.groupBy ?? []; - - const suppressedProps = pick(fields, suppressedBy) as Record< - string, - string[] | number[] | undefined - >; const suppressionTerms = suppressedBy.map((field) => { - const value = suppressedProps[field] ?? null; + const value = input != null ? robustGet({ document: input, key: field }) ?? null : null; const sortedValue = Array.isArray(value) ? (sortBy(value) as string[] | number[]) : value; return { field, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index bf0899978f2b2..271a2ce64883e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -5,14 +5,25 @@ * 2.0. */ import { createHash } from 'crypto'; -import { chunk, get, invert, isEmpty, partition } from 'lodash'; +import { chunk, get, invert, isEmpty, merge, partition } from 'lodash'; import moment from 'moment'; +import objectHash from 'object-hash'; import dateMath from '@kbn/datemath'; import { isCCSRemoteIndexName } from '@kbn/es-query'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportResult } from '@elastic/elasticsearch'; -import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; +import { + ALERT_UUID, + ALERT_RULE_UUID, + ALERT_RULE_PARAMETERS, + TIMESTAMP, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_TERMS, +} from '@kbn/rule-data-utils'; import type { ListArray, ExceptionListItemSchema, @@ -33,6 +44,7 @@ import type { import { parseDuration } from '@kbn/alerting-plugin/server'; import type { ExceptionListClient, ListClient, ListPluginSetup } from '@kbn/lists-plugin/server'; import type { SanitizedRuleAction } from '@kbn/alerting-plugin/common'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; import type { TimestampOverride } from '../../../../../common/api/detection_engine/model/rule_schema'; import type { Privilege } from '../../../../../common/api/detection_engine'; import { RuleExecutionStatusEnum } from '../../../../../common/api/detection_engine/rule_monitoring'; @@ -51,6 +63,7 @@ import type { } from '../types'; import type { ShardError } from '../../../types'; import type { + CompleteRule, EqlRuleParams, EsqlRuleParams, MachineLearningRuleParams, @@ -65,9 +78,20 @@ import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { BaseFieldsLatest, DetectionAlert, + EqlBuildingBlockFieldsLatest, + EqlShellFieldsLatest, + WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; import { ENABLE_CCS_READ_WARNING_SETTING } from '../../../../../common/constants'; import type { GenericBulkCreateResponse } from '../factories'; +import type { ConfigType } from '../../../../config'; +import type { + ExtraFieldsForShellAlert, + WrappedEqlShellOptionalSubAlertsType, +} from '../eql/build_alert_group_from_sequence'; +import type { BuildReasonMessage } from './reason_formatters'; +import { getSuppressionTerms } from './suppression_utils'; +import { robustGet } from './source_fields_merging/utils/robust_field_access'; export const MAX_RULE_GAP_RATIO = 4; @@ -1027,3 +1051,96 @@ export const getDisabledActionsWarningText = ({ return `${alertsGeneratedText} connector ${actionTypesJoined} is not enabled. To send notifications, you need a higher Security Analytics license / tier`; } }; + +export interface SharedParams { + spaceId: string; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; + intendedTimestamp: Date | undefined; +} + +export type RuleWithInMemorySuppression = + | ThreatRuleParams + | EqlRuleParams + | MachineLearningRuleParams; + +export interface SequenceSuppressionTermsAndFieldsParams { + shellAlert: WrappedFieldsLatest; + buildingBlockAlerts: Array>; + spaceId: string; + completeRule: CompleteRule; + indicesToQuery: string[]; + alertTimestampOverride: Date | undefined; + primaryTimestamp: string; + secondaryTimestamp?: string; +} + +export type SequenceSuppressionTermsAndFieldsFactory = ( + shellAlert: WrappedEqlShellOptionalSubAlertsType, + buildingBlockAlerts: Array>, + buildReasonMessage: BuildReasonMessage +) => WrappedFieldsLatest & { + subAlerts: Array>; +}; + +export const buildShellAlertSuppressionTermsAndFields = ({ + shellAlert, + buildingBlockAlerts, + spaceId, + completeRule, + alertTimestampOverride, + primaryTimestamp, + secondaryTimestamp, +}: SequenceSuppressionTermsAndFieldsParams): WrappedFieldsLatest< + EqlShellFieldsLatest & SuppressionFieldsLatest +> & { + subAlerts: Array>; +} => { + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + input: shellAlert._source, + }); + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const primarySuppressionTime = robustGet({ + key: primaryTimestamp, + document: shellAlert._source, + }) as string | undefined; + + const secondarySuppressionTime = + secondaryTimestamp && + (robustGet({ + key: secondaryTimestamp, + document: shellAlert._source, + }) as string | undefined); + + const suppressionTime = new Date( + primarySuppressionTime ?? + secondarySuppressionTime ?? + alertTimestampOverride ?? + shellAlert._source[TIMESTAMP] + ); + + const suppressionFields: ExtraFieldsForShellAlert = { + [ALERT_INSTANCE_ID]: instanceId, + [ALERT_SUPPRESSION_TERMS]: suppressionTerms, + [ALERT_SUPPRESSION_START]: suppressionTime, + [ALERT_SUPPRESSION_END]: suppressionTime, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }; + + merge(shellAlert._source, suppressionFields); + + return { + _id: shellAlert._id, + _index: shellAlert._index, + _source: shellAlert._source as EqlShellFieldsLatest & SuppressionFieldsLatest, + subAlerts: buildingBlockAlerts, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts index 2e40026c38e4a..6cb176ba70f77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/wrap_suppressed_alerts.ts @@ -9,28 +9,19 @@ import objectHash from 'object-hash'; import { TIMESTAMP } from '@kbn/rule-data-utils'; import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; -import type { SignalSourceHit } from '../types'; +import type { SignalSourceHit } from '../types'; import type { BaseFieldsLatest, WrappedFieldsLatest, } from '../../../../../common/api/detection_engine/model/alerts'; -import type { ConfigType } from '../../../../config'; -import type { - CompleteRule, - EqlRuleParams, - MachineLearningRuleParams, - ThreatRuleParams, -} from '../../rule_schema'; -import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; + import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert'; import { getSuppressionAlertFields, getSuppressionTerms } from './suppression_utils'; +import type { SharedParams } from './utils'; import { generateId } from './utils'; - import type { BuildReasonMessage } from './reason_formatters'; -type RuleWithInMemorySuppression = ThreatRuleParams | EqlRuleParams | MachineLearningRuleParams; - /** * wraps suppressed alerts * creates instanceId hash, which is used to search on time interval alerts @@ -51,22 +42,12 @@ export const wrapSuppressedAlerts = ({ intendedTimestamp, }: { events: SignalSourceHit[]; - spaceId: string; - completeRule: CompleteRule; - mergeStrategy: ConfigType['alertMergeStrategy']; - indicesToQuery: string[]; buildReasonMessage: BuildReasonMessage; - alertTimestampOverride: Date | undefined; - ruleExecutionLogger: IRuleExecutionLogForExecutors; - publicBaseUrl: string | undefined; - primaryTimestamp: string; - secondaryTimestamp?: string; - intendedTimestamp: Date | undefined; -}): Array> => { +} & SharedParams): Array> => { return events.map((event) => { const suppressionTerms = getSuppressionTerms({ alertSuppression: completeRule?.ruleParams?.alertSuppression, - fields: event.fields, + input: event.fields, }); const id = generateId( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh index e8f5c627d3c88..b40c416943295 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh @@ -22,6 +22,7 @@ do { curl -s -k \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: 123' \ + -H 'elastic-api-version: 2023-10-31' \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/rules \ -d @${RULE} \ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json new file mode 100644 index 0000000000000..8c5531188379b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query.json @@ -0,0 +1,62 @@ +{ + "name": "EQL sequence duration", + "description": "Rule with an eql query", + "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], + "rule_id": "rule-id-eql-1", + "enabled": true, + "index": ["auditbeat*", "packetbeat*"], + "interval": "30s", + "query": "sequence with maxspan=10m [any where agent.type == \"auditbeat\"] [any where event.category == \"network_traffic\"]", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 99, + "to": "now", + "from": "now-120s", + "severity": "high", + "type": "eql", + "language": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": ["http://www.example.com/some-article-about-attack"], + "alert_suppression": { + "group_by": ["agent.name"], + "duration": { "value": 5, "unit": "h" }, + "missing_fields_strategy": "suppress" + }, + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json new file mode 100644 index 0000000000000..a3df1cfe24938 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/sequence_eql_query_no_duration.json @@ -0,0 +1,61 @@ +{ + "name": "EQL sequence no duration", + "description": "Rule with an eql query", + "false_positives": ["https://www.example.com/some-article-about-a-false-positive"], + "rule_id": "rule-id-eql-2", + "enabled": true, + "index": ["auditbeat*", "packetbeat*"], + "interval": "30s", + "query": "sequence with maxspan=10m [any where agent.type == \"auditbeat\"] [any where event.category == \"network_traffic\"]", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 99, + "to": "now", + "from": "now-120s", + "severity": "high", + "type": "eql", + "language": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": ["http://www.example.com/some-article-about-attack"], + "alert_suppression": { + "group_by": ["agent.name"], + "missing_fields_strategy": "suppress" + }, + "version": 1 +} diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 9eb1d589d3cce..c55448084128f 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -37836,8 +37836,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "Au moins une correspondance d'indicateur est requise.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "Requête EQL", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "Une requête EQL est requise.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "La suppression n'est pas prise en charge pour les requêtes de séquence EQL.", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Remplacez la requête EQL par une requête non séquentielle ou supprimez les champs de suppression.", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "Seuil de score d'anomalie", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Tâche de Machine Learning", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "Requête personnalisée", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index dc9158040da92..47ee6963c7380 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -37693,8 +37693,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "1 つ以上のインジケーター一致が必要です。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL クエリ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQLクエリは必須です。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQLシーケンスクエリでは抑制はサポートされていません。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} EQLクエリを非シーケンスクエリに変更するか、抑制フィールドを削除してください。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "異常スコアしきい値", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "機械学習ジョブ", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "カスタムクエリー", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 8341cc8aebead..9bb107dd5e8d0 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -37120,8 +37120,6 @@ "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError": "至少需要一个指标匹配。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.EqlQueryBarLabel": "EQL 查询", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlQueryFieldRequiredError": "EQL 查询必填。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText": "EQL 序列查询不支持阻止。", - "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText": "{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} 将 EQL 查询更改为非序列查询，或移除阻止字段。", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldAnomalyThresholdLabel": "异常分数阈值", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldMachineLearningJobIdLabel": "Machine Learning 作业", "xpack.securitySolution.detectionEngine.createRule.stepDefineRule.fieldQuerBarLabel": "定制查询", diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index a0d2ee79a7b46..25b7a40c9701e 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -84,6 +84,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s 'previewTelemetryUrlEnabled', 'riskScoringPersistence', 'riskScoringRoutesEnabled', + 'alertSuppressionForSequenceEqlRuleEnabled', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify(PRECONFIGURED_ACTION_CONNECTORS)}`, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts index 0c3069b3c3b62..d933b1b7274d5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts @@ -8,8 +8,12 @@ import expect from 'expect'; import { v4 as uuidv4 } from 'uuid'; import sortBy from 'lodash/sortBy'; +import partition from 'lodash/partition'; -import { EqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { + DetectionAlert, + EqlRuleCreateProps, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; import { ALERT_SUPPRESSION_START, ALERT_SUPPRESSION_END, @@ -23,6 +27,8 @@ import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_U import { getSuppressionMaxSignalsWarning as getSuppressionMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { RiskEnrichmentFields } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/enrichments/types'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/types'; import { createRule, deleteAllRules, @@ -45,11 +51,13 @@ import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/ut const getQuery = (id: string) => `any where id == "${id}"`; const getSequenceQuery = (id: string) => - `sequence by id [any where id == "${id}"] [any where id == "${id}"]`; + `sequence [any where id == "${id}"] [any where id == "${id}"]`; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); + const esDeleteAllIndices = getService('esDeleteAllIndices'); + const es = getService('es'); const log = getService('log'); const { @@ -61,6 +69,10 @@ export default ({ getService }: FtrProviderContext) => { log, }); + const partitionSequenceBuildingBlocks = ( + alerts: Array> + ) => partition(alerts, (alert) => alert?._source?.['kibana.alert.group.index'] == null); + // NOTE: Add to second quality gate after feature is GA describe('@ess @serverless Alert Suppression for EQL rules', () => { before(async () => { @@ -69,10 +81,14 @@ export default ({ getService }: FtrProviderContext) => { after(async () => { await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + await esDeleteAllIndices('.preview.alerts*'); }); afterEach(async () => { - await deleteAllAlerts(supertest, log, es); + await deleteAllAlerts(supertest, log, es, [ + '.preview.alerts-security.alerts-*', + '.alerts-security.alerts-*', + ]); await deleteAllRules(supertest, log); }); @@ -1778,40 +1794,1507 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('sequence queries', () => { - it('logs a warning if suppression is configured', async () => { + describe('@skipInServerless sequence queries with suppression "per rule execution"', () => { + it('suppresses alerts in a given rule execution', async () => { const id = uuidv4(); - await indexGeneratedSourceDocuments({ - docsCount: 10, - seed: () => ({ id }), + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp2, + }; + + // sequence alert 1 is made up of doc1 and doc1WithLaterTimestamp, + // sequence alert 2 is made up of doc1WithLaterTimestamp and doc2WithLaterTimestamp + // sequence alert 2 is suppressed because it shares the same + // host.name value as sequence alert 1 + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithLaterTimestamp]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one created alert and one suppressed alert + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts when the rule is a building block type rule', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp2, + }; + + // sequence alert 1 is made up of doc1 and doc1WithLaterTimestamp, + // sequence alert 2 is made up of doc1WithLaterTimestamp and doc2WithLaterTimestamp + // sequence alert 2 is suppressed because it shares the same + // host.name value as sequence alert 1 + + await indexListOfSourceDocuments([doc1, doc1WithLaterTimestamp, doc2WithLaterTimestamp]); + + const buildingBlockRuleType: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + building_block_type: 'default', + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule: buildingBlockRuleType, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], }); + // we expect one created alert and one suppressed alert + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partition( + previewAlerts, + (alert) => alert?._source?.[ALERT_SUPPRESSION_DOCS_COUNT] != null + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts in a given rule execution when a subsequent event for a sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // the second sequence alert will have null as the value for + // suppression field host.name because of logic defined in the + // objectPairIntersection function defined in + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithNoHost, + { ...doc2WithNoHost, '@timestamp': '2020-10-28T06:53:01.000Z' }, + ]); const rule: EqlRuleCreateProps = { ...getEqlRuleForAlertTesting(['ecs_compliant']), query: getSequenceQuery(id), alert_suppression: { - group_by: ['agent.name'], - duration: { - value: 300, - unit: 'm', + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect two sequence alerts + // each sequence alert having two building block alerts + expect(previewAlerts.length).toEqual(6); + const [sequenceAlerts, buildingBlockAlerts] = + partitionSequenceBuildingBlocks(previewAlerts); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlerts.length).toEqual(2); + + expect(sequenceAlerts[0]?._source).toEqual({ + ...sequenceAlerts[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + expect(sequenceAlerts[1]?._source).toEqual({ + ...sequenceAlerts[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts in a given rule execution when doNotSuppress is set and more than one sequence has the suppression field undefined', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: undefined, + }; + + // this should generate three sequence alerts + // sequence alert 1 will be doc1 and doc1WithLaterTimestamp + // sequence alert 2 will be doc1WithLaterTimestamp and doc2WithNoHost + // sequence alert 3 will be doc2WithNoHost and doc3WithNoHost + // This logic is defined in objectPairIntersection + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + // Any sequence comprised of events where one field contains a value, followed by any number of + // events in the sequence where the value is null or undefined will have that field + // stripped from the sequence alert. So given that, we expect three alerts here + // The sequence comprised of [doc1, doc1WithLaterTimestamp] will have + // host.name and 'host-a' as the suppression values + // the other two sequence alerts comprised of [doc1WithLaterTimestamp, doc2WithNoHost] + // and [doc2WithNoHost, doc3WithNoHost] will have no suppression values set because + // of the logic outlined above + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithNoHost, + doc3WithNoHost, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [TIMESTAMP], + }); + // we expect one alert and one suppressed alert + // and two building block alerts per shell alert, let's confirm that + const [sequenceAlerts, buildingBlockAlerts] = + partitionSequenceBuildingBlocks(previewAlerts); + expect(buildingBlockAlerts.length).toEqual(6); + expect(sequenceAlerts.length).toEqual(3); + const [suppressedSequenceAlerts] = partition( + sequenceAlerts, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + + expect(suppressedSequenceAlerts[0]._source).toEqual({ + ...suppressedSequenceAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + }); + + it('does not suppress alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(6); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 + ); + expect(buildingBlockAlerts.length).toEqual(4); + expect(sequenceAlert.length).toEqual(2); + expect(suppressedSequenceAlerts.length).toEqual(0); + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: undefined, + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: undefined, + }); + }); + + it('suppresses alerts when suppression field value is undefined for a sequence alert in a given rule execution', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: undefined, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: undefined, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: undefined, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], missing_fields_strategy: 'suppress', }, from: 'now-35m', interval: '30m', }; - const { logs } = await previewRule({ + const { previewId } = await previewRule({ supertest, rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), invocationCount: 1, }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('does not suppress alerts when "doNotSuppress" is set and suppression field value is undefined for a sequence alert in a given rule execution', async () => { + // This logic should be understood within the confines of the + // objectPairIntersection function defined in + // x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts + // Any sequence comprised of events where one field contains a value, followed by any number of + // events in the sequence where the value is null or undefined will have that field + // stripped from the sequence alert. So given that, we expect three alerts here + // all with the suppression value as undefined + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:51:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:53:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:01.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: undefined }, + }; + + const doc2WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp2, + host: { name: undefined }, + }; + + const doc3WithNoHost = { + ...doc1, + '@timestamp': laterTimestamp3, + host: { name: undefined }, + }; + + await indexListOfSourceDocuments([doc1, doc1WithNoHost, doc2WithNoHost, doc3WithNoHost]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; - const [{ warnings }] = logs; + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); - expect(warnings).toContain( - 'Suppression is not supported for EQL sequence queries. The rule will proceed without suppression.' + expect(previewAlerts.length).toEqual(9); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 ); + // no alerts should be suppressed because doNotSuppress is set + expect(suppressedSequenceAlerts.length).toEqual(0); + expect(buildingBlockAlerts.length).toEqual(6); + // 3 sequence alerts comprised of + // (doc1 + doc1WithNoHost), (doc1WithNoHost + doc2WithNoHost), (doc2WithNoHost + doc3WithNoHost) + expect(sequenceAlert.length).toEqual(3); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: undefined, + [ALERT_SUPPRESSION_DOCS_COUNT]: undefined, + }); + }); + + it('does not suppress alerts outside of the current rule execution search range', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:05:00.000Z'; // this should not count towards events + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp2 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + }; + const doc2WithLaterTimestamp = { ...doc1WithLaterTimestamp, '@timestamp': laterTimestamp2 }; + const doc3WithLaterTimestamp = { ...doc2WithLaterTimestamp, '@timestamp': laterTimestamp3 }; + + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithLaterTimestamp, + doc3WithLaterTimestamp, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]?._source).toEqual({ + ...sequenceAlert[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses sequence alert with suppression value of "null" when all events have unique values for suppression field', async () => { + /* + Sequence alerts only contain values which make up the intersection of + a given field in all events for that sequence. + + So for a sequence alert where the two events contain host name values of + host-a and host-b, the sequence alert will have 'null' for that value and will + suppress on null values if missing_fields_strategy is set to 'suppress' + */ + const id = uuidv4(); + const timestamp = '2020-10-28T06:50:00.000Z'; + const laterTimestamp = '2020-10-28T06:50:01.000Z'; + const laterTimestamp2 = '2020-10-28T06:51:00.000Z'; + const laterTimestamp3 = '2020-10-28T06:53:00.000Z'; + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + }; + const doc1WithLaterTimestamp = { + ...doc1, + '@timestamp': laterTimestamp, + host: { name: 'host-b' }, + }; + const doc2WithLaterTimestamp = { + ...doc1WithLaterTimestamp, + '@timestamp': laterTimestamp2, + host: { name: 'host-c' }, + }; + const doc3WithLaterTimestamp = { + ...doc2WithLaterTimestamp, + '@timestamp': laterTimestamp3, + host: { name: 'host-d' }, + }; + await indexListOfSourceDocuments([ + doc1, + doc1WithLaterTimestamp, + doc2WithLaterTimestamp, + doc3WithLaterTimestamp, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + // we expect one alert and two suppressed alerts + // and two building block alerts, let's confirm that + expect(previewAlerts.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('suppresses alerts on a field with array values', async () => { + const id = uuidv4(); + + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:46:00.000Z'; + const timestamp3 = '2020-10-28T06:47:00.000Z'; + + const doc1 = { + id, + '@timestamp': timestamp, + host: { name: ['host-a', 'host-b'] }, + }; + + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': timestamp2 }, + { ...doc1, '@timestamp': timestamp3 }, + ]); + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); + expect(previewAlerts.length).toEqual(3); // one sequence, two building block + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a', 'host-b'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('suppresses alerts with missing fields and multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; + + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, + + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, + + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, + + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 100, + sort: [ALERT_SUPPRESSION_START], // sorting on null fields was preventing the alerts from yielding + }); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); + + // for sequence alerts if neither of the fields are there, we cannot suppress + expect(sequenceAlert.length).toEqual(4); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: 10, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(sequenceAlert[1]._source).toEqual({ + ...sequenceAlert[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: 10, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sequenceAlert[2]._source).toEqual({ + ...sequenceAlert[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + + expect(sequenceAlert[3]._source).toEqual({ + ...sequenceAlert[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('does not suppress alerts when "doNotSuppress" is set and we have alerts with missing fields and multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const timestamp2 = '2020-10-28T06:45:01.000Z'; + const timestamp3 = '2020-10-28T06:45:02.000Z'; + const timestamp4 = '2020-10-28T06:45:03.000Z'; + const timestamp5 = '2020-10-28T06:45:04.000Z'; + const timestamp6 = '2020-10-28T06:45:05.000Z'; + const timestamp7 = '2020-10-28T06:45:06.000Z'; + const timestamp8 = '2020-10-28T06:45:07.000Z'; + const timestamp9 = '2020-10-28T06:45:08.000Z'; + const timestamp10 = '2020-10-28T06:45:09.000Z'; + const timestamp11 = '2020-10-28T06:45:10.000Z'; + const timestamp12 = '2020-10-28T06:45:11.000Z'; + + const noMissingFieldsDoc = { + id, + '@timestamp': timestamp, + host: { name: 'host-a' }, + agent: { name: 'agent-a', version: 10 }, + }; + + const missingNameFieldsDoc = { + ...noMissingFieldsDoc, + agent: { version: 10 }, + }; + + const missingVersionFieldsDoc = { + ...noMissingFieldsDoc, + agent: { name: 'agent-a' }, + }; + + const missingAgentFieldsDoc = { + ...noMissingFieldsDoc, + agent: undefined, + }; + + // 4 alerts should be suppressed: 1 for each pair of documents + await indexListOfSourceDocuments([ + noMissingFieldsDoc, + { ...noMissingFieldsDoc, '@timestamp': timestamp2 }, + { ...noMissingFieldsDoc, '@timestamp': timestamp3 }, + + { ...missingNameFieldsDoc, '@timestamp': timestamp4 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp5 }, + { ...missingNameFieldsDoc, '@timestamp': timestamp6 }, + + { ...missingVersionFieldsDoc, '@timestamp': timestamp7 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp8 }, + { ...missingVersionFieldsDoc, '@timestamp': timestamp9 }, + + { ...missingAgentFieldsDoc, '@timestamp': timestamp10 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp11 }, + { ...missingAgentFieldsDoc, '@timestamp': timestamp12 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 100, + }); + const [sequenceAlert] = partitionSequenceBuildingBlocks(previewAlerts); + + // for sequence alerts if neither of the fields are there, we cannot suppress + const [suppressedSequenceAlerts] = partition( + sequenceAlert, + (alert) => (alert?._source?.['kibana.alert.suppression.docs_count'] as number) >= 0 + ); + expect(suppressedSequenceAlerts.length).toEqual(1); + expect(suppressedSequenceAlerts[0]._source).toEqual({ + ...suppressedSequenceAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: 10, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('deduplicates multiple alerts while suppressing on rule interval only', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + const secondTimestamp2 = '2020-10-28T06:10:01.000Z'; + const secondTimestamp3 = '2020-10-28T06:10:02.000Z'; + const secondTimestamp4 = '2020-10-28T06:10:03.000Z'; + const secondTimestamp5 = '2020-10-28T06:10:04.000Z'; + const secondTimestamp6 = '2020-10-28T06:10:05.000Z'; + + const doc1 = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + + // 4 alert should be suppressed + await indexListOfSourceDocuments([ + doc1, + { ...doc1, '@timestamp': secondTimestamp }, + { ...doc1, '@timestamp': secondTimestamp2 }, + { ...doc1, '@timestamp': secondTimestamp3 }, + { ...doc1, '@timestamp': secondTimestamp4 }, + { ...doc1, '@timestamp': secondTimestamp5 }, + { ...doc1, '@timestamp': secondTimestamp6 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-50m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, // 2 invocations so we can see that the same sequences are not generated / suppressed again. + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks(previewAlerts); + + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 5, // if deduplication failed this would 12, or the previewAlerts count would be double + }); + }); + }); + + describe('@skipInServerless sequence queries with suppression duration', () => { + it('suppresses alerts across two rule executions when the suppression duration exceeds the rule interval', async () => { + const id = uuidv4(); + const firstTimestamp = new Date(Date.now() - 1000).toISOString(); + const firstTimestamp2 = new Date().toISOString(); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstDocument, + { ...firstDocument, '@timestamp': firstTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + // suppression start equal to alert timestamp + const suppressionStart = sequenceAlert[0]._source?.[TIMESTAMP]; + + expect(sequenceAlert[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: suppressionStart, + [ALERT_SUPPRESSION_END]: suppressionStart, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + // index an event that happened 1 second before the next event in the sequence + const secondTimestamp = new Date(Date.now() - 1000).toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfSourceDocuments([secondDocument]); + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + + const [sequenceAlert2] = partitionSequenceBuildingBlocks(secondAlerts.hits.hits); + + expect(sequenceAlert2.length).toEqual(1); + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // 1 alert from second rule run, that's why 1 suppressed + }); + // suppression end value should be greater than second document timestamp, but lesser than current time + const suppressionEnd = new Date( + sequenceAlert2[0]._source?.[ALERT_SUPPRESSION_END] as string + ).getTime(); + expect(suppressionEnd).toBeLessThan(new Date().getTime()); + expect(suppressionEnd).toBeGreaterThan(new Date(secondTimestamp).getDate()); + }); + + it('does not suppress alerts outside of duration', async () => { + const id = uuidv4(); + // this timestamp is 1 minute in the past + const firstTimestamp = new Date(Date.now() - 5000).toISOString(); + const firstTimestamp2 = new Date(Date.now() - 5500).toISOString(); + + const firstDocument = { + id, + '@timestamp': firstTimestamp, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstDocument, + { ...firstDocument, '@timestamp': firstTimestamp2 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: getSequenceQuery(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 7, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-10s', + interval: '5s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits.length).toEqual(3); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits + ); + expect(buildingBlockAlerts.length).toEqual(2); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + + const secondTimestamp = new Date(Date.now() - 1000).toISOString(); + const secondTimestamp2 = new Date(Date.now()).toISOString(); + const secondDocument = { + id, + '@timestamp': secondTimestamp, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. + // the second rule run should generate a new alert + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + + await indexListOfSourceDocuments([ + secondDocument, + { ...secondDocument, '@timestamp': secondTimestamp2 }, + ]); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(Date.now() + 5000); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + + const [sequenceAlert2] = partitionSequenceBuildingBlocks(secondAlerts.hits.hits); + + expect(sequenceAlert2.length).toEqual(2); + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed + }); + expect(sequenceAlert2[1]._source).toEqual({ + ...sequenceAlert2[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // 1 alert from second rule run, that's why 1 suppressed + }); + }); + + it('suppresses alerts when query has 3 sequences', async () => { + // the first sequence alert is comprised of events [timestamp1, ...2, ...3] + // the other suppressed alerts will be made up of the following sequences + // [timestamp2, timestamp3, timestamp4] + // [timestamp3, timestamp4, timestamp5] + // [timestamp4, timestamp5, timestamp6] + const id = uuidv4(); + const dateNow = Date.now(); + const timestamp1 = new Date(dateNow - 5000).toISOString(); + const timestamp2 = new Date(dateNow - 5500).toISOString(); + const timestamp3 = new Date(dateNow - 5800).toISOString(); + const timestamp4 = new Date(dateNow - 6000).toISOString(); + const timestamp5 = new Date(dateNow - 6100).toISOString(); + const timestamp6 = new Date(dateNow - 6200).toISOString(); + + const firstSequenceEvent = { + id, + '@timestamp': timestamp1, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstSequenceEvent, + { ...firstSequenceEvent, '@timestamp': timestamp2 }, + { ...firstSequenceEvent, '@timestamp': timestamp3 }, + ]); + + const secondSequenceEvent = { + id, + '@timestamp': timestamp4, + host: { + name: 'host-a', + }, + }; + + await indexListOfSourceDocuments([ + secondSequenceEvent, + { ...secondSequenceEvent, '@timestamp': timestamp5 }, + { ...secondSequenceEvent, '@timestamp': timestamp6 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: `sequence [any where id == "${id}"] [any where id == "${id}"] [any where id == "${id}"]`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 10, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-30s', + interval: '10s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one shell alert + // and three building block alerts + expect(alerts.hits.hits.length).toEqual(4); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits + ); + expect(buildingBlockAlerts.length).toEqual(3); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }); + }); + + it('does not suppress alerts outside of duration when query with 3 sequences', async () => { + const id = uuidv4(); + const dateNow = Date.now(); + const timestampSequenceEvent1 = new Date(dateNow - 5000).toISOString(); + const timestampSequenceEvent2 = new Date(dateNow - 5500).toISOString(); + const timestampSequenceEvent3 = new Date(dateNow - 5800).toISOString(); + + const firstSequenceEvent = { + id, + '@timestamp': timestampSequenceEvent1, + host: { + name: 'host-a', + }, + }; + await indexListOfSourceDocuments([ + firstSequenceEvent, + { ...firstSequenceEvent, '@timestamp': timestampSequenceEvent2 }, + { ...firstSequenceEvent, '@timestamp': timestampSequenceEvent3 }, + ]); + + const rule: EqlRuleCreateProps = { + ...getEqlRuleForAlertTesting(['ecs_compliant']), + query: `sequence [any where id == "${id}"] [any where id == "${id}"] [any where id == "${id}"]`, + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 7, + unit: 's', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-10s', + interval: '5s', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + // we expect one shell alert + // and three building block alerts + expect(alerts.hits.hits.length).toEqual(4); + const [sequenceAlert, buildingBlockAlerts] = partitionSequenceBuildingBlocks( + alerts.hits.hits + ); + expect(buildingBlockAlerts.length).toEqual(3); + expect(sequenceAlert.length).toEqual(1); + + expect(sequenceAlert[0]._source).toEqual({ + ...sequenceAlert[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }); + + const dateNow2 = Date.now(); + const secondTimestampEventSequence = new Date(dateNow2 - 2000).toISOString(); + const secondTimestampEventSequence2 = new Date(dateNow2 - 1000).toISOString(); + const secondTimestampEventSequence3 = new Date(dateNow2).toISOString(); + + const secondSequenceEvent = { + id, + '@timestamp': secondTimestampEventSequence, + host: { + name: 'host-a', + }, + }; + + // Add a new document, then disable and re-enable to trigger another rule run. + // the second rule run should generate a new alert + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + + await indexListOfSourceDocuments([ + secondSequenceEvent, + { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence2 }, + { ...secondSequenceEvent, '@timestamp': secondTimestampEventSequence3 }, + ]); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp2 = new Date(dateNow2 + 55000); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp2 + ); + + const [sequenceAlert2, buildingBlockAlerts2] = partitionSequenceBuildingBlocks( + secondAlerts.hits.hits + ); + + // two sequence alerts because the second one happened + // outside of the rule's suppression duration + expect(sequenceAlert2.length).toEqual(2); + expect(buildingBlockAlerts2.length).toEqual(6); + // timestamps should be different for two alerts, showing they were + // created in different rule executions + expect(sequenceAlert2[0]?._source?.[TIMESTAMP]).not.toEqual( + sequenceAlert2[1]?._source?.[TIMESTAMP] + ); + + expect(sequenceAlert2[0]._source).toEqual({ + ...sequenceAlert2[0]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // only one alert created, so zero documents are suppressed + }); + expect(sequenceAlert2[1]._source).toEqual({ + ...sequenceAlert2[1]?._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // only one alert created, so zero documents are suppressed + }); }); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/trial_license_complete_tier/configs/serverless.config.ts index db3ad04b3d1b9..987b9c5004add 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/general_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -17,5 +17,8 @@ export default createTestConfig({ 'testing_ignored.constant', '/testing_regex*/', ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, ], }); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index f02968945087d..6b10122213ca7 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -44,6 +44,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // See https://github.com/elastic/kibana/pull/125396 for details '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, // mock cloud to enable the guided onboarding tour in e2e tests '--xpack.cloud.id=test', `--home.disableWelcomeScreen=true`, diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts index 31855c3efff15..8f4b6a10faeb8 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression.cy.ts @@ -38,7 +38,7 @@ const SUPPRESS_BY_FIELDS = ['agent.type']; describe( 'Detection Rule Creation - EQL Rules - With Alert Suppression', { - tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + tags: ['@ess', '@skipInServerlessMKI'], }, () => { describe('with non-sequence queries', () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts index 8f07781b7732c..2cf8526a2f9c3 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/eql_rule_suppression_sequence.cy.ts @@ -8,20 +8,45 @@ import { getEqlSequenceRule } from '../../../../objects/rule'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; +import { getDetails } from '../../../../tasks/rule_details'; import { CREATE_RULE_URL } from '../../../../urls/navigation'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { fillDefineEqlRule, selectEqlRuleType } from '../../../../tasks/create_new_rule'; +import { + fillAlertSuppressionFields, + fillAboutRuleMinimumAndContinue, + createRuleWithoutEnabling, + skipScheduleRuleAction, + continueFromDefineStep, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, + fillDefineEqlRule, + selectEqlRuleType, +} from '../../../../tasks/create_new_rule'; -import { TOOLTIP } from '../../../../screens/common'; import { - ALERT_SUPPRESSION_FIELDS, - ALERT_SUPPRESSION_FIELDS_INPUT, -} from '../../../../screens/create_new_rule'; + DEFINITION_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_BY_DETAILS, + SUPPRESS_MISSING_FIELD, + DETAILS_TITLE, +} from '../../../../screens/rule_details'; + +const SUPPRESS_BY_FIELDS = ['agent.type']; describe( 'Detection Rule Creation - EQL Rules - With Alert Suppression', { - tags: ['@ess', '@serverless'], + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + tags: ['@ess', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForSequenceEqlRuleEnabled', + ])}`, + ], + }, }, () => { describe('with sequence queries ', () => { @@ -36,11 +61,70 @@ describe( fillDefineEqlRule(rule); }); - it('disables the suppression fields and presents an informative tooltip', () => { - cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.disabled'); + it('creates a rule with a "per rule execution" suppression duration', () => { + // selecting only suppression fields, the rest options would be default + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + }); + }); + + it('creates a rule with a "per time interval" suppression duration', () => { + const expectedSuppressByFields = SUPPRESS_BY_FIELDS.slice(0, 1); + + // fill suppress by fields and select non-default suppression options + fillAlertSuppressionFields(expectedSuppressByFields); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(45, 'm'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); - cy.get(ALERT_SUPPRESSION_FIELDS).trigger('mouseover'); - cy.get(TOOLTIP).contains('Suppression is not supported for EQL sequence queries.'); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', expectedSuppressByFields.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '45m'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); }); }); }