[8.x] [Security Solution][Detection Engine] Reduce CPU usage when creating alerts from source documents (#192287)

x-pack/plugins/security_solution/scripts/quickstart/modules/data/index.ts

@@ -29,6 +29,30 @@ export const buildLargeDocument = ({
   return doc;
 };
 
+export const buildLargeNestedDocument = ({
+  fieldsPerObject,
+  levels,
+  fieldSize,
+}: {
+  fieldsPerObject: number;
+  levels: number;
+  fieldSize: number;
+}): Record<string, unknown> => {
+  if (levels === 1) {
+    return buildLargeDocument({ numFields: fieldsPerObject, fieldSize });
+  } else {
+    const doc: Record<string, unknown> = {};
+    range(fieldsPerObject).forEach((idx) => {
+      doc[`level_${levels}_field${idx}`] = buildLargeNestedDocument({
+        fieldsPerObject,
+        levels: levels - 1,
+        fieldSize,
+      });
+    });
+    return doc;
+  }
+};
+
 export const addTimestampToDoc = ({
   timestamp = new Date(),
   doc,

x-pack/plugins/security_solution/scripts/quickstart/modules/mappings/index.ts

@@ -14,7 +14,7 @@ import type {
 import { mappingFromFieldMap } from '@kbn/alerting-plugin/common';
 import { ecsFieldMap } from '@kbn/alerts-as-data-utils';
 
-export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap);
+export const getEcsMapping = () => mappingFromFieldMap(ecsFieldMap, false);
 
 export interface GenerateLargeMappingPropertiesProps {
   size: number;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { isEmpty } from 'lodash';
+import { isEmpty, partition } from 'lodash';
 import agent from 'elastic-apm-node';
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
@@ -375,8 +375,17 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
             );
 
             const legacySignalFields: string[] = Object.keys(aadFieldConversion);
+            const [ignoreFieldsRegexes, ignoreFieldsStandard] = partition(
+              [...ignoreFields, ...legacySignalFields],
+              (field: string) => field.startsWith('/') && field.endsWith('/')
+            );
+            const ignoreFieldsObject: Record<string, boolean> = {};
+            ignoreFieldsStandard.forEach((field) => {
+              ignoreFieldsObject[field] = true;
+            });
             const wrapHits = wrapHitsFactory({
-              ignoreFields: [...ignoreFields, ...legacySignalFields],
+              ignoreFields: ignoreFieldsObject,
+              ignoreFieldsRegexes,
               mergeStrategy,
               completeRule,
               spaceId,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts

@@ -12,8 +12,8 @@ import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_pat
 import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants';
 import type { ConfigType } from '../../../../config';
 import type { Ancestor, SignalSource, SignalSourceHit } from '../types';
-import { buildAlert, buildAncestors, generateAlertId } from '../factories/utils/build_alert';
-import { buildBulkBody } from '../factories/utils/build_bulk_body';
+import { buildAlertFields, buildAncestors, generateAlertId } from '../factories/utils/build_alert';
+import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
 import type { EqlSequence } from '../../../../../common/detection_engine/types';
 import { generateBuildingBlockIds } from '../factories/utils/generate_building_block_ids';
 import type { BuildReasonMessage } from '../utils/reason_formatters';
@@ -59,20 +59,21 @@ export const buildAlertGroupFromSequence = (
   let baseAlerts: BaseFieldsLatest[] = [];
   try {
     baseAlerts = sequence.events.map((event) =>
-      buildBulkBody(
+      transformHitToAlert({
         spaceId,
         completeRule,
-        event,
+        doc: event,
         mergeStrategy,
-        [],
-        false,
+        ignoreFields: {},
+        ignoreFieldsRegexes: [],
+        applyOverrides: false,
         buildReasonMessage,
         indicesToQuery,
         alertTimestampOverride,
         ruleExecutionLogger,
-        'placeholder-alert-uuid', // This is overriden below
-        publicBaseUrl
-      )
+        alertUuid: 'placeholder-alert-uuid', // This is overriden below
+        publicBaseUrl,
+      })
     );
   } catch (error) {
     ruleExecutionLogger.error(error);
@@ -153,16 +154,16 @@ export const buildAlertRoot = (
     severity: completeRule.ruleParams.severity,
     mergedDoc: mergedAlerts as SignalSourceHit,
   });
-  const doc = buildAlert(
-    wrappedBuildingBlocks,
+  const doc = buildAlertFields({
+    docs: wrappedBuildingBlocks,
     completeRule,
     spaceId,
     reason,
     indicesToQuery,
-    'placeholder-uuid', // These will be overriden below
+    alertUuid: 'placeholder-uuid', // These will be overriden below
     publicBaseUrl, // Not necessary now, but when the ID is created ahead of time this can be passed
-    alertTimestampOverride
-  );
+    alertTimestampOverride,
+  });
   const alertId = generateAlertId(doc);
   const alertUrl = getAlertDetailsUrl({
     alertId,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts

@@ -16,7 +16,7 @@ import type { ConfigType } from '../../../../config';
 import type { CompleteRule, EsqlRuleParams } from '../../rule_schema';
 import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters';
 import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
-import { buildBulkBody } from '../factories/utils/build_bulk_body';
+import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
 import type { SignalSource } from '../types';
 import { generateAlertId } from './utils';
 
@@ -55,20 +55,21 @@ export const wrapEsqlAlerts = ({
       index: i,
     });
 
-    const baseAlert: BaseFieldsLatest = buildBulkBody(
+    const baseAlert: BaseFieldsLatest = transformHitToAlert({
       spaceId,
       completeRule,
-      event,
+      doc: event,
       mergeStrategy,
-      [],
-      true,
-      buildReasonMessageForNewTermsAlert,
-      [],
+      ignoreFields: {},
+      ignoreFieldsRegexes: [],
+      applyOverrides: true,
+      buildReasonMessage: buildReasonMessageForNewTermsAlert,
+      indicesToQuery: [],
       alertTimestampOverride,
       ruleExecutionLogger,
-      id,
-      publicBaseUrl
-    );
+      alertUuid: id,
+      publicBaseUrl,
+    });
 
     return {
       _id: id,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts

@@ -19,7 +19,7 @@ import type { ConfigType } from '../../../../config';
 import type { CompleteRule, EsqlRuleParams } from '../../rule_schema';
 import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters';
 import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring';
-import { buildBulkBody } from '../factories/utils/build_bulk_body';
+import { transformHitToAlert } from '../factories/utils/transform_hit_to_alert';
 import type { SignalSource } from '../types';
 import { getSuppressionAlertFields, getSuppressionTerms } from '../utils';
 import { generateAlertId } from './utils';
@@ -73,20 +73,21 @@ export const wrapSuppressedEsqlAlerts = ({
 
       const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]);
 
-      const baseAlert: BaseFieldsLatest = buildBulkBody(
+      const baseAlert: BaseFieldsLatest = transformHitToAlert({
         spaceId,
         completeRule,
-        event,
+        doc: event,
         mergeStrategy,
-        [],
-        true,
-        buildReasonMessageForNewTermsAlert,
-        [],
+        ignoreFields: {},
+        ignoreFieldsRegexes: [],
+        applyOverrides: true,
+        buildReasonMessage: buildReasonMessageForNewTermsAlert,
+        indicesToQuery: [],
         alertTimestampOverride,
         ruleExecutionLogger,
-        id,
-        publicBaseUrl
-      );
+        alertUuid: id,
+        publicBaseUrl,
+      });
 
       return {
         _id: id,