[8.x] [Security Solution][Detection Engine] removes field_caps call f…

…or all fields in index during rule execution (#193869) (#194433)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Security Solution][Detection Engine] removes field_caps call for all
fields in index during rule execution
(#193869)](#193869)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Vitalii
Dmyterko","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-09-30T13:15:24Z","message":"[Security
Solution][Detection Engine] removes field_caps call for all fields in
index during rule execution (#193869)\n\n## Summary\r\n\r\n- addresses
https://github.com/elastic/kibana/issues/187059\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>","sha":"b005ea907b7173b2aa7ab0974f9a2fcfef08de0d","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:
SecuritySolution","backport:prev-minor","Team:Detection Engine","8.16
candidate"],"title":"[Security Solution][Detection Engine] removes
field_caps call for all fields in index during rule
execution","number":193869,"url":"https://github.com/elastic/kibana/pull/193869","mergeCommit":{"message":"[Security
Solution][Detection Engine] removes field_caps call for all fields in
index during rule execution (#193869)\n\n## Summary\r\n\r\n- addresses
https://github.com/elastic/kibana/issues/187059\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>","sha":"b005ea907b7173b2aa7ab0974f9a2fcfef08de0d"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/193869","number":193869,"mergeCommit":{"message":"[Security
Solution][Detection Engine] removes field_caps call for all fields in
index during rule execution (#193869)\n\n## Summary\r\n\r\n- addresses
https://github.com/elastic/kibana/issues/187059\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>","sha":"b005ea907b7173b2aa7ab0974f9a2fcfef08de0d"}}]}]
BACKPORT-->

Co-authored-by: Vitalii Dmyterko <[email protected]>

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts

@@ -11,7 +11,6 @@ import agent from 'elastic-apm-node';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { TIMESTAMP } from '@kbn/rule-data-utils';
 import { createPersistenceRuleTypeWrapper } from '@kbn/rule-registry-plugin/server';
-import type { DataViewFieldBase } from '@kbn/es-query';
 import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists';
 import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map';
 import type { FieldMap } from '@kbn/alerts-as-data-utils';
@@ -26,8 +25,6 @@ import {
   hasTimestampFields,
   isMachineLearningParams,
   isEsqlParams,
-  isQueryParams,
-  isEqlParams,
   getDisabledActionsWarningText,
 } from './utils/utils';
 import { DEFAULT_MAX_SIGNALS, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants';
@@ -47,7 +44,6 @@ import { withSecuritySpan } from '../../../utils/with_security_span';
 import { getInputIndex, DataViewError } from './utils/get_input_output_index';
 import { TIMESTAMP_RUNTIME_FIELD } from './constants';
 import { buildTimestampRuntimeMapping } from './utils/build_timestamp_runtime_mapping';
-import { getFieldsForWildcard } from './utils/get_fields_for_wildcard';
 import { alertsFieldMap, rulesFieldMap } from '../../../../common/field_maps';
 import { sendAlertSuppressionTelemetryEvent } from './utils/telemetry/send_alert_suppression_telemetry_event';
 
@@ -139,7 +135,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
           } = options;
           let runState = state;
           let inputIndex: string[] = [];
-          let inputIndexFields: DataViewFieldBase[] = [];
           let runtimeMappings: estypes.MappingRuntimeFields | undefined;
           const { from, maxSignals, timestampOverride, timestampOverrideFallbackDisabled, to } =
             params;
@@ -337,21 +332,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
             });
           }
 
-          if (
-            !isMachineLearningParams(params) &&
-            !isEsqlParams(params) &&
-            !isQueryParams(params) &&
-            !isEqlParams(params)
-          ) {
-            const dataViews = await services.getDataViews();
-            inputIndexFields = await getFieldsForWildcard({
-              index: inputIndex,
-              dataViews,
-              language: params.language,
-              ruleExecutionLogger,
-            });
-          }
-
           try {
             const { listClient, exceptionsClient } = getListClient({
               esClient: services.scopedClusterClient.asCurrentUser,
@@ -426,7 +406,6 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper =
                     inputIndex,
                     exceptionFilter,
                     unprocessedExceptions,
-                    inputIndexFields,
                     runtimeMappings: {
                       ...runtimeMappings,
                       ...timestampRuntimeMappings,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -76,7 +76,6 @@ export const createIndicatorMatchAlertType = (
           secondaryTimestamp,
           exceptionFilter,
           unprocessedExceptions,
-          inputIndexFields,
         },
         services,
         spaceId,
@@ -119,7 +118,6 @@ export const createIndicatorMatchAlertType = (
         secondaryTimestamp,
         exceptionFilter,
         unprocessedExceptions,
-        inputIndexFields,
         wrapSuppressedHits,
         runOpts,
         licensing,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/indicator_match.ts

@@ -15,7 +15,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type { ListClient } from '@kbn/lists-plugin/server';
-import type { Filter, DataViewFieldBase } from '@kbn/es-query';
+import type { Filter } from '@kbn/es-query';
 import type { RuleRangeTuple, BulkCreate, WrapHits, WrapSuppressedHits, RunOpts } from '../types';
 import type { ITelemetryEventsSender } from '../../../telemetry/sender';
 import { createThreatSignals } from './threat_mapping/create_threat_signals';
@@ -43,7 +43,6 @@ export const indicatorMatchExecutor = async ({
   secondaryTimestamp,
   exceptionFilter,
   unprocessedExceptions,
-  inputIndexFields,
   wrapSuppressedHits,
   runOpts,
   licensing,
@@ -65,7 +64,6 @@ export const indicatorMatchExecutor = async ({
   secondaryTimestamp?: string;
   exceptionFilter: Filter | undefined;
   unprocessedExceptions: ExceptionListItemSchema[];
-  inputIndexFields: DataViewFieldBase[];
   wrapSuppressedHits: WrapSuppressedHits;
   runOpts: RunOpts<ThreatRuleParams>;
   licensing: LicensingPluginSetup;
@@ -106,7 +104,6 @@ export const indicatorMatchExecutor = async ({
       secondaryTimestamp,
       exceptionFilter,
       unprocessedExceptions,
-      inputIndexFields,
       runOpts,
       licensing,
       experimentalFeatures,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_event_signal.ts

@@ -144,6 +144,7 @@ export const createEventSignal = async ({
       index: inputIndex,
       exceptionFilter,
       fields: inputIndexFields,
+      loadFields: true,
     });
 
     ruleExecutionLogger.debug(`${ids?.length} matched signals found`);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signal.ts

@@ -81,6 +81,7 @@ export const createThreatSignal = async ({
       index: inputIndex,
       exceptionFilter,
       fields: inputIndexFields,
+      loadFields: true,
     });
 
     ruleExecutionLogger.debug(

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/create_threat_signals.ts

@@ -36,8 +36,8 @@ import { getEventCount, getEventList } from './get_event_count';
 import { getMappingFilters } from './get_mapping_filters';
 import { THREAT_PIT_KEEP_ALIVE } from '../../../../../../common/cti/constants';
 import { getMaxSignalsWarning, getSafeSortIds } from '../../utils/utils';
-import { getFieldsForWildcard } from '../../utils/get_fields_for_wildcard';
 import { getDataTierFilter } from '../../utils/get_data_tier_filter';
+import { getQueryFields } from '../../utils/get_query_fields';
 
 export const createThreatSignals = async ({
   alertId,
@@ -72,7 +72,6 @@ export const createThreatSignals = async ({
   secondaryTimestamp,
   exceptionFilter,
   unprocessedExceptions,
-  inputIndexFields,
   licensing,
   experimentalFeatures,
 }: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
@@ -115,6 +114,14 @@ export const createThreatSignals = async ({
   const allEventFilters = [...filters, eventMappingFilter, ...dataTiersFilters];
   const allThreatFilters = [...threatFilters, indicatorMappingFilter, ...dataTiersFilters];
 
+  const dataViews = await services.getDataViews();
+  const inputIndexFields = await getQueryFields({
+    dataViews,
+    index: inputIndex,
+    query,
+    language,
+  });
+
   const eventCount = await getEventCount({
     esClient: services.scopedClusterClient.asCurrentUser,
     index: inputIndex,
@@ -140,12 +147,11 @@ export const createThreatSignals = async ({
     if (newPitId) threatPitId = newPitId;
   };
 
-  const dataViews = await services.getDataViews();
-  const threatIndexFields = await getFieldsForWildcard({
-    index: threatIndex,
-    language: threatLanguage ?? 'kuery',
+  const threatIndexFields = await getQueryFields({
     dataViews,
-    ruleExecutionLogger,
+    index: threatIndex,
+    query: threatQuery,
+    language: threatLanguage,
   });
 
   const threatListCount = await getThreatListCount({

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/types.ts

@@ -77,7 +77,6 @@ export interface CreateThreatSignalsOptions {
   secondaryTimestamp?: string;
   exceptionFilter: Filter | undefined;
   unprocessedExceptions: ExceptionListItemSchema[];
-  inputIndexFields: DataViewFieldBase[];
   runOpts: RunOpts<ThreatRuleParams>;
   licensing: LicensingPluginSetup;
   experimentalFeatures: ExperimentalFeatures;

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts

@@ -110,7 +110,6 @@ export const createNewTermsAlertType = (
           unprocessedExceptions,
           alertTimestampOverride,
           publicBaseUrl,
-          inputIndexFields,
           alertWithSuppression,
         },
         services,
@@ -135,7 +134,7 @@ export const createNewTermsAlertType = (
         type: params.type,
         query: params.query,
         exceptionFilter,
-        fields: inputIndexFields,
+        loadFields: true,
       };
       const esFilter = await getFilter(filterArgs);
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts

@@ -58,7 +58,6 @@ export const queryExecutor = async ({
       services,
       index: runOpts.inputIndex,
       exceptionFilter: runOpts.exceptionFilter,
-      fields: runOpts.inputIndexFields,
       loadFields: true,
     });
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts

@@ -74,7 +74,6 @@ export const createThresholdAlertType = (
           aggregatableTimestampField,
           exceptionFilter,
           unprocessedExceptions,
-          inputIndexFields,
         },
         services,
         startedAt,
@@ -99,7 +98,6 @@ export const createThresholdAlertType = (
         aggregatableTimestampField,
         exceptionFilter,
         unprocessedExceptions,
-        inputIndexFields,
         spaceId,
         runOpts: execOptions.runOpts,
         licensing,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.test.ts

@@ -22,6 +22,8 @@ import { ruleExecutionLogMock } from '../../rule_monitoring/mocks';
 import type { RunOpts } from '../types';
 import type { ExperimentalFeatures } from '../../../../../common';
 
+jest.mock('../utils/get_filter', () => ({ getFilter: jest.fn() }));
+
 describe('threshold_executor', () => {
   let alertServices: RuleExecutorServicesMock;
   let ruleExecutionLogger: ReturnType<typeof ruleExecutionLogMock.forExecutors.create>;
@@ -107,7 +109,6 @@ describe('threshold_executor', () => {
         aggregatableTimestampField: TIMESTAMP,
         exceptionFilter: undefined,
         unprocessedExceptions: [],
-        inputIndexFields: [],
         spaceId: 'default',
         runOpts: {} as RunOpts<ThresholdRuleParams>,
         licensing,
@@ -173,7 +174,6 @@ describe('threshold_executor', () => {
         aggregatableTimestampField: TIMESTAMP,
         exceptionFilter: undefined,
         unprocessedExceptions: [getExceptionListItemSchemaMock()],
-        inputIndexFields: [],
         spaceId: 'default',
         runOpts: {} as RunOpts<ThresholdRuleParams>,
         licensing,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/threshold.ts

@@ -18,7 +18,7 @@ import type {
   RuleExecutorServices,
 } from '@kbn/alerting-plugin/server';
 import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server';
-import type { Filter, DataViewFieldBase } from '@kbn/es-query';
+import type { Filter } from '@kbn/es-query';
 import type { CompleteRule, ThresholdRuleParams } from '../../rule_schema';
 import { getFilter } from '../utils/get_filter';
 import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals';
@@ -64,7 +64,6 @@ export const thresholdExecutor = async ({
   aggregatableTimestampField,
   exceptionFilter,
   unprocessedExceptions,
-  inputIndexFields,
   spaceId,
   runOpts,
   licensing,
@@ -87,7 +86,6 @@ export const thresholdExecutor = async ({
   aggregatableTimestampField: string;
   exceptionFilter: Filter | undefined;
   unprocessedExceptions: ExceptionListItemSchema[];
-  inputIndexFields: DataViewFieldBase[];
   spaceId: string;
   runOpts: RunOpts<ThresholdRuleParams>;
   licensing: LicensingPluginSetup;
@@ -135,7 +133,7 @@ export const thresholdExecutor = async ({
       services,
       index: inputIndex,
       exceptionFilter,
-      fields: inputIndexFields,
+      loadFields: true,
     });
 
     // Look for new events over threshold

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -32,7 +32,7 @@ import type {
 } from '@kbn/rule-registry-plugin/server';
 import type { EcsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
 import type { TypeOfFieldMap } from '@kbn/rule-registry-plugin/common/field_map';
-import type { Filter, DataViewFieldBase } from '@kbn/es-query';
+import type { Filter } from '@kbn/es-query';
 
 import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server';
 import type { RulePreviewLoggedRequest } from '../../../../common/api/detection_engine/rule_preview/rule_preview.gen';
@@ -104,7 +104,6 @@ export interface RunOpts<TParams extends RuleParams> {
   alertWithSuppression: SuppressedAlertService;
   refreshOnIndexingAlerts: RefreshTypes;
   publicBaseUrl: string | undefined;
-  inputIndexFields: DataViewFieldBase[];
   experimentalFeatures?: ExperimentalFeatures;
 }