From 02a0c503f433310938c961ce666766aa7fa5d8da Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 23 Feb 2024 12:32:14 -0600
Subject: [PATCH 01/21] Document current EQL rule behavior with timestamp
 fields

The conclusion is that we currently need _both_ of these fields for the
EQL rule to operate if the data doesn't have a @timestamp at all.
However, the UI does not allow you to set both of these fields.
---
 .../non_ecs_timestamp_field/data.json         | 41 +++++++++++
 .../non_ecs_timestamp_field/mappings.json     | 36 ++++++++++
 .../execution_logic/eql.ts                    | 72 +++++++++++++++++++
 3 files changed, 149 insertions(+)
 create mode 100644 x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json
 create mode 100644 x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json

diff --git a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json b/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json
new file mode 100644
index 0000000000000..f70d74ddb80f0
--- /dev/null
+++ b/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json
@@ -0,0 +1,41 @@
+{
+  "type": "doc",
+  "value": {
+    "index": "non_ecs_timestamp_field",
+    "source": {
+      "timestamp": 1608131778,
+      "locale": "pt",
+      "created_at": 1622676795,
+      "event.category": "configuration"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "non_ecs_timestamp_field",
+    "source": {
+      "timestamp": 1608131778,
+      "locale": "es",
+      "created_at": 1622676790,
+      "event.category": "configuration"
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "non_ecs_timestamp_field",
+    "source": {
+      "timestamp": 1608131779,
+      "locale": "ua",
+      "created_at": 1622676785,
+      "event.category": "configuration"
+    },
+    "type": "_doc"
+  }
+}
diff --git a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json b/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json
new file mode 100644
index 0000000000000..729684a7f58f2
--- /dev/null
+++ b/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json
@@ -0,0 +1,36 @@
+{
+  "type": "index",
+  "value": {
+    "index": "non_ecs_timestamp_field",
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        },
+        "locale": {
+          "type": "keyword"
+        },
+        "created_at": {
+          "type": "date",
+          "format": "epoch_second"
+        },
+        "event": {
+          "properties": {
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}
diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
index d702295bf1f2d..3fa96a9084e40 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
@@ -657,5 +657,77 @@ export default ({ getService }: FtrProviderContext) => {
         expect(fullAlert?.['host.asset.criticality']).to.eql('high_impact');
       });
     });
+
+    describe.only('using data without a @timestamp field', () => {
+      before(async () => {
+        await esArchiver.load(
+          'x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field'
+        );
+      });
+
+      after(async () => {
+        await esArchiver.unload(
+          'x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field'
+        );
+      });
+
+      it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
+          timestamp_field: 'timestamp',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.be.empty();
+        expect(_log.warnings).to.contain(
+          'The following indices are missing the timestamp field "@timestamp": ["non_ecs_timestamp_field"]'
+        );
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts).to.be.empty();
+      });
+
+      it('specifying only timestamp_override results in an error, and no alerts are generated', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
+          timestamp_override: 'timestamp',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.contain(
+          'An error occurred during rule execution: message: "verification_exception\n\tRoot causes:\n\t\tverification_exception: Found 1 problem\nline -1:-1: Unknown column [@timestamp]"'
+        );
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts).to.be.empty();
+      });
+
+      it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
+          timestamp_field: 'timestamp',
+          timestamp_override: 'timestamp',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.be.empty();
+        expect(_log.warnings).to.be.empty();
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+
+        expect(previewAlerts).to.have.length(3);
+      });
+    });
   });
 };

From 78380394af723f2ac71a6d09bd302d0934e95360 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 23 Feb 2024 12:52:42 -0600
Subject: [PATCH 02/21] Add similar tests for data with @timestamp

---
 .../execution_logic/eql.ts                    | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
index 3fa96a9084e40..3dcce1b197065 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
@@ -658,6 +658,69 @@ export default ({ getService }: FtrProviderContext) => {
       });
     });
 
+    describe.only('using data with a @timestamp field', () => {
+      it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['auditbeat-*']),
+          timestamp_field: 'event.created',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.be.empty();
+        expect(_log.warnings).to.eql([
+          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
+        ]);
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts.length).to.be.greaterThan(0);
+      });
+
+      it('specifying only timestamp_override results in an error, and no alerts are generated', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['auditbeat-*']),
+          timestamp_override: 'event.created',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.be.empty();
+        expect(_log.warnings).to.eql([
+          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
+        ]);
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts.length).to.be.greaterThan(0);
+      });
+
+      it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
+        const rule: EqlRuleCreateProps = {
+          ...getEqlRuleForAlertTesting(['auditbeat-*']),
+          timestamp_field: 'event.created',
+          timestamp_override: 'event.created',
+        };
+
+        const {
+          previewId,
+          logs: [_log],
+        } = await previewRule({ supertest, rule });
+
+        expect(_log.errors).to.be.empty();
+        expect(_log.warnings).to.eql([
+          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
+        ]);
+
+        const previewAlerts = await getPreviewAlerts({ es, previewId });
+        expect(previewAlerts.length).to.be.greaterThan(0);
+      });
+    });
+
     describe.only('using data without a @timestamp field', () => {
       before(async () => {
         await esArchiver.load(

From 413ed8ca74285294eb8f7ebcbc70353e10464370 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 23 Feb 2024 21:38:09 -0600
Subject: [PATCH 03/21] Add tests validating that our EQL search strategy
 respects certain parameters

This is a sanity check to allowing our EQL validation functionality to
leverage these same options.
---
 .../eql_search/eql_search_strategy.test.ts    | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
index 475c43a5daed6..150c77ebc7ca5 100644
--- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
+++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
@@ -277,6 +277,51 @@ describe('EQL search strategy', () => {
 
         expect(requestOptions).toEqual({ ignore: [400], meta: true, signal: undefined });
       });
+
+      describe('EQL-specific arguments', () => {
+        it('passes along a timestamp_field argument', async () => {
+          const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
+          const request: EqlSearchStrategyRequest = {
+            // @ts-expect-error timestamp_field not allowed at top level when using `typesWithBodyKey`
+            params: { index: 'all', timestamp_field: 'timestamp' },
+          };
+
+          await firstValueFrom(eqlSearch.search(request, {}, mockDeps));
+          const [[actualParams]] = mockEqlSearch.mock.calls;
+
+          expect(actualParams).toEqual(expect.objectContaining({ timestamp_field: 'timestamp' }));
+        });
+
+        it('passes along an event_category_field argument', async () => {
+          const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
+          const request: EqlSearchStrategyRequest = {
+            // @ts-expect-error timestamp_field not allowed at top level when using `typesWithBodyKey`
+            params: { index: 'all', event_category_field: 'event_category' },
+          };
+
+          await firstValueFrom(eqlSearch.search(request, {}, mockDeps));
+          const [[actualParams]] = mockEqlSearch.mock.calls;
+
+          expect(actualParams).toEqual(
+            expect.objectContaining({ event_category_field: 'event_category' })
+          );
+        });
+
+        it('passes along a tiebreaker_field argument', async () => {
+          const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
+          const request: EqlSearchStrategyRequest = {
+            // @ts-expect-error tiebreaker_field not allowed at top level when using `typesWithBodyKey`
+            params: { index: 'all', tiebreaker_field: 'event_category' },
+          };
+
+          await firstValueFrom(eqlSearch.search(request, {}, mockDeps));
+          const [[actualParams]] = mockEqlSearch.mock.calls;
+
+          expect(actualParams).toEqual(
+            expect.objectContaining({ tiebreaker_field: 'event_category' })
+          );
+        });
+      });
     });
 
     describe('response', () => {

From cedecbba73dceba195dc590d54d10bbde4e6ba26 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 17:07:34 -0500
Subject: [PATCH 04/21] EQL validation function accepts EQL options as a
 parameter

This will break our calls, which is what we want (they need to pass
options now).
---
 .../security_solution/public/common/hooks/eql/api.ts       | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
index a707946be7625..0942470e0d3fa 100644
--- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
+++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
@@ -11,6 +11,7 @@ import type { EqlSearchStrategyRequest, EqlSearchStrategyResponse } from '@kbn/d
 import { EQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import type { EqlOptionsSelected } from '../../../../common/search_strategy';
 import {
   getValidationErrors,
   isErrorResponse,
@@ -23,6 +24,7 @@ interface Params {
   data: DataPublicPluginStart;
   signal: AbortSignal;
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
+  options: Omit<EqlOptionsSelected, 'query' | 'size'>;
 }
 
 export const validateEql = async ({
@@ -31,6 +33,7 @@ export const validateEql = async ({
   query,
   signal,
   runtimeMappings,
+  options,
 }: Params): Promise<{ valid: boolean; errors: string[] }> => {
   const { rawResponse: response } = await firstValueFrom(
     data.search.search<EqlSearchStrategyRequest, EqlSearchStrategyResponse>(
@@ -38,6 +41,10 @@ export const validateEql = async ({
         params: {
           index: dataViewTitle,
           body: { query, runtime_mappings: runtimeMappings, size: 0 },
+          // @ts-expect-error top-level keys are unexpected in {@link EqlSearchRequest}
+          timestamp_field: options?.timestampField,
+          tiebreaker_field: options?.tiebreakerField,
+          event_category_field: options?.eventCategoryField,
         },
         options: { ignore: [400] },
       },

From b2fa55c4af5e7c85644146548bcf4bd000357d1d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 20:48:21 -0500
Subject: [PATCH 05/21] Makes eqlOptions a used field on our Define Step form

Previously, the eqlOptions field was referenced in the schema of the
Define Step form, but as there was no component controlling the field,
it was unknown to the form data, and consequently unavailable to the
queryBar validator (which is ultimately our goal here).

By adding a component for the eqlOptions field (albeit hidden), and
updating our onOptionsChange callback to additionally set that field
data, the rule form now has eqlOptions available during validation of
the EQL query.
---
 .../components/eql_query_bar/validators.ts    | 11 ++-
 .../components/step_define_rule/index.tsx     | 73 +++++++++++--------
 .../public/shared_imports.ts                  |  2 +-
 3 files changed, 54 insertions(+), 32 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts
index 83b610eaaff7f..b3d6abc62d0b1 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.ts
@@ -58,7 +58,7 @@ export const eqlValidator = async (
   const [{ value, formData }] = args;
   const { query: queryValue } = value as FieldValueQueryBar;
   const query = queryValue.query as string;
-  const { dataViewId, index, ruleType } = formData as DefineStepRule;
+  const { dataViewId, index, ruleType, eqlOptions } = formData as DefineStepRule;
 
   const needsValidation =
     (ruleType === undefined && !isEmpty(query)) || (isEqlRule(ruleType) && !isEmpty(query));
@@ -82,7 +82,14 @@ export const eqlValidator = async (
     }
 
     const signal = new AbortController().signal;
-    const response = await validateEql({ data, query, signal, dataViewTitle, runtimeMappings });
+    const response = await validateEql({
+      data,
+      query,
+      signal,
+      dataViewTitle,
+      runtimeMappings,
+      options: eqlOptions,
+    });
 
     if (response?.valid === false) {
       return {
diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
index 87551441136ca..1301175bafaee 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
@@ -59,7 +59,14 @@ import { StepContentWrapper } from '../../../rule_creation/components/step_conte
 import { ThresholdInput } from '../threshold_input';
 import { SuppressionInfoIcon } from '../suppression_info_icon';
 import { EsqlInfoIcon } from '../../../rule_creation/components/esql_info_icon';
-import { Field, Form, getUseField, UseField, UseMultiFields } from '../../../../shared_imports';
+import {
+  Field,
+  Form,
+  getUseField,
+  HiddenField,
+  UseField,
+  UseMultiFields,
+} from '../../../../shared_imports';
 import type { FormHook } from '../../../../shared_imports';
 import { schema } from './schema';
 import { getTermsAggregationFields } from './utils';
@@ -768,14 +775,20 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
       onOpenTimeline,
     ]
   );
+
   const onOptionsChange = useCallback(
     (field: FieldsEqlOptions, value: string | undefined) => {
-      setOptionsSelected((prevOptions) => ({
-        ...prevOptions,
-        [field]: value,
-      }));
+      setOptionsSelected((prevOptions) => {
+        const newOptions = {
+          ...prevOptions,
+          [field]: value,
+        };
+
+        setFieldValue('eqlOptions', newOptions);
+        return newOptions;
+      });
     },
-    [setOptionsSelected]
+    [setFieldValue, setOptionsSelected]
   );
 
   const optionsData = useMemo(
@@ -838,29 +851,31 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
               </StyledVisibleContainer>
               <EuiSpacer size="s" />
               {isEqlRule(ruleType) ? (
-                <UseField
-                  key="EqlQueryBar"
-                  path="queryBar"
-                  component={EqlQueryBar}
-                  componentProps={{
-                    optionsData,
-                    optionsSelected,
-                    isSizeOptionDisabled: true,
-                    onOptionsChange,
-                    onValidityChange: setIsQueryBarValid,
-                    idAria: 'detectionEngineStepDefineRuleEqlQueryBar',
-                    isDisabled: isLoading,
-                    isLoading: isIndexPatternLoading,
-                    indexPattern,
-                    showFilterBar: true,
-                    // isLoading: indexPatternsLoading,
-                    dataTestSubj: 'detectionEngineStepDefineRuleEqlQueryBar',
-                  }}
-                  config={{
-                    ...schema.queryBar,
-                    label: i18n.EQL_QUERY_BAR_LABEL,
-                  }}
-                />
+                <>
+                  <UseField
+                    key="EqlQueryBar"
+                    path="queryBar"
+                    component={EqlQueryBar}
+                    componentProps={{
+                      optionsData,
+                      optionsSelected,
+                      isSizeOptionDisabled: true,
+                      onOptionsChange,
+                      onValidityChange: setIsQueryBarValid,
+                      idAria: 'detectionEngineStepDefineRuleEqlQueryBar',
+                      isDisabled: isLoading,
+                      isLoading: isIndexPatternLoading,
+                      indexPattern,
+                      showFilterBar: true,
+                      dataTestSubj: 'detectionEngineStepDefineRuleEqlQueryBar',
+                    }}
+                    config={{
+                      ...schema.queryBar,
+                      label: i18n.EQL_QUERY_BAR_LABEL,
+                    }}
+                  />
+                  <UseField path="eqlOptions" component={HiddenField} />
+                </>
               ) : isEsqlRule(ruleType) ? (
                 EsqlQueryBarMemo
               ) : (
diff --git a/x-pack/plugins/security_solution/public/shared_imports.ts b/x-pack/plugins/security_solution/public/shared_imports.ts
index 00993cfd7c6da..8686aecb3b99f 100644
--- a/x-pack/plugins/security_solution/public/shared_imports.ts
+++ b/x-pack/plugins/security_solution/public/shared_imports.ts
@@ -28,6 +28,6 @@ export {
   useFormData,
   VALIDATION_TYPES,
 } from '@kbn/es-ui-shared-plugin/static/forms/hook_form_lib';
-export { Field, SelectField } from '@kbn/es-ui-shared-plugin/static/forms/components';
+export { Field, SelectField, HiddenField } from '@kbn/es-ui-shared-plugin/static/forms/components';
 export { fieldValidators } from '@kbn/es-ui-shared-plugin/static/forms/helpers';
 export type { ERROR_CODE } from '@kbn/es-ui-shared-plugin/static/forms/helpers/field_validators/types';

From 3dc18c1c7b3a0f5234ffe34d2d8c6cf377179f3d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 20:52:19 -0500
Subject: [PATCH 06/21] Ensure that changes to EQL options cause the EQL query
 to be revalidated
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is one case where the form lib makes that super easy 👍.
---
 .../rule_creation_ui/components/step_define_rule/schema.tsx  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx
index 8a43698ab5ad4..49eba2d124d31 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx
@@ -130,9 +130,10 @@ export const schema: FormSchema<DefineStepRule> = {
     ),
     validations: [],
   },
-  eqlOptions: {},
+  eqlOptions: {
+    fieldsToValidateOnChange: ['eqlOptions', 'queryBar'],
+  },
   queryBar: {
-    fieldsToValidateOnChange: ['queryBar'],
     validations: [
       {
         validator: (

From 4105e699e41b6429aee355f8f6bc73615ffdd94d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 20:52:56 -0500
Subject: [PATCH 07/21] Add a test validating how onOptionsChange is called

This was mostly a sanity check, but good to document this API IMO.
---
 .../eql_query_bar/eql_query_bar.test.tsx      | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx
index aec082224e044..a9a520e802460 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.test.tsx
@@ -13,6 +13,7 @@ import { mockQueryBar } from '../../../rule_management_ui/components/rules_table
 import type { EqlQueryBarProps } from './eql_query_bar';
 import { EqlQueryBar } from './eql_query_bar';
 import { getEqlValidationError } from './validators.mock';
+import { fireEvent, render, within } from '@testing-library/react';
 
 jest.mock('../../../../common/lib/kibana');
 
@@ -117,4 +118,38 @@ describe('EqlQueryBar', () => {
 
     expect(wrapper.find('[data-test-subj="eql-validation-errors-popover"]').exists()).toEqual(true);
   });
+
+  describe('EQL options interaction', () => {
+    const mockOptionsData = {
+      keywordFields: [],
+      dateFields: [{ label: 'timestamp', value: 'timestamp' }],
+      nonDateFields: [],
+    };
+
+    it('invokes onOptionsChange when the EQL options change', () => {
+      const onOptionsChangeMock = jest.fn();
+
+      const { getByTestId, getByText } = render(
+        <TestProviders>
+          <EqlQueryBar
+            dataTestSubj="myQueryBar"
+            field={mockField}
+            isLoading={false}
+            optionsData={mockOptionsData}
+            indexPattern={mockIndexPattern}
+            onOptionsChange={onOptionsChangeMock}
+          />
+        </TestProviders>
+      );
+
+      // open options popover
+      fireEvent.click(getByTestId('eql-settings-trigger'));
+      // display combobox options
+      within(getByTestId(`eql-timestamp-field`)).getByRole('combobox').focus();
+      // select timestamp
+      getByText('timestamp').click();
+
+      expect(onOptionsChangeMock).toHaveBeenCalledWith('timestampField', 'timestamp');
+    });
+  });
 });

From 39d005b4028d7d6c9cb551ecac3de1208761b46c Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 20:54:15 -0500
Subject: [PATCH 08/21] Remove unused interface

---
 .../components/eql_query_bar/eql_query_bar.tsx            | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx
index fe1962a71f6c6..607b7a3ef2bb2 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/eql_query_bar.tsx
@@ -11,7 +11,7 @@ import { Subscription } from 'rxjs';
 import styled from 'styled-components';
 import deepEqual from 'fast-deep-equal';
 import { EuiFormRow, EuiSpacer, EuiTextArea } from '@elastic/eui';
-import type { DataViewBase, Filter, Query } from '@kbn/es-query';
+import type { DataViewBase } from '@kbn/es-query';
 import { FilterManager } from '@kbn/data-plugin/public';
 
 import type { FieldHook } from '../../../../shared_imports';
@@ -58,12 +58,6 @@ const StyledFormRow = styled(EuiFormRow)`
   }
 `;
 
-export interface FieldValueQueryBar {
-  filters: Filter[];
-  query: Query;
-  saved_id?: string;
-}
-
 export interface EqlQueryBarProps {
   dataTestSubj: string;
   field: FieldHook<DefineStepRule['queryBar']>;

From daf87a46a2c375479100823f4030b2aebc7c4980 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 21:22:45 -0500
Subject: [PATCH 09/21] Add an eqlOptions form field to the timeline EQL form

By adding the field to the form itself (as a hidden field), and
modifying the onOptionsChange callback to additionally update the form
value, we now have that field value available to us when validating the
EQL query itself.

This is all near-identical to the changes just made to
the Define Step rule creation form.
---
 .../timeline/query_bar/eql/index.tsx          | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx
index b1e1e0934d1d6..8e165a2303036 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/index.tsx
@@ -10,7 +10,10 @@ import React, { memo, useCallback, useEffect, useMemo, useRef, useState } from '
 import { useDispatch } from 'react-redux';
 import styled from 'styled-components';
 
-import type { FieldsEqlOptions } from '../../../../../../common/search_strategy';
+import type {
+  EqlOptionsSelected,
+  FieldsEqlOptions,
+} from '../../../../../../common/search_strategy';
 import { useSourcererDataView } from '../../../../../common/containers/sourcerer';
 import { useDeepEqualSelector } from '../../../../../common/hooks/use_selector';
 import { SourcererScopeName } from '../../../../../common/store/sourcerer/model';
@@ -31,6 +34,7 @@ import { getEqlOptions } from './selectors';
 interface TimelineEqlQueryBar {
   index: string[];
   eqlQueryBar: FieldValueQueryBar;
+  eqlOptions: EqlOptionsSelected;
 }
 
 const defaultValues = {
@@ -40,6 +44,7 @@ const defaultValues = {
     filters: [],
     saved_id: null,
   },
+  eqlOptions: {},
 };
 
 const schema: FormSchema<TimelineEqlQueryBar> = {
@@ -47,6 +52,9 @@ const schema: FormSchema<TimelineEqlQueryBar> = {
     fieldsToValidateOnChange: ['index', 'eqlQueryBar'],
     validations: [],
   },
+  eqlOptions: {
+    fieldsToValidateOnChange: ['eqlOptions', 'eqlQueryBar'],
+  },
   eqlQueryBar: {
     validations: [
       {
@@ -89,18 +97,20 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string })
     options: { stripEmptyFields: false },
     schema,
   });
-  const { getFields } = form;
+  const { getFields, setFieldValue } = form;
 
   const onOptionsChange = useCallback(
-    (field: FieldsEqlOptions, value: string | undefined) =>
+    (field: FieldsEqlOptions, value: string | undefined) => {
       dispatch(
         timelineActions.updateEqlOptions({
           id: timelineId,
           field,
           value,
         })
-      ),
-    [dispatch, timelineId]
+      );
+      setFieldValue('eqlOptions', { ...optionsSelected, [field]: value });
+    },
+    [dispatch, optionsSelected, setFieldValue, timelineId]
   );
 
   const [{ eqlQueryBar: formEqlQueryBar }] = useFormData<TimelineEqlQueryBar>({
@@ -179,6 +189,7 @@ export const EqlQueryBarTimeline = memo(({ timelineId }: { timelineId: string })
   return (
     <Form form={form} data-test-subj="EqlQueryBarTimeline">
       <HiddenUseField key="Index" path="index" />
+      <HiddenUseField key="EqlOptions" path="eqlOptions" />
       <UseField
         key="EqlQueryBar"
         path="eqlQueryBar"

From 10a47fdaba5860a44638aada78c0cf8c37b2c580 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 11 Mar 2024 21:25:48 -0500
Subject: [PATCH 10/21] Remove default values from timeline EQL options

* The default value of `''` for `tiebreaker_field` is invalid and must be
  changed by the user
* The redux store selector was using default values that both didn't make sense and
  were never used
* Removes defaulting behavior for options in the timeline EQL search
  strategy, since sending `timestamp_field: undefined` is equivalent to
  not specifying any value (and thus falls back to the equivalent EQL
  default)
---
 .../timeline/query_bar/eql/selectors.tsx           | 14 +++-----------
 .../public/timelines/store/defaults.ts             |  6 +++---
 .../server/search_strategy/timeline/eql/helpers.ts |  4 ++--
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
index e533ef9175178..bf7e4d73203bc 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
@@ -13,17 +13,9 @@ export const getEqlOptions = () =>
     selectTimeline,
     (timeline) =>
       timeline?.eqlOptions ?? {
-        eventCategoryField: [{ label: 'event.category' }],
-        tiebreakerField: [
-          {
-            label: '',
-          },
-        ],
-        timestampField: [
-          {
-            label: '@timestamp',
-          },
-        ],
+        eventCategoryField: undefined,
+        tiebreakerField: undefined,
+        timestampField: undefined,
         size: 100,
         query: '',
       }
diff --git a/x-pack/plugins/security_solution/public/timelines/store/defaults.ts b/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
index 9fb94d9f5b1b9..b622489d0028b 100644
--- a/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
+++ b/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
@@ -27,9 +27,9 @@ export const timelineDefaults: SubsetTimelineModel &
   dateRange: { start, end },
   description: '',
   eqlOptions: {
-    eventCategoryField: 'event.category',
-    tiebreakerField: '',
-    timestampField: '@timestamp',
+    eventCategoryField: undefined,
+    tiebreakerField: undefined,
+    timestampField: undefined,
     query: '',
     size: 100,
   },
diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
index 2b4f562954df1..b1b07c6967318 100644
--- a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
+++ b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
@@ -44,7 +44,7 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string,
     index: options.defaultIndex,
     ignore_unavailable: true,
     body: {
-      event_category_field: options.eventCategoryField ?? 'event.category',
+      event_category_field: options.eventCategoryField,
       filter: {
         bool: {
           filter: requestFilter,
@@ -57,7 +57,7 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string,
           }
         : {}),
       size: options.size ?? 100,
-      timestamp_field: options.timestampField ?? '@timestamp',
+      timestamp_field: options.timestampField,
       fields: [
         { field: '*', include_unmapped: true },
         {

From bbd6a8df093b7424fefa61f4d4143c9431352e2c Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 12 Mar 2024 13:32:36 -0500
Subject: [PATCH 11/21] Rename new archive to be more accurate

While the data I wrote was "non-ECS," that's not a necessary feature of
the bug here. It simply needs to _not_ have a @timestamp field.

This also updates the corresponding FTR tests to not be exclusive
(whoops).
---
 .../no_@timestamp_field/data.json             | 38 +++++++++++++++++
 .../mappings.json                             | 14 +++----
 .../non_ecs_timestamp_field/data.json         | 41 -------------------
 .../execution_logic/eql.ts                    | 24 +++++------
 4 files changed, 55 insertions(+), 62 deletions(-)
 create mode 100644 x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json
 rename x-pack/test/functional/es_archives/security_solution/{non_ecs_timestamp_field => no_@timestamp_field}/mappings.json (67%)
 delete mode 100644 x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json

diff --git a/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json b/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json
new file mode 100644
index 0000000000000..92b3341164979
--- /dev/null
+++ b/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json
@@ -0,0 +1,38 @@
+{
+  "type": "doc",
+  "value": {
+    "index": "no_@timestamp_field",
+    "source": {
+      "locale": "pt",
+      "event.category": "configuration",
+      "event.ingested": 1608131778
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "no_@timestamp_field",
+    "source": {
+      "locale": "es",
+      "event.category": "configuration",
+      "event.ingested": 1608131778
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "no_@timestamp_field",
+    "source": {
+      "locale": "ua",
+      "event.category": "configuration",
+      "event.ingested": 1608131779
+    },
+    "type": "_doc"
+  }
+}
diff --git a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json b/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json
similarity index 67%
rename from x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json
rename to x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json
index 729684a7f58f2..76652ff749bdf 100644
--- a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/mappings.json
+++ b/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json
@@ -1,26 +1,22 @@
 {
   "type": "index",
   "value": {
-    "index": "non_ecs_timestamp_field",
+    "index": "no_@timestamp_field",
     "mappings": {
       "dynamic": "strict",
       "properties": {
-        "timestamp": {
-          "type": "date",
-          "format": "epoch_second"
-        },
         "locale": {
           "type": "keyword"
         },
-        "created_at": {
-          "type": "date",
-          "format": "epoch_second"
-        },
         "event": {
           "properties": {
             "category": {
               "ignore_above": 1024,
               "type": "keyword"
+            },
+            "ingested": {
+              "type": "date",
+              "format": "epoch_second"
             }
           }
         }
diff --git a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json b/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json
deleted file mode 100644
index f70d74ddb80f0..0000000000000
--- a/x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field/data.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
-  "type": "doc",
-  "value": {
-    "index": "non_ecs_timestamp_field",
-    "source": {
-      "timestamp": 1608131778,
-      "locale": "pt",
-      "created_at": 1622676795,
-      "event.category": "configuration"
-    },
-    "type": "_doc"
-  }
-}
-
-{
-  "type": "doc",
-  "value": {
-    "index": "non_ecs_timestamp_field",
-    "source": {
-      "timestamp": 1608131778,
-      "locale": "es",
-      "created_at": 1622676790,
-      "event.category": "configuration"
-    },
-    "type": "_doc"
-  }
-}
-
-{
-  "type": "doc",
-  "value": {
-    "index": "non_ecs_timestamp_field",
-    "source": {
-      "timestamp": 1608131779,
-      "locale": "ua",
-      "created_at": 1622676785,
-      "event.category": "configuration"
-    },
-    "type": "_doc"
-  }
-}
diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
index 3dcce1b197065..798901df39dc5 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
@@ -658,7 +658,7 @@ export default ({ getService }: FtrProviderContext) => {
       });
     });
 
-    describe.only('using data with a @timestamp field', () => {
+    describe('using data with a @timestamp field', () => {
       it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
@@ -721,23 +721,23 @@ export default ({ getService }: FtrProviderContext) => {
       });
     });
 
-    describe.only('using data without a @timestamp field', () => {
+    describe('using data without a @timestamp field', () => {
       before(async () => {
         await esArchiver.load(
-          'x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field'
+          'x-pack/test/functional/es_archives/security_solution/no_@timestamp_field'
         );
       });
 
       after(async () => {
         await esArchiver.unload(
-          'x-pack/test/functional/es_archives/security_solution/non_ecs_timestamp_field'
+          'x-pack/test/functional/es_archives/security_solution/no_@timestamp_field'
         );
       });
 
       it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
-          timestamp_field: 'timestamp',
+          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          timestamp_field: 'event.ingested',
         };
 
         const {
@@ -747,7 +747,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         expect(_log.errors).to.be.empty();
         expect(_log.warnings).to.contain(
-          'The following indices are missing the timestamp field "@timestamp": ["non_ecs_timestamp_field"]'
+          'The following indices are missing the timestamp field "@timestamp": ["no_@timestamp_field"]'
         );
 
         const previewAlerts = await getPreviewAlerts({ es, previewId });
@@ -756,8 +756,8 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('specifying only timestamp_override results in an error, and no alerts are generated', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
-          timestamp_override: 'timestamp',
+          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          timestamp_override: 'event.ingested',
         };
 
         const {
@@ -775,9 +775,9 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['non_ecs_timestamp_field']),
-          timestamp_field: 'timestamp',
-          timestamp_override: 'timestamp',
+          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          timestamp_field: 'event.ingested',
+          timestamp_override: 'event.ingested',
         };
 
         const {

From fa03d220468e80d57619ba3ebf521399497ac953 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 12 Mar 2024 16:32:57 -0500
Subject: [PATCH 12/21] Rename archive, again

While the '@' in the file/folder seems to work, I'm a little dubious of
the portability. Using normal characters here just to be safe.
---
 .../data.json                                        |  6 +++---
 .../mappings.json                                    |  2 +-
 .../execution_logic/eql.ts                           | 12 ++++++------
 3 files changed, 10 insertions(+), 10 deletions(-)
 rename x-pack/test/functional/es_archives/security_solution/{no_@timestamp_field => no_at_timestamp_field}/data.json (82%)
 rename x-pack/test/functional/es_archives/security_solution/{no_@timestamp_field => no_at_timestamp_field}/mappings.json (93%)

diff --git a/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json b/x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/data.json
similarity index 82%
rename from x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json
rename to x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/data.json
index 92b3341164979..3ee1387b01e5c 100644
--- a/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/data.json
+++ b/x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/data.json
@@ -1,7 +1,7 @@
 {
   "type": "doc",
   "value": {
-    "index": "no_@timestamp_field",
+    "index": "no_at_timestamp_field",
     "source": {
       "locale": "pt",
       "event.category": "configuration",
@@ -14,7 +14,7 @@
 {
   "type": "doc",
   "value": {
-    "index": "no_@timestamp_field",
+    "index": "no_at_timestamp_field",
     "source": {
       "locale": "es",
       "event.category": "configuration",
@@ -27,7 +27,7 @@
 {
   "type": "doc",
   "value": {
-    "index": "no_@timestamp_field",
+    "index": "no_at_timestamp_field",
     "source": {
       "locale": "ua",
       "event.category": "configuration",
diff --git a/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json b/x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/mappings.json
similarity index 93%
rename from x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json
rename to x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/mappings.json
index 76652ff749bdf..76f0aa8521fc3 100644
--- a/x-pack/test/functional/es_archives/security_solution/no_@timestamp_field/mappings.json
+++ b/x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field/mappings.json
@@ -1,7 +1,7 @@
 {
   "type": "index",
   "value": {
-    "index": "no_@timestamp_field",
+    "index": "no_at_timestamp_field",
     "mappings": {
       "dynamic": "strict",
       "properties": {
diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
index 798901df39dc5..9fc5cd055d137 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
@@ -724,19 +724,19 @@ export default ({ getService }: FtrProviderContext) => {
     describe('using data without a @timestamp field', () => {
       before(async () => {
         await esArchiver.load(
-          'x-pack/test/functional/es_archives/security_solution/no_@timestamp_field'
+          'x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field'
         );
       });
 
       after(async () => {
         await esArchiver.unload(
-          'x-pack/test/functional/es_archives/security_solution/no_@timestamp_field'
+          'x-pack/test/functional/es_archives/security_solution/no_at_timestamp_field'
         );
       });
 
       it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          ...getEqlRuleForAlertTesting(['no_at_timestamp_field']),
           timestamp_field: 'event.ingested',
         };
 
@@ -747,7 +747,7 @@ export default ({ getService }: FtrProviderContext) => {
 
         expect(_log.errors).to.be.empty();
         expect(_log.warnings).to.contain(
-          'The following indices are missing the timestamp field "@timestamp": ["no_@timestamp_field"]'
+          'The following indices are missing the timestamp field "@timestamp": ["no_at_timestamp_field"]'
         );
 
         const previewAlerts = await getPreviewAlerts({ es, previewId });
@@ -756,7 +756,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('specifying only timestamp_override results in an error, and no alerts are generated', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          ...getEqlRuleForAlertTesting(['no_at_timestamp_field']),
           timestamp_override: 'event.ingested',
         };
 
@@ -775,7 +775,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
         const rule: EqlRuleCreateProps = {
-          ...getEqlRuleForAlertTesting(['no_@timestamp_field']),
+          ...getEqlRuleForAlertTesting(['no_at_timestamp_field']),
           timestamp_field: 'event.ingested',
           timestamp_override: 'event.ingested',
         };

From b2aec768b12069fa7163a7e9fee2501adefdd3e0 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 12 Mar 2024 17:02:01 -0500
Subject: [PATCH 13/21] More accurate cypress selector

This is retrieving the general rule index pattern input (of which there
is no other reference); I think the naming was due to it originally
being introduced while testing the IM rule type.
---
 .../rule_creation/indicator_match_rule.cy.ts           |  4 ++--
 .../cypress/screens/create_new_rule.ts                 |  2 +-
 .../cypress/tasks/create_new_rule.ts                   | 10 +++++-----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts
index 9ce0bbb5be44a..c62c13144b5ab 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/indicator_match_rule.cy.ts
@@ -84,7 +84,7 @@ import {
   getIndicatorAndButton,
   getIndicatorAtLeastOneInvalidationText,
   getIndicatorDeleteButton,
-  getIndicatorIndex,
+  getRuleIndexInput,
   getIndicatorIndexComboField,
   getIndicatorIndicatorIndex,
   getIndicatorInvalidationText,
@@ -141,7 +141,7 @@ describe('indicator match', { tags: ['@ess', '@serverless'] }, () => {
         });
 
         it('Contains a predefined index pattern', () => {
-          getIndicatorIndex().should('have.text', getIndexPatterns().join(''));
+          getRuleIndexInput().should('have.text', getIndexPatterns().join(''));
         });
 
         it('Does NOT show invalidation text on initial page load if indicator index pattern is filled out', () => {
diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
index 0b521f68a787a..bb23ff99a7aa9 100644
--- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
+++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
@@ -69,7 +69,7 @@ export const THREAT_MATCH_CUSTOM_QUERY_INPUT =
 export const THREAT_MATCH_QUERY_INPUT =
   '[data-test-subj="detectionEngineStepDefineThreatRuleQueryBar"] [data-test-subj="queryInput"]';
 
-export const THREAT_MATCH_INDICATOR_INDEX =
+export const CUSTOM_INDEX_PATTERN_INPUT =
   '[data-test-subj="detectionEngineStepDefineRuleIndices"] [data-test-subj="comboBoxInput"]';
 
 export const THREAT_MATCH_INDICATOR_INDICATOR_INDEX =
diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts
index 62091e0e39a5f..c909788bef6d2 100644
--- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts
+++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts
@@ -105,7 +105,7 @@ import {
   THREAT_MAPPING_COMBO_BOX_INPUT,
   THREAT_MATCH_AND_BUTTON,
   THREAT_MATCH_CUSTOM_QUERY_INPUT,
-  THREAT_MATCH_INDICATOR_INDEX,
+  CUSTOM_INDEX_PATTERN_INPUT,
   THREAT_MATCH_INDICATOR_INDICATOR_INDEX,
   THREAT_MATCH_OR_BUTTON,
   THREAT_MATCH_QUERY_INPUT,
@@ -652,7 +652,7 @@ export const fillIndexAndIndicatorIndexPattern = (
 ) => {
   getIndexPatternClearButton().click();
 
-  getIndicatorIndex().type(`${indexPattern}{enter}`);
+  getRuleIndexInput().type(`${indexPattern}{enter}`);
   getIndicatorIndicatorIndex().type(`{backspace}{enter}${indicatorIndex}{enter}`);
 };
 
@@ -689,9 +689,9 @@ export const getAboutContinueButton = () => cy.get(ABOUT_CONTINUE_BTN);
 /** Returns the continue button on the step of define */
 export const getDefineContinueButton = () => cy.get(DEFINE_CONTINUE_BUTTON);
 
-/** Returns the indicator index pattern */
-export const getIndicatorIndex = () => {
-  return cy.get(THREAT_MATCH_INDICATOR_INDEX).eq(0);
+/** Returns the custom rule index pattern input */
+export const getRuleIndexInput = () => {
+  return cy.get(CUSTOM_INDEX_PATTERN_INPUT).eq(0);
 };
 
 /** Returns the indicator's indicator index */

From 0d8ad8b521b27a7a6b7a1289124e2180c1831fee Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 12 Mar 2024 17:37:30 -0500
Subject: [PATCH 14/21] Add a cypress test around the EQL option/validation
 functionality

This automates the process that I've been using to test this: given data
without an `@timestamp` (the default time field for EQL), you'll see
errors when we attempt to validate the EQL query UNTIL you choose
another timestamp field via the EQL options.
---
 .../event_correlation_rule.cy.ts              | 39 +++++++++++++++++++
 .../cypress/screens/create_new_rule.ts        |  5 +++
 .../no_at_timestamp_field/data.json           | 38 ++++++++++++++++++
 .../no_at_timestamp_field/mappings.json       | 32 +++++++++++++++
 4 files changed, 114 insertions(+)
 create mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
 create mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
index f02ec20ffd685..05ff0d9bfb92a 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
@@ -49,6 +49,8 @@ import {
   fillAboutRuleAndContinue,
   fillDefineEqlRuleAndContinue,
   fillScheduleRuleAndContinue,
+  getIndexPatternClearButton,
+  getRuleIndexInput,
   selectEqlRuleType,
   waitForAlertsToPopulate,
 } from '../../../../tasks/create_new_rule';
@@ -56,6 +58,13 @@ import { login } from '../../../../tasks/login';
 import { visit } from '../../../../tasks/navigation';
 import { openRuleManagementPageViaBreadcrumbs } from '../../../../tasks/rules_management';
 import { CREATE_RULE_URL } from '../../../../urls/navigation';
+import {
+  EQL_OPTIONS_POPOVER_TRIGGER,
+  EQL_OPTIONS_TIMESTAMP_INPUT,
+  EQL_QUERY_INPUT,
+  EQL_QUERY_VALIDATION_ERROR,
+  RULES_CREATION_FORM,
+} from '../../../../screens/create_new_rule';
 
 describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => {
   beforeEach(() => {
@@ -174,4 +183,34 @@ describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => {
         });
     });
   });
+
+  describe('with source data requiring EQL overrides', () => {
+    before(() => {
+      cy.task('esArchiverLoad', { archiveName: 'no_at_timestamp_field' });
+    });
+
+    after(() => {
+      cy.task('esArchiverUnload', 'no_at_timestamp_field');
+    });
+
+    it('includes EQL options in query validation', () => {
+      login();
+      visit(CREATE_RULE_URL);
+      selectEqlRuleType();
+      getIndexPatternClearButton().click();
+      getRuleIndexInput().type(`no_at_timestamp_field{enter}`);
+
+      cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).should('exist');
+      cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).should('be.visible');
+      cy.get(RULES_CREATION_FORM).find(EQL_QUERY_INPUT).type('any where true');
+
+      cy.get(EQL_QUERY_VALIDATION_ERROR).should('be.visible');
+      cy.get(EQL_QUERY_VALIDATION_ERROR).should('have.text', '1');
+
+      cy.get(RULES_CREATION_FORM).find(EQL_OPTIONS_POPOVER_TRIGGER).click();
+      cy.get(EQL_OPTIONS_TIMESTAMP_INPUT).type('event.ingested{enter}');
+
+      cy.get(EQL_QUERY_VALIDATION_ERROR).should('not.exist');
+    });
+  });
 });
diff --git a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
index bb23ff99a7aa9..41ff094a8f66e 100644
--- a/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
+++ b/x-pack/test/security_solution_cypress/cypress/screens/create_new_rule.ts
@@ -114,6 +114,11 @@ export const EQL_QUERY_VALIDATION_SPINNER = '[data-test-subj="eql-validation-loa
 
 export const EQL_QUERY_VALIDATION_ERROR = '[data-test-subj="eql-validation-errors-popover-button"]';
 
+export const EQL_OPTIONS_POPOVER_TRIGGER = '[data-test-subj="eql-settings-trigger"]';
+
+export const EQL_OPTIONS_TIMESTAMP_INPUT =
+  '[data-test-subj="eql-timestamp-field"] [data-test-subj="comboBoxInput"]';
+
 export const IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK =
   '[data-test-subj="importQueryFromSavedTimeline"]';
 
diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
new file mode 100644
index 0000000000000..3ee1387b01e5c
--- /dev/null
+++ b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
@@ -0,0 +1,38 @@
+{
+  "type": "doc",
+  "value": {
+    "index": "no_at_timestamp_field",
+    "source": {
+      "locale": "pt",
+      "event.category": "configuration",
+      "event.ingested": 1608131778
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "no_at_timestamp_field",
+    "source": {
+      "locale": "es",
+      "event.category": "configuration",
+      "event.ingested": 1608131778
+    },
+    "type": "_doc"
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "index": "no_at_timestamp_field",
+    "source": {
+      "locale": "ua",
+      "event.category": "configuration",
+      "event.ingested": 1608131779
+    },
+    "type": "_doc"
+  }
+}
diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json
new file mode 100644
index 0000000000000..76f0aa8521fc3
--- /dev/null
+++ b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json
@@ -0,0 +1,32 @@
+{
+  "type": "index",
+  "value": {
+    "index": "no_at_timestamp_field",
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "locale": {
+          "type": "keyword"
+        },
+        "event": {
+          "properties": {
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "ingested": {
+              "type": "date",
+              "format": "epoch_second"
+            }
+          }
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}

From 3a9d04059f9e8bb22ca77aeff15d7028cb7f3d98 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Tue, 12 Mar 2024 21:50:11 -0500
Subject: [PATCH 15/21] Prevent component props from being passed as DOM
 attributes

When you don't specify a component with UseField, it effectively renders
something like `() => <input {...props} />`, which in this case meant
that `euiFieldProps` was being set as a DOM attribute on the input
element, and react was complaining about that.

This solves the issue by specifying a component to use (HiddenField,
which is effectively `() => null`), and consequently removes the
unnecessary `display: none` wrapper as well.

Since this field is never displayed, one wonders why it has those
`euiFieldProps` at all, but that's a question for another day.
---
 .../components/step_define_rule/index.tsx     | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
index 1301175bafaee..d8b4d2694dcb0 100644
--- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
+++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
@@ -827,17 +827,16 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     <>
       <StepContentWrapper addPadding={!isUpdateView}>
         <Form form={form} data-test-subj="stepDefineRule">
-          <StyledVisibleContainer isVisible={false}>
-            <UseField
-              path="dataSourceType"
-              componentProps={{
-                euiFieldProps: {
-                  fullWidth: true,
-                  placeholder: '',
-                },
-              }}
-            />
-          </StyledVisibleContainer>
+          <UseField
+            path="dataSourceType"
+            component={HiddenField}
+            componentProps={{
+              euiFieldProps: {
+                fullWidth: true,
+                placeholder: '',
+              },
+            }}
+          />
           <UseField
             path="ruleType"
             component={SelectRuleType}

From 532dd15575879255005df922b704d26fa44aeadf Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 13 Mar 2024 12:00:10 -0500
Subject: [PATCH 16/21] Revert "Remove default values from timeline EQL
 options"

This reverts commit 10a47fdaba5860a44638aada78c0cf8c37b2c580.

While we're still discussing how to handle the invalid default for
`tiebreaker_field`, and the discrepancy with the rule creation form,
these changes are not immediately necessary to fix the bug at hand.
---
 .../timeline/query_bar/eql/selectors.tsx           | 14 +++++++++++---
 .../public/timelines/store/defaults.ts             |  6 +++---
 .../server/search_strategy/timeline/eql/helpers.ts |  4 ++--
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
index bf7e4d73203bc..e533ef9175178 100644
--- a/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
+++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/query_bar/eql/selectors.tsx
@@ -13,9 +13,17 @@ export const getEqlOptions = () =>
     selectTimeline,
     (timeline) =>
       timeline?.eqlOptions ?? {
-        eventCategoryField: undefined,
-        tiebreakerField: undefined,
-        timestampField: undefined,
+        eventCategoryField: [{ label: 'event.category' }],
+        tiebreakerField: [
+          {
+            label: '',
+          },
+        ],
+        timestampField: [
+          {
+            label: '@timestamp',
+          },
+        ],
         size: 100,
         query: '',
       }
diff --git a/x-pack/plugins/security_solution/public/timelines/store/defaults.ts b/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
index b622489d0028b..9fb94d9f5b1b9 100644
--- a/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
+++ b/x-pack/plugins/security_solution/public/timelines/store/defaults.ts
@@ -27,9 +27,9 @@ export const timelineDefaults: SubsetTimelineModel &
   dateRange: { start, end },
   description: '',
   eqlOptions: {
-    eventCategoryField: undefined,
-    tiebreakerField: undefined,
-    timestampField: undefined,
+    eventCategoryField: 'event.category',
+    tiebreakerField: '',
+    timestampField: '@timestamp',
     query: '',
     size: 100,
   },
diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
index b1b07c6967318..2b4f562954df1 100644
--- a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
+++ b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts
@@ -44,7 +44,7 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string,
     index: options.defaultIndex,
     ignore_unavailable: true,
     body: {
-      event_category_field: options.eventCategoryField,
+      event_category_field: options.eventCategoryField ?? 'event.category',
       filter: {
         bool: {
           filter: requestFilter,
@@ -57,7 +57,7 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record<string,
           }
         : {}),
       size: options.size ?? 100,
-      timestamp_field: options.timestampField,
+      timestamp_field: options.timestampField ?? '@timestamp',
       fields: [
         { field: '*', include_unmapped: true },
         {

From 5b1885497e1c53b7c6a3513ed74e7300c80f709d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 13 Mar 2024 12:09:26 -0500
Subject: [PATCH 17/21] Guard against using timeline default of '' for
 tibreakerField

An empty string is the default value for the EQL option
`tiebreaker_field` in the timeline EQL form. However, if this is passed
directly to ES an error will be thrown, as this is never a valid value
for that option.

A similar guard exists in the timeline EQL search strategy; we're adding
one to our validation call as well now that it has to deal with those
values.
---
 x-pack/plugins/security_solution/public/common/hooks/eql/api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
index 0942470e0d3fa..22b49f5c12f1c 100644
--- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
+++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
@@ -43,7 +43,7 @@ export const validateEql = async ({
           body: { query, runtime_mappings: runtimeMappings, size: 0 },
           // @ts-expect-error top-level keys are unexpected in {@link EqlSearchRequest}
           timestamp_field: options?.timestampField,
-          tiebreaker_field: options?.tiebreakerField,
+          tiebreaker_field: options?.tiebreakerField || undefined,
           event_category_field: options?.eventCategoryField,
         },
         options: { ignore: [400] },

From e837f19373ff9f7b99dcc6825878f62b055afc1a Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Wed, 13 Mar 2024 12:59:09 -0500
Subject: [PATCH 18/21] validateEql options parameter is optional

These additional parameters do not need to be specified in order to
produce a valid query, but I am requiring that the key be specified for
sanity's sake.
---
 x-pack/plugins/security_solution/public/common/hooks/eql/api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
index 22b49f5c12f1c..6a911c522d884 100644
--- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
+++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
@@ -24,7 +24,7 @@ interface Params {
   data: DataPublicPluginStart;
   signal: AbortSignal;
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
-  options: Omit<EqlOptionsSelected, 'query' | 'size'>;
+  options: Omit<EqlOptionsSelected, 'query' | 'size'> | undefined;
 }
 
 export const validateEql = async ({

From 7ac409a010eaea4f1cdb317a997e50c1f50c273d Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Fri, 15 Mar 2024 13:58:32 -0500
Subject: [PATCH 19/21] Remove duplicated es_archive thanks to new cypress task
 functionality

In #178724, we added the ability to retrieve archives from another
location (namely, where our FTR archives live). This allows me to delete
the duplicated archive specific for the cypress usage, and share the FTR
archive between both. Nice!
---
 .../event_correlation_rule.cy.ts              |  4 +-
 .../no_at_timestamp_field/data.json           | 38 -------------------
 .../no_at_timestamp_field/mappings.json       | 32 ----------------
 3 files changed, 2 insertions(+), 72 deletions(-)
 delete mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
 delete mode 100644 x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json

diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
index 7b4c4a7ed86f8..17b767be45565 100644
--- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
+++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/event_correlation_rule.cy.ts
@@ -186,11 +186,11 @@ describe('EQL rules', { tags: ['@ess', '@serverless'] }, () => {
 
   describe('with source data requiring EQL overrides', () => {
     before(() => {
-      cy.task('esArchiverLoad', { archiveName: 'no_at_timestamp_field' });
+      cy.task('esArchiverLoad', { archiveName: 'no_at_timestamp_field', type: 'ftr' });
     });
 
     after(() => {
-      cy.task('esArchiverUnload', 'no_at_timestamp_field');
+      cy.task('esArchiverUnload', { archiveName: 'no_at_timestamp_field', type: 'ftr' });
     });
 
     it('includes EQL options in query validation', () => {
diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
deleted file mode 100644
index 3ee1387b01e5c..0000000000000
--- a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/data.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-  "type": "doc",
-  "value": {
-    "index": "no_at_timestamp_field",
-    "source": {
-      "locale": "pt",
-      "event.category": "configuration",
-      "event.ingested": 1608131778
-    },
-    "type": "_doc"
-  }
-}
-
-{
-  "type": "doc",
-  "value": {
-    "index": "no_at_timestamp_field",
-    "source": {
-      "locale": "es",
-      "event.category": "configuration",
-      "event.ingested": 1608131778
-    },
-    "type": "_doc"
-  }
-}
-
-{
-  "type": "doc",
-  "value": {
-    "index": "no_at_timestamp_field",
-    "source": {
-      "locale": "ua",
-      "event.category": "configuration",
-      "event.ingested": 1608131779
-    },
-    "type": "_doc"
-  }
-}
diff --git a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json b/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json
deleted file mode 100644
index 76f0aa8521fc3..0000000000000
--- a/x-pack/test/security_solution_cypress/es_archives/no_at_timestamp_field/mappings.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-  "type": "index",
-  "value": {
-    "index": "no_at_timestamp_field",
-    "mappings": {
-      "dynamic": "strict",
-      "properties": {
-        "locale": {
-          "type": "keyword"
-        },
-        "event": {
-          "properties": {
-            "category": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "ingested": {
-              "type": "date",
-              "format": "epoch_second"
-            }
-          }
-        }
-      }
-    },
-    "settings": {
-      "index": {
-        "number_of_replicas": "1",
-        "number_of_shards": "1"
-      }
-    }
-  }
-}

From 676718221ebedf65c7f9716ccd96ecae769bbbb5 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 18 Mar 2024 17:18:00 -0500
Subject: [PATCH 20/21] Fix type incompatibility with EQL search strategy

The actual ES client/EQL endpoint allows for specifying options at both
the top level _and_ within the request's `body` parameter. However, we
had only typed the latter in our original implementation.

This aligns the types with the actual (demonstrable, as evidenced by all
the @ts-expect-errors) behavior of the server, using a
union type cribbed from responseops (see #171348).

Consequently, all these `@ts-expect-error`s can now be removed.
---
 .../data/common/search/strategies/eql_search/types.ts        | 5 +++--
 .../search/strategies/eql_search/eql_search_strategy.test.ts | 4 ----
 .../plugins/security_solution/public/common/hooks/eql/api.ts | 1 -
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/plugins/data/common/search/strategies/eql_search/types.ts b/src/plugins/data/common/search/strategies/eql_search/types.ts
index cb325e0a4ec33..732eb77a01927 100644
--- a/src/plugins/data/common/search/strategies/eql_search/types.ts
+++ b/src/plugins/data/common/search/strategies/eql_search/types.ts
@@ -6,14 +6,15 @@
  * Side Public License, v 1.
  */
 
-import type { EqlSearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { EqlSearchRequest } from '@elastic/elasticsearch/lib/api/types';
+import type { EqlSearchRequest as EqlSearchRequestWithBody } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { TransportRequestOptions } from '@elastic/elasticsearch';
 
 import { IKibanaSearchRequest, IKibanaSearchResponse } from '../../types';
 
 export const EQL_SEARCH_STRATEGY = 'eql';
 
-export type EqlRequestParams = EqlSearchRequest;
+export type EqlRequestParams = EqlSearchRequest | EqlSearchRequestWithBody;
 
 export interface EqlSearchStrategyRequest extends IKibanaSearchRequest<EqlRequestParams> {
   /**
diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
index 150c77ebc7ca5..d78f152a31cc2 100644
--- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
+++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.test.ts
@@ -190,7 +190,6 @@ describe('EQL search strategy', () => {
               options,
               params: {
                 ...params,
-                // @ts-expect-error not allowed at top level when using `typesWithBodyKey`
                 wait_for_completion_timeout: '5ms',
                 keep_on_completion: false,
               },
@@ -282,7 +281,6 @@ describe('EQL search strategy', () => {
         it('passes along a timestamp_field argument', async () => {
           const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
           const request: EqlSearchStrategyRequest = {
-            // @ts-expect-error timestamp_field not allowed at top level when using `typesWithBodyKey`
             params: { index: 'all', timestamp_field: 'timestamp' },
           };
 
@@ -295,7 +293,6 @@ describe('EQL search strategy', () => {
         it('passes along an event_category_field argument', async () => {
           const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
           const request: EqlSearchStrategyRequest = {
-            // @ts-expect-error timestamp_field not allowed at top level when using `typesWithBodyKey`
             params: { index: 'all', event_category_field: 'event_category' },
           };
 
@@ -310,7 +307,6 @@ describe('EQL search strategy', () => {
         it('passes along a tiebreaker_field argument', async () => {
           const eqlSearch = eqlSearchStrategyProvider(mockSearchConfig, mockLogger);
           const request: EqlSearchStrategyRequest = {
-            // @ts-expect-error tiebreaker_field not allowed at top level when using `typesWithBodyKey`
             params: { index: 'all', tiebreaker_field: 'event_category' },
           };
 
diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
index 6a911c522d884..46d01bf15d577 100644
--- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
+++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts
@@ -41,7 +41,6 @@ export const validateEql = async ({
         params: {
           index: dataViewTitle,
           body: { query, runtime_mappings: runtimeMappings, size: 0 },
-          // @ts-expect-error top-level keys are unexpected in {@link EqlSearchRequest}
           timestamp_field: options?.timestampField,
           tiebreaker_field: options?.tiebreakerField || undefined,
           event_category_field: options?.eventCategoryField,

From f3e55d52877141c86bad0399eb6c1a236826a2a5 Mon Sep 17 00:00:00 2001
From: Ryland Herrick <ryalnd@gmail.com>
Date: Mon, 18 Mar 2024 17:32:18 -0500
Subject: [PATCH 21/21] Fix test descriptions to align with test bodies

A few of these were updated in the course of development without the
title being updated. This clarifies the behavior in the case of data
with `@timestamp`, as well as replacing the vague "behaves as expected"
descriptions with more helpful ones.
---
 .../execution_logic/eql.ts                    | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
index c8f5a95dc30e3..19b3943fd33b2 100644
--- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
+++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/eql.ts
@@ -748,7 +748,10 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('using data with a @timestamp field', () => {
-      it('specifying only timestamp_field results in a warning, and no alerts are generated', async () => {
+      const expectedWarning =
+        'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.';
+
+      it('specifying only timestamp_field results in alert creation with an expected warning', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
           timestamp_field: 'event.created',
@@ -760,15 +763,13 @@ export default ({ getService }: FtrProviderContext) => {
         } = await previewRule({ supertest, rule });
 
         expect(_log.errors).to.be.empty();
-        expect(_log.warnings).to.eql([
-          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
-        ]);
+        expect(_log.warnings).to.eql([expectedWarning]);
 
         const previewAlerts = await getPreviewAlerts({ es, previewId });
         expect(previewAlerts.length).to.be.greaterThan(0);
       });
 
-      it('specifying only timestamp_override results in an error, and no alerts are generated', async () => {
+      it('specifying only timestamp_override results in alert creation with an expected warning', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
           timestamp_override: 'event.created',
@@ -780,15 +781,13 @@ export default ({ getService }: FtrProviderContext) => {
         } = await previewRule({ supertest, rule });
 
         expect(_log.errors).to.be.empty();
-        expect(_log.warnings).to.eql([
-          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
-        ]);
+        expect(_log.warnings).to.eql([expectedWarning]);
 
         const previewAlerts = await getPreviewAlerts({ es, previewId });
         expect(previewAlerts.length).to.be.greaterThan(0);
       });
 
-      it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
+      it('specifying both timestamp_override and timestamp_field results in alert creation with an expected warning', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
           timestamp_field: 'event.created',
@@ -801,9 +800,7 @@ export default ({ getService }: FtrProviderContext) => {
         } = await previewRule({ supertest, rule });
 
         expect(_log.errors).to.be.empty();
-        expect(_log.warnings).to.eql([
-          'This rule reached the maximum alert limit for the rule execution. Some alerts were not created.',
-        ]);
+        expect(_log.warnings).to.eql([expectedWarning]);
 
         const previewAlerts = await getPreviewAlerts({ es, previewId });
         expect(previewAlerts.length).to.be.greaterThan(0);
@@ -862,7 +859,7 @@ export default ({ getService }: FtrProviderContext) => {
         expect(previewAlerts).to.be.empty();
       });
 
-      it('specifying both timestamp_override and timestamp_field behaves as expected', async () => {
+      it('specifying both timestamp_override and timestamp_field results in alert creation with no warnings or errors', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['no_at_timestamp_field']),
           timestamp_field: 'event.ingested',