From d48453956b5e584f2af5c1fd6393417b60cba396 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:28:14 +0100 Subject: [PATCH] [Security Solution][Detection Engine] fixes ES|QL data tier filter from adv settings (#196390) ## Summary - fixes absent data tier filter for ES|QL rule - followup to https://github.com/elastic/kibana/pull/186908 ### Demo https://github.com/user-attachments/assets/a6f1290f-ea77-43bf-8def-42712ca5d1b0 ### How to test Create a deployment with cold and frozen data tiers and use following commands to create index and ILM
Data tiers commands ```JSON PUT /_cluster/settings { "persistent": { "indices.lifecycle.poll_interval": "1m" } } PUT /_ilm/policy/filtering_data_tiers { "policy": { "phases": { "frozen": { "min_age": "10m", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots", "force_merge_index": true } } }, "cold": { "min_age": "1m", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots", "force_merge_index": true }, "set_priority": { "priority": 0 } } }, "hot": { "min_age": "0ms", "actions": { "set_priority": { "priority": 100 } } } } } } PUT /_index_template/filtering_data_tiers_template { "index_patterns": [ "filtering_data_tiers*" ], "template": { "settings": { "index.lifecycle.name": "filtering_data_tiers", "index.lifecycle.rollover_alias": "test-filtering_data_tiers" }, "mappings": { "_meta": { "version": "1.6.0" }, "properties": { "@timestamp": { "type": "date" }, "host": { "properties": { "name": { "type": "keyword", "ignore_above": 1024 } } } } } } } PUT /filtering_data_tiers-000001 { "aliases": { "filtering_data_tiers": { "is_write_index": true } } } POST filtering_data_tiers/_doc { "@timestamp": "2024-07-08T17:00:01.000Z", "host.name": "test-0" } ```
(cherry picked from commit c79f0ae78633c81beebd3f95735326cc872be7f6) --- .../detection_engine/rule_types/esql/esql.ts | 7 +++- .../execution_logic/esql.ts | 37 +++++++++++++++++++ .../detections_response/utils/index.ts | 1 + .../utils/set_advanced_settings.ts | 26 +++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index 173d722d782a1..0adc9c1d77d3d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -30,6 +30,7 @@ import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters'; import type { RulePreviewLoggedRequest } from '../../../../../common/api/detection_engine/rule_preview/rule_preview.gen'; import type { CreateRuleOptions, RunOpts, SignalSource } from '../types'; import { logEsqlRequest } from '../utils/logged_requests'; +import { getDataTierFilter } from '../utils/get_data_tier_filter'; import * as i18n from '../translations'; import { @@ -90,6 +91,10 @@ export const esqlExecutor = async ({ return withSecuritySpan('esqlExecutor', async () => { const result = createSearchAfterReturnType(); let size = tuple.maxSignals; + const dataTiersFilters = await getDataTierFilter({ + uiSettingsClient: services.uiSettingsClient, + }); + try { while ( result.createdSignalsCount <= tuple.maxSignals && @@ -100,7 +105,7 @@ export const esqlExecutor = async ({ from: tuple.from.toISOString(), to: tuple.to.toISOString(), size, - filters: [], + filters: dataTiersFilters, primaryTimestamp, secondaryTimestamp, exceptionFilter, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts index 723a2a7d2dfa3..bf431e0021053 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql.ts @@ -14,6 +14,7 @@ import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/comm import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; import { getMaxSignalsWarning as getMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; +import { EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION } from '@kbn/security-solution-plugin/common/constants'; import { getPreviewAlerts, previewRule, @@ -25,6 +26,7 @@ import { scheduleRuleRun, stopAllManualRuns, waitForBackfillExecuted, + setAdvancedSettings, } from '../../../../utils'; import { deleteAllRules, @@ -1428,6 +1430,12 @@ export default ({ getService }: FtrProviderContext) => { await indexEnhancedDocuments({ documents: [doc1], interval, id }); }); + afterEach(async () => { + await setAdvancedSettings(supertest, { + [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: [], + }); + }); + it('should not return requests property when not enabled', async () => { const { logs } = await previewRule({ supertest, @@ -1463,6 +1471,35 @@ export default ({ getService }: FtrProviderContext) => { 'POST /ecs_compliant/_search?ignore_unavailable=true' ); }); + it('should not return requests with any data tier filter', async () => { + const { logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests![0].request).not.toContain('data_frozen'); + }); + it('should return requests with included data tiers filters from advanced settings', async () => { + await setAdvancedSettings(supertest, { + [EXCLUDED_DATA_TIERS_FOR_RULE_EXECUTION]: ['data_frozen'], + }); + const { logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + enableLoggedRequests: true, + }); + + const requests = logs[0].requests; + + expect(requests![0].request).toMatch( + /"must_not":\s*\[\s*{\s*"terms":\s*{\s*"_tier":\s*\[\s*"data_frozen"\s*\]/ + ); + }); }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts index 5667762ce95c4..2c12400b7f169 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts @@ -24,3 +24,4 @@ export * from './get_stats'; export * from './get_detection_metrics_from_body'; export * from './get_stats_url'; export * from './combine_to_ndjson'; +export * from './set_advanced_settings'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts new file mode 100644 index 0000000000000..98fe191096253 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/set_advanced_settings.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ELASTIC_HTTP_VERSION_HEADER, + X_ELASTIC_INTERNAL_ORIGIN_REQUEST, +} from '@kbn/core-http-common'; + +import type SuperTest from 'supertest'; + +export const setAdvancedSettings = async ( + supertest: SuperTest.Agent, + settings: Record +) => { + return supertest + .post('/internal/kibana/settings') + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send({ changes: settings }) + .expect(200); +};