From e9f7d68a07f1a97e608914d3c69a4fb600abb809 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 16 Sep 2024 16:58:13 +0200 Subject: [PATCH] [Observability] Create observability-specific setting for excluding data tiers from queries (#192570) part of [#190559](https://github.com/elastic/kibana/issues/190559) ## Summary This PR introduces a new `Advanced Settings` under `Observabilty` to provide a way of configuring the exclusion of indices in the `data_cold` and/or `data_frozen` tiers from queries. The change will help to address issues encountered in O11y, most specifically in APM, and could also affect Infra and other features, with unbounded queries targeting the frozen tier. ### For reviewers This PR replaces https://github.com/elastic/kibana/pull/192276 --------- Co-authored-by: Elastic Machine (cherry picked from commit f029f8086a6731b5f435775c915d46e110a34658) --- .../settings/setting_ids/index.ts | 1 + .../settings/observability_project/index.ts | 1 + .../server/collectors/management/schema.ts | 7 +++++++ .../server/collectors/management/types.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 9 +++++++++ .../observability/common/ui_settings_keys.ts | 1 + .../observability/server/ui_settings.ts | 19 +++++++++++++++++++ 7 files changed, 39 insertions(+) diff --git a/packages/kbn-management/settings/setting_ids/index.ts b/packages/kbn-management/settings/setting_ids/index.ts index 08ce7f3579229..0f79a5fff0506 100644 --- a/packages/kbn-management/settings/setting_ids/index.ts +++ b/packages/kbn-management/settings/setting_ids/index.ts @@ -150,6 +150,7 @@ export const OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING = 'observability:aiAssistantSimulatedFunctionCalling'; export const OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN = 'observability:aiAssistantSearchConnectorIndexPattern'; +export const OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS = 'observability:searchExcludedDataTiers'; // Reporting settings export const XPACK_REPORTING_CUSTOM_PDF_LOGO_ID = 'xpackReporting:customPdfLogo'; diff --git a/packages/serverless/settings/observability_project/index.ts b/packages/serverless/settings/observability_project/index.ts index 470964954d166..85f6327bf0a07 100644 --- a/packages/serverless/settings/observability_project/index.ts +++ b/packages/serverless/settings/observability_project/index.ts @@ -37,4 +37,5 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [ settings.OBSERVABILITY_AI_ASSISTANT_LOGS_INDEX_PATTERN_ID, settings.OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING, settings.OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN, + settings.OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS, ]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index d1ab81f3e60a7..52c0df738246a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -694,4 +694,11 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, }, + 'observability:searchExcludedDataTiers': { + type: 'array', + items: { + type: 'keyword', + _meta: { description: 'Non-default value of setting.' }, + }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index c66f4f07a296e..0a0ebe8ebbac6 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -181,4 +181,5 @@ export interface UsageStats { 'aiAssistant:preferredAIAssistantType': string; 'observability:profilingFetchTopNFunctionsFromStacktraces': boolean; 'securitySolution:excludedDataTiersForRuleExecution': string[]; + 'observability:searchExcludedDataTiers': string[]; } diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 70fbeec73bc5d..77e050334803b 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -10355,6 +10355,15 @@ } } }, + "observability:searchExcludedDataTiers": { + "type": "array", + "items": { + "type": "keyword", + "_meta": { + "description": "Non-default value of setting." + } + } + }, "banners:placement": { "type": "keyword", "_meta": { diff --git a/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts b/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts index fe43cd30705db..efceaca9a0427 100644 --- a/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts +++ b/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts @@ -48,3 +48,4 @@ export const profilingAzureCostDiscountRate = 'observability:profilingAzureCostD export const apmEnableTransactionProfiling = 'observability:apmEnableTransactionProfiling'; export const profilingFetchTopNFunctionsFromStacktraces = 'observability:profilingFetchTopNFunctionsFromStacktraces'; +export const searchExcludedDataTiers = 'observability:searchExcludedDataTiers'; diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts index a19f36e5fb93a..ad7afb004775a 100644 --- a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @@ -46,6 +46,7 @@ import { apmEnableServiceInventoryTableSearchBar, profilingFetchTopNFunctionsFromStacktraces, enableInfrastructureContainerAssetView, + searchExcludedDataTiers, } from '../common/ui_settings_keys'; const betaLabel = i18n.translate('xpack.observability.uiSettings.betaLabel', { @@ -640,6 +641,24 @@ export const uiSettings: Record = { schema: schema.boolean(), requiresPageReload: false, }, + [searchExcludedDataTiers]: { + category: [observabilityFeatureId], + name: i18n.translate('xpack.observability.searchExcludedDataTiers', { + defaultMessage: 'Excluded data tiers from search', + }), + description: i18n.translate( + 'xpack.observability.advancedSettings.searchExcludedDataTiersDesc', + { + defaultMessage: `Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. + When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`, + } + ), + value: [], + schema: schema.arrayOf( + schema.oneOf([schema.literal('data_cold'), schema.literal('data_frozen')]) + ), + requiresPageReload: false, + }, }; function throttlingDocsLink({ href }: { href: string }) {