[Observability] Create observability-specific setting for excluding d…

…ata tiers from queries (#192570) part of [#190559](#190559) ## Summary This PR introduces a new `Advanced Settings` under `Observabilty` to provide a way of configuring the exclusion of indices in the `data_cold` and/or `data_frozen` tiers from queries. The change will help to address issues encountered in O11y, most specifically in APM, and could also affect Infra and other features, with unbounded queries targeting the frozen tier. ### For reviewers This PR replaces #192276 --------- Co-authored-by: Elastic Machine <[email protected]>
elastic · Sep 16, 2024 · f029f80 · f029f80
1 parent 6689169
commit f029f80
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 0 deletions.
diff --git a/packages/kbn-management/settings/setting_ids/index.ts b/packages/kbn-management/settings/setting_ids/index.ts
@@ -150,6 +150,7 @@ export const OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING =
   'observability:aiAssistantSimulatedFunctionCalling';
 export const OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN =
   'observability:aiAssistantSearchConnectorIndexPattern';
+export const OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS = 'observability:searchExcludedDataTiers';
 
 // Reporting settings
 export const XPACK_REPORTING_CUSTOM_PDF_LOGO_ID = 'xpackReporting:customPdfLogo';

diff --git a/packages/serverless/settings/observability_project/index.ts b/packages/serverless/settings/observability_project/index.ts
@@ -37,4 +37,5 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [
   settings.OBSERVABILITY_AI_ASSISTANT_LOGS_INDEX_PATTERN_ID,
   settings.OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING,
   settings.OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN,
+  settings.OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS,
 ];
diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts
@@ -694,4 +694,11 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'observability:searchExcludedDataTiers': {
+    type: 'array',
+    items: {
+      type: 'keyword',
+      _meta: { description: 'Non-default value of setting.' },
+    },
+  },
 };
diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts
@@ -181,4 +181,5 @@ export interface UsageStats {
   'aiAssistant:preferredAIAssistantType': string;
   'observability:profilingFetchTopNFunctionsFromStacktraces': boolean;
   'securitySolution:excludedDataTiersForRuleExecution': string[];
+  'observability:searchExcludedDataTiers': string[];
 }
diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
@@ -10355,6 +10355,15 @@
             }
           }
         },
+        "observability:searchExcludedDataTiers": {
+          "type": "array",
+          "items": {
+            "type": "keyword",
+            "_meta": {
+              "description": "Non-default value of setting."
+            }
+          }
+        },
         "banners:placement": {
           "type": "keyword",
           "_meta": {

diff --git a/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts b/x-pack/plugins/observability_solution/observability/common/ui_settings_keys.ts
@@ -48,3 +48,4 @@ export const profilingAzureCostDiscountRate = 'observability:profilingAzureCostD
 export const apmEnableTransactionProfiling = 'observability:apmEnableTransactionProfiling';
 export const profilingFetchTopNFunctionsFromStacktraces =
   'observability:profilingFetchTopNFunctionsFromStacktraces';
+export const searchExcludedDataTiers = 'observability:searchExcludedDataTiers';
diff --git a/x-pack/plugins/observability_solution/observability/server/ui_settings.ts b/x-pack/plugins/observability_solution/observability/server/ui_settings.ts
@@ -46,6 +46,7 @@ import {
   apmEnableServiceInventoryTableSearchBar,
   profilingFetchTopNFunctionsFromStacktraces,
   enableInfrastructureContainerAssetView,
+  searchExcludedDataTiers,
 } from '../common/ui_settings_keys';
 
 const betaLabel = i18n.translate('xpack.observability.uiSettings.betaLabel', {
@@ -640,6 +641,24 @@ export const uiSettings: Record<string, UiSettings> = {
     schema: schema.boolean(),
     requiresPageReload: false,
   },
+  [searchExcludedDataTiers]: {
+    category: [observabilityFeatureId],
+    name: i18n.translate('xpack.observability.searchExcludedDataTiers', {
+      defaultMessage: 'Excluded data tiers from search',
+    }),
+    description: i18n.translate(
+      'xpack.observability.advancedSettings.searchExcludedDataTiersDesc',
+      {
+        defaultMessage: `Specify the data tiers to exclude from search, such as data_cold and/or data_frozen. 
+        When configured, indices allocated in the selected tiers will be ignored from search requests. Affected apps: APM`,
+      }
+    ),
+    value: [],
+    schema: schema.arrayOf(
+      schema.oneOf([schema.literal('data_cold'), schema.literal('data_frozen')])
+    ),
+    requiresPageReload: false,
+  },
 };
 
 function throttlingDocsLink({ href }: { href: string }) {