[data views] Provide method of excluding data tiers when getting fiel…

…d list (#167946)

## Summary

This PR implements an advanced setting that allows the exclusion of
listed data tiers when getting a field list. The expected common use
case would be excluding frozen indices to speed up slow field caps
calls.

There is no serverless functionality since serverless doesn't have data
tiers.

src/plugins/data_views/common/constants.ts

@@ -69,3 +69,5 @@ export const FIELDS_PATH = '/internal/data_views/fields';
  * @public
  */
 export const EXISTING_INDICES_PATH = '/internal/data_views/_existing_indices';
+
+export const DATA_VIEWS_FIELDS_EXCLUDED_TIERS = 'data_views:fields_excluded_data_tiers';

src/plugins/data_views/common/types.ts

@@ -544,5 +544,6 @@ export interface HasDataService {
 
 export interface ClientConfigType {
   scriptedFieldsEnabled?: boolean;
+  dataTiersExcludedForFields?: string;
   fieldListCachingEnabled?: boolean;
 }

src/plugins/data_views/server/index.ts

@@ -46,6 +46,13 @@ const configSchema = schema.object({
     schema.boolean({ defaultValue: false }),
     schema.never()
   ),
+
+  dataTiersExcludedForFields: schema.conditional(
+    schema.contextRef('serverless'),
+    true,
+    schema.never(),
+    schema.boolean({ defaultValue: true })
+  ),
   fieldListCachingEnabled: schema.conditional(
     schema.contextRef('serverless'),
     true,

src/plugins/data_views/server/plugin.ts

@@ -16,6 +16,7 @@ import { registerIndexPatternsUsageCollector } from './register_index_pattern_us
 import { createScriptedFieldsDeprecationsConfig } from './deprecations';
 import { DATA_VIEW_SAVED_OBJECT_TYPE, LATEST_VERSION } from '../common';
 import type { ClientConfigType } from '../common/types';
+import { dataTiersUiSettingsConfig } from './ui_settings';
 import {
   DataViewsServerPluginSetup,
   DataViewsServerPluginStart,
@@ -50,6 +51,9 @@ export class DataViewsServerPlugin
 
     const config = this.initializerContext.config.get<ClientConfigType>();
 
+    if (config.dataTiersExcludedForFields) {
+      core.uiSettings.register(dataTiersUiSettingsConfig);
+    }
     if (config.fieldListCachingEnabled) {
       core.uiSettings.register(cacheMaxAge);
     }

src/plugins/data_views/server/rest_api_routes/internal/fields.ts

@@ -15,9 +15,10 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_PATH as path } from '../../../common/constants';
+import { FIELDS_PATH as path, DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../../../common/constants';
 import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants';
+import { getIndexFilterDsl } from './utils';
 
 export function calculateHash(srcBuffer: Buffer) {
   const hash = createHash('sha1');
@@ -31,6 +32,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
     const uiSettings = core.uiSettings.client;
     const { asCurrentUser } = core.elasticsearch.client;
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const {
       pattern,
       meta_fields: metaFields,
@@ -59,6 +63,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
+        indexFilter: getIndexFilterDsl({ excludedTiers }),
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -17,7 +17,11 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_FOR_WILDCARD_PATH as path } from '../../../common/constants';
+import {
+  FIELDS_FOR_WILDCARD_PATH as path,
+  DATA_VIEWS_FIELDS_EXCLUDED_TIERS,
+} from '../../../common/constants';
+import { getIndexFilterDsl } from './utils';
 
 /**
  * Accepts one of the following:
@@ -115,8 +119,13 @@ export const validate: FullValidationConfig<any, any, any> = {
 
 const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> =
   (isRollupsEnabled) => async (context, request, response) => {
-    const { asCurrentUser } = (await context.core).elasticsearch.client;
+    const core = await context.core;
+    const { asCurrentUser } = core.elasticsearch.client;
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+
     const {
       pattern,
       meta_fields: metaFields,
@@ -149,7 +158,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
-        indexFilter,
+        indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }),
         allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });

src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getIndexFilterDsl } from './utils';
+
+describe('getIndexFilterDsl', () => {
+  const indexFilter = { term: { _id: '1' } };
+  const tiersQuery = {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier: ['data_cold', 'data_frozen'],
+          },
+        },
+      ],
+    },
+  };
+  const excludedTiers = 'data_cold, data_frozen';
+
+  it('no indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({})).toBeUndefined();
+  });
+
+  it('indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({ indexFilter })).toEqual(indexFilter);
+  });
+
+  it('excluded tiers, no indexFilter', () => {
+    expect(getIndexFilterDsl({ excludedTiers })).toEqual(tiersQuery);
+  });
+
+  it('indexFilter and excluded tiers', () => {
+    const combinedQuery = {
+      bool: {
+        must: [indexFilter, tiersQuery],
+      },
+    };
+
+    expect(getIndexFilterDsl({ indexFilter, excludedTiers })).toEqual(combinedQuery);
+  });
+});

src/plugins/data_views/server/rest_api_routes/internal/utils.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { QueryDslQueryContainer } from '../../../common/types';
+
+const excludeTiersDsl = (excludedTiers: string) => {
+  const _tier = excludedTiers.split(',').map((tier) => tier.trim());
+  return {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier,
+          },
+        },
+      ],
+    },
+  };
+};
+
+interface GetIndexFilterDslOptions {
+  indexFilter?: QueryDslQueryContainer;
+  excludedTiers?: string;
+}
+
+export const getIndexFilterDsl = ({
+  indexFilter,
+  excludedTiers,
+}: GetIndexFilterDslOptions): QueryDslQueryContainer | undefined => {
+  if (!indexFilter) {
+    return excludedTiers ? excludeTiersDsl(excludedTiers) : undefined;
+  }
+
+  return !excludedTiers
+    ? indexFilter
+    : {
+        bool: {
+          must: [indexFilter, excludeTiersDsl(excludedTiers)],
+        },
+      };
+};

src/plugins/data_views/server/ui_settings.ts

@@ -8,8 +8,25 @@
 
 import { i18n } from '@kbn/i18n';
 import { schema } from '@kbn/config-schema';
+import type { UiSettingsParams } from '@kbn/core/server';
+import { DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../common/constants';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants';
 
+export const dataTiersUiSettingsConfig: Record<string, UiSettingsParams> = {
+  [DATA_VIEWS_FIELDS_EXCLUDED_TIERS]: {
+    name: i18n.translate('dataViews.advancedSettings.dataTiersName', {
+      defaultMessage: 'Data tiers excluded from field requests',
+    }),
+    value: '',
+    type: 'string',
+    description: i18n.translate('dataViews.advancedSettings.dataTiersText', {
+      defaultMessage:
+        'Exclude fields from specified tiers (such as data_frozen) for faster performance. Comma delimit to exclude multiple tiers - data_warm,data_cold',
+    }),
+    schema: schema.string(),
+  },
+};
+
 export const cacheMaxAge = {
   'data_views:cache_max_age': {
     name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', {

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -620,4 +620,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'integer',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'data_views:fields_excluded_data_tiers': {
+    type: 'keyword',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -164,4 +164,5 @@ export interface UsageStats {
   'observability:profilingDatacenterPUE': number;
   'observability:profilingCostPervCPUPerHour': number;
   'observability:profilingAWSCostDiscountRate': number;
+  'data_views:fields_excluded_data_tiers': string;
 }

src/plugins/telemetry/schema/oss_plugins.json

@@ -10245,6 +10245,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "data_views:fields_excluded_data_tiers": {
+          "type": "keyword",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },