From ae52d87d3ee4d728a31a3c3446180f5d3aee3264 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Wed, 31 Jan 2024 06:50:54 -0600 Subject: [PATCH] [data views] Provide method of excluding data tiers when getting field list (#167946) ## Summary This PR implements an advanced setting that allows the exclusion of listed data tiers when getting a field list. The expected common use case would be excluding frozen indices to speed up slow field caps calls. There is no serverless functionality since serverless doesn't have data tiers. --- src/plugins/data_views/common/constants.ts | 2 + src/plugins/data_views/common/types.ts | 1 + src/plugins/data_views/server/index.ts | 7 +++ src/plugins/data_views/server/plugin.ts | 4 ++ .../server/rest_api_routes/internal/fields.ts | 7 ++- .../rest_api_routes/internal/fields_for.ts | 15 ++++-- .../rest_api_routes/internal/utils.test.ts | 47 +++++++++++++++++++ .../server/rest_api_routes/internal/utils.ts | 46 ++++++++++++++++++ src/plugins/data_views/server/ui_settings.ts | 17 +++++++ .../server/collectors/management/schema.ts | 4 ++ .../server/collectors/management/types.ts | 1 + src/plugins/telemetry/schema/oss_plugins.json | 6 +++ 12 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts create mode 100644 src/plugins/data_views/server/rest_api_routes/internal/utils.ts diff --git a/src/plugins/data_views/common/constants.ts b/src/plugins/data_views/common/constants.ts index 43dc67362bf26..243170592c62f 100644 --- a/src/plugins/data_views/common/constants.ts +++ b/src/plugins/data_views/common/constants.ts @@ -69,3 +69,5 @@ export const FIELDS_PATH = '/internal/data_views/fields'; * @public */ export const EXISTING_INDICES_PATH = '/internal/data_views/_existing_indices'; + +export const DATA_VIEWS_FIELDS_EXCLUDED_TIERS = 'data_views:fields_excluded_data_tiers'; diff --git a/src/plugins/data_views/common/types.ts b/src/plugins/data_views/common/types.ts index 7fb7ddedbccd4..6ba4b6819a045 100644 --- a/src/plugins/data_views/common/types.ts +++ b/src/plugins/data_views/common/types.ts @@ -544,5 +544,6 @@ export interface HasDataService { export interface ClientConfigType { scriptedFieldsEnabled?: boolean; + dataTiersExcludedForFields?: string; fieldListCachingEnabled?: boolean; } diff --git a/src/plugins/data_views/server/index.ts b/src/plugins/data_views/server/index.ts index d8382046183a3..2455ee0725e37 100644 --- a/src/plugins/data_views/server/index.ts +++ b/src/plugins/data_views/server/index.ts @@ -46,6 +46,13 @@ const configSchema = schema.object({ schema.boolean({ defaultValue: false }), schema.never() ), + + dataTiersExcludedForFields: schema.conditional( + schema.contextRef('serverless'), + true, + schema.never(), + schema.boolean({ defaultValue: true }) + ), fieldListCachingEnabled: schema.conditional( schema.contextRef('serverless'), true, diff --git a/src/plugins/data_views/server/plugin.ts b/src/plugins/data_views/server/plugin.ts index 2da7d128b837a..20d72a0edcfec 100644 --- a/src/plugins/data_views/server/plugin.ts +++ b/src/plugins/data_views/server/plugin.ts @@ -16,6 +16,7 @@ import { registerIndexPatternsUsageCollector } from './register_index_pattern_us import { createScriptedFieldsDeprecationsConfig } from './deprecations'; import { DATA_VIEW_SAVED_OBJECT_TYPE, LATEST_VERSION } from '../common'; import type { ClientConfigType } from '../common/types'; +import { dataTiersUiSettingsConfig } from './ui_settings'; import { DataViewsServerPluginSetup, DataViewsServerPluginStart, @@ -50,6 +51,9 @@ export class DataViewsServerPlugin const config = this.initializerContext.config.get(); + if (config.dataTiersExcludedForFields) { + core.uiSettings.register(dataTiersUiSettingsConfig); + } if (config.fieldListCachingEnabled) { core.uiSettings.register(cacheMaxAge); } diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts index cee8efc8b6d1d..37ef1eb5e14c7 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields.ts @@ -15,9 +15,10 @@ import type { DataViewsServerPluginStartDependencies, } from '../../types'; import type { FieldDescriptorRestResponse } from '../route_types'; -import { FIELDS_PATH as path } from '../../../common/constants'; +import { FIELDS_PATH as path, DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../../../common/constants'; import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for'; import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants'; +import { getIndexFilterDsl } from './utils'; export function calculateHash(srcBuffer: Buffer) { const hash = createHash('sha1'); @@ -31,6 +32,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I const uiSettings = core.uiSettings.client; const { asCurrentUser } = core.elasticsearch.client; const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); + const excludedTiers = await core.uiSettings.client.get( + DATA_VIEWS_FIELDS_EXCLUDED_TIERS + ); const { pattern, meta_fields: metaFields, @@ -59,6 +63,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I allow_no_indices: allowNoIndex || false, includeUnmapped, }, + indexFilter: getIndexFilterDsl({ excludedTiers }), ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), }); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts index 3c89cdef17955..98b4086bf161e 100644 --- a/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts +++ b/src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts @@ -17,7 +17,11 @@ import type { DataViewsServerPluginStartDependencies, } from '../../types'; import type { FieldDescriptorRestResponse } from '../route_types'; -import { FIELDS_FOR_WILDCARD_PATH as path } from '../../../common/constants'; +import { + FIELDS_FOR_WILDCARD_PATH as path, + DATA_VIEWS_FIELDS_EXCLUDED_TIERS, +} from '../../../common/constants'; +import { getIndexFilterDsl } from './utils'; /** * Accepts one of the following: @@ -115,8 +119,13 @@ export const validate: FullValidationConfig = { const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> = (isRollupsEnabled) => async (context, request, response) => { - const { asCurrentUser } = (await context.core).elasticsearch.client; + const core = await context.core; + const { asCurrentUser } = core.elasticsearch.client; + const excludedTiers = await core.uiSettings.client.get( + DATA_VIEWS_FIELDS_EXCLUDED_TIERS + ); const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled()); + const { pattern, meta_fields: metaFields, @@ -149,7 +158,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I allow_no_indices: allowNoIndex || false, includeUnmapped, }, - indexFilter, + indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }), allowHidden, ...(parsedFields.length > 0 ? { fields: parsedFields } : {}), }); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts b/src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts new file mode 100644 index 0000000000000..0a2b45f29fd12 --- /dev/null +++ b/src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getIndexFilterDsl } from './utils'; + +describe('getIndexFilterDsl', () => { + const indexFilter = { term: { _id: '1' } }; + const tiersQuery = { + bool: { + must_not: [ + { + terms: { + _tier: ['data_cold', 'data_frozen'], + }, + }, + ], + }, + }; + const excludedTiers = 'data_cold, data_frozen'; + + it('no indexFilter, no excluded tiers', () => { + expect(getIndexFilterDsl({})).toBeUndefined(); + }); + + it('indexFilter, no excluded tiers', () => { + expect(getIndexFilterDsl({ indexFilter })).toEqual(indexFilter); + }); + + it('excluded tiers, no indexFilter', () => { + expect(getIndexFilterDsl({ excludedTiers })).toEqual(tiersQuery); + }); + + it('indexFilter and excluded tiers', () => { + const combinedQuery = { + bool: { + must: [indexFilter, tiersQuery], + }, + }; + + expect(getIndexFilterDsl({ indexFilter, excludedTiers })).toEqual(combinedQuery); + }); +}); diff --git a/src/plugins/data_views/server/rest_api_routes/internal/utils.ts b/src/plugins/data_views/server/rest_api_routes/internal/utils.ts new file mode 100644 index 0000000000000..ab3845fb2ec9e --- /dev/null +++ b/src/plugins/data_views/server/rest_api_routes/internal/utils.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { QueryDslQueryContainer } from '../../../common/types'; + +const excludeTiersDsl = (excludedTiers: string) => { + const _tier = excludedTiers.split(',').map((tier) => tier.trim()); + return { + bool: { + must_not: [ + { + terms: { + _tier, + }, + }, + ], + }, + }; +}; + +interface GetIndexFilterDslOptions { + indexFilter?: QueryDslQueryContainer; + excludedTiers?: string; +} + +export const getIndexFilterDsl = ({ + indexFilter, + excludedTiers, +}: GetIndexFilterDslOptions): QueryDslQueryContainer | undefined => { + if (!indexFilter) { + return excludedTiers ? excludeTiersDsl(excludedTiers) : undefined; + } + + return !excludedTiers + ? indexFilter + : { + bool: { + must: [indexFilter, excludeTiersDsl(excludedTiers)], + }, + }; +}; diff --git a/src/plugins/data_views/server/ui_settings.ts b/src/plugins/data_views/server/ui_settings.ts index 3ba489a4b6b28..3ea93470418e1 100644 --- a/src/plugins/data_views/server/ui_settings.ts +++ b/src/plugins/data_views/server/ui_settings.ts @@ -8,8 +8,25 @@ import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; +import type { UiSettingsParams } from '@kbn/core/server'; +import { DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../common/constants'; import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants'; +export const dataTiersUiSettingsConfig: Record = { + [DATA_VIEWS_FIELDS_EXCLUDED_TIERS]: { + name: i18n.translate('dataViews.advancedSettings.dataTiersName', { + defaultMessage: 'Data tiers excluded from field requests', + }), + value: '', + type: 'string', + description: i18n.translate('dataViews.advancedSettings.dataTiersText', { + defaultMessage: + 'Exclude fields from specified tiers (such as data_frozen) for faster performance. Comma delimit to exclude multiple tiers - data_warm,data_cold', + }), + schema: schema.string(), + }, +}; + export const cacheMaxAge = { 'data_views:cache_max_age': { name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', { diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 32b133f210dad..a527feefa754b 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -620,4 +620,8 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'integer', _meta: { description: 'Non-default value of setting.' }, }, + 'data_views:fields_excluded_data_tiers': { + type: 'keyword', + _meta: { description: 'Non-default value of setting.' }, + }, }; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 0832675cd54cf..6c5945a9f6805 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -164,4 +164,5 @@ export interface UsageStats { 'observability:profilingDatacenterPUE': number; 'observability:profilingCostPervCPUPerHour': number; 'observability:profilingAWSCostDiscountRate': number; + 'data_views:fields_excluded_data_tiers': string; } diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 7686be9584e4e..23239ab0b0ca8 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -10245,6 +10245,12 @@ "_meta": { "description": "Non-default value of setting." } + }, + "data_views:fields_excluded_data_tiers": { + "type": "keyword", + "_meta": { + "description": "Non-default value of setting." + } } } },