diff --git a/x-pack/plugins/index_management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx b/x-pack/plugins/index_management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx index 9332509dc26b9..a9675b592aa39 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx +++ b/x-pack/plugins/index_management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx @@ -9,7 +9,11 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { setupEnvironment } from '../helpers'; -import { getMatchingIndices, getFieldsFromIndices } from '../helpers/fixtures'; +import { + getMatchingIndices, + getFieldsFromIndices, + getMatchingDataStreams, +} from '../helpers/fixtures'; import { CreateEnrichPoliciesTestBed, setup } from './create_enrich_policy.helpers'; import { getESPolicyCreationApiCall } from '../../../common/lib'; @@ -57,6 +61,7 @@ describe('Create enrich policy', () => { hasAllPrivileges: true, missingPrivileges: { cluster: [] }, }); + httpRequestsMockHelpers.setGetMatchingDataStreams(getMatchingDataStreams()); await act(async () => { testBed = await setup(httpSetup); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts index a70ad0ea552ec..0741b44495600 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts @@ -62,6 +62,9 @@ export const createTestEnrichPolicy = (name: string, type: EnrichPolicyType) => export const getMatchingIndices = () => ({ indices: ['test-1', 'test-2', 'test-3', 'test-4', 'test-5'], }); +export const getMatchingDataStreams = () => ({ + dataStreams: ['test-6', 'test-7', 'test-8', 'test-9', 'test-10'], +}); export const getFieldsFromIndices = () => ({ commonFields: [], diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts index 79daba4c73867..0504f9bf40b7a 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts @@ -172,6 +172,13 @@ const registerHttpRequestMockHelpers = ( response, error ); + const setGetMatchingDataStreams = (response?: HttpResponse, error?: ResponseError) => + mockResponse( + 'POST', + `${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`, + response, + error + ); const setGetFieldsFromIndices = (response?: HttpResponse, error?: ResponseError) => mockResponse( @@ -249,6 +256,7 @@ const registerHttpRequestMockHelpers = ( setGetPrivilegesResponse, setCreateEnrichPolicy, setInferenceModels, + setGetMatchingDataStreams, }; }; diff --git a/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx b/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx index 6e743c6bd0781..175bb812dbd5f 100644 --- a/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx +++ b/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx @@ -94,18 +94,15 @@ export const configurationFormSchema: FormSchema = { }, sourceIndices: { - label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel', { - defaultMessage: 'Source indices', + label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceLabel', { + defaultMessage: 'Source', }), validations: [ { validator: fieldValidators.emptyField( - i18n.translate( - 'xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError', - { - defaultMessage: 'At least one source index is required.', - } - ) + i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceRequiredError', { + defaultMessage: 'At least one source is required.', + }) ), }, ], diff --git a/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx b/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx index 9181449f2dbac..825a977638417 100644 --- a/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx +++ b/x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx @@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n'; import { uniq, isEmpty } from 'lodash'; import { EuiFormRow, EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui'; import type { EuiComboBoxProps } from '@elastic/eui'; -import { getMatchingIndices } from '../../../../services/api'; +import { getMatchingDataStreams, getMatchingIndices } from '../../../../services/api'; import type { FieldHook } from '../../../../../shared_imports'; import { getFieldValidityAndErrorMessage } from '../../../../../shared_imports'; @@ -25,23 +25,76 @@ interface Props { [key: string]: any; } -const getIndexOptions = async (patternString: string) => { - const options: IOption[] = []; +interface GetMatchingOptionsParams { + matches: string[]; + optionsMessage: string; + noMatchingMessage: string; +} +const i18nTexts = { + indices: { + options: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', { + defaultMessage: 'Based on your indices', + }), + noMatches: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', { + defaultMessage: 'No indices match your search criteria.', + }), + }, + dataStreams: { + options: i18n.translate( + 'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.optionsLabel', + { + defaultMessage: 'Based on your data streams', + } + ), + noMatches: i18n.translate( + 'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.noMatchingOption', + { + defaultMessage: 'No data streams match your search criteria.', + } + ), + }, + sourcePlaceholder: i18n.translate( + 'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.placeholder', + { + defaultMessage: 'Select source indices and data streams.', + } + ), +}; + +const getIndexOptions = async (patternString: string) => { if (!patternString) { - return options; + return []; } - const { data } = await getMatchingIndices(patternString); - const matchingIndices = data.indices; + const { data: indicesData } = await getMatchingIndices(patternString); + const { data: dataStreamsData } = await getMatchingDataStreams(patternString); - if (matchingIndices.length) { - const matchingOptions = uniq([...matchingIndices]); + const indices = getMatchingOptions({ + matches: indicesData.indices, + optionsMessage: i18nTexts.indices.options, + noMatchingMessage: i18nTexts.indices.noMatches, + }); + const dataStreams = getMatchingOptions({ + matches: dataStreamsData.dataStreams, + optionsMessage: i18nTexts.dataStreams.options, + noMatchingMessage: i18nTexts.dataStreams.noMatches, + }); + + return [...indices, ...dataStreams]; +}; + +const getMatchingOptions = ({ + matches, + optionsMessage, + noMatchingMessage, +}: GetMatchingOptionsParams) => { + const options: IOption[] = []; + if (matches.length) { + const matchingOptions = uniq([...matches]); options.push({ - label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', { - defaultMessage: 'Based on your indices', - }), + label: optionsMessage, options: matchingOptions .map((match) => { return { @@ -53,13 +106,10 @@ const getIndexOptions = async (patternString: string) => { }); } else { options.push({ - label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', { - defaultMessage: 'No indices match your search criteria.', - }), + label: noMatchingMessage, options: [], }); } - return options; }; @@ -71,7 +121,6 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => { const onSearchChange = useCallback( async (search: string) => { const indexPattern = isEmpty(search) ? '*' : search; - setIsIndiciesLoading(true); setIndexOptions(await getIndexOptions(indexPattern)); setIsIndiciesLoading(false); @@ -98,6 +147,7 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => { > ({ path: `${INTERNAL_API_BASE_PATH}/enrich_policies/get_fields_from_indices`, diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts index fb548b49623ee..98a9c7f34d018 100644 --- a/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts +++ b/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts @@ -366,4 +366,33 @@ describe('Enrich policies API', () => { expect(searchMock).toHaveBeenCalled(); }); }); + + describe('Get matching indices - POST /api/index_management/enrich_policies/get_matching_data_streams', () => { + const getDataStreamsMock = router.getMockESApiFn('indices.getDataStream'); + + it('Return matching data streams', async () => { + const mockRequest: RequestMock = { + method: 'post', + path: addInternalBasePath('/enrich_policies/get_matching_data_streams'), + body: { + pattern: 'test', + }, + }; + + getDataStreamsMock.mockResolvedValue({ + body: {}, + statusCode: 200, + }); + + const res = await router.runRequest(mockRequest); + + expect(res).toEqual({ + body: { + dataStreams: [], + }, + }); + + expect(getDataStreamsMock).toHaveBeenCalledWith({ name: '*test*', expand_wildcards: 'open' }); + }); + }); }); diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts index 6ecc6e9eb7ade..32b7d0b64718c 100644 --- a/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts +++ b/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts @@ -145,3 +145,15 @@ export async function getIndices(dataClient: IScopedClusterClient, pattern: stri return indices.buckets ? indices.buckets.map((bucket) => bucket.key) : []; } +export async function getDataStreams( + dataClient: IScopedClusterClient, + pattern: string, + limit = 10 +) { + const response = await dataClient.asCurrentUser.indices.getDataStream({ + name: pattern, + expand_wildcards: 'open', + }); + + return response.data_streams ? response.data_streams.map((dataStream) => dataStream.name) : []; +} diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts index bb6ef8b1fff50..ff165876a6ee9 100644 --- a/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts @@ -13,7 +13,13 @@ import { RouteDependencies } from '../../../types'; import { addInternalBasePath } from '..'; import { enrichPoliciesActions } from '../../../lib/enrich_policies'; import { serializeAsESPolicy } from '../../../../common/lib'; -import { normalizeFieldsList, getIndices, FieldCapsList, getCommonFields } from './helpers'; +import { + normalizeFieldsList, + getIndices, + FieldCapsList, + getCommonFields, + getDataStreams, +} from './helpers'; const validationSchema = schema.object({ policy: schema.object({ @@ -105,6 +111,33 @@ export function registerCreateRoute({ router, lib: { handleEsError } }: RouteDep } } ); + router.post( + { + path: addInternalBasePath('/enrich_policies/get_matching_data_streams'), + validate: { body: getMatchingIndicesSchema }, + }, + async (context, request, response) => { + let { pattern } = request.body; + const client = (await context.core).elasticsearch.client as IScopedClusterClient; + + // Add wildcards to the search query to match the behavior of the + // index pattern search in the Kibana UI. + if (!pattern.startsWith('*')) { + pattern = `*${pattern}`; + } + if (!pattern.endsWith('*')) { + pattern = `${pattern}*`; + } + + try { + const dataStreams = await getDataStreams(client, pattern); + + return response.ok({ body: { dataStreams } }); + } catch (error) { + return handleEsError({ error, response }); + } + } + ); router.post( { diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 765be17d98b5d..0d443d929fc0a 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -22567,8 +22567,6 @@ "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "Requête (facultative)", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Plage", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} correspond à un nombre, une date ou une plage d'adresses IP.", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "Index source", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "Au moins un index source est requis.", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "Une valeur est requise pour le type de politique.", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "Détermine comment faire correspondre les données avec les documents entrants.", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "Charger un fichier", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 3f1b10bdce8ae..f74354a42c117 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -22538,8 +22538,6 @@ "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "クエリー（任意）", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Range", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type}は、番号、日付、またはIPアドレスの範囲と一致します。", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "ソースインデックス", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "ソースインデックスが最低1つ必要です。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "ポリシータイプ値が必要です。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "どのようにデータを受信ドキュメントに一致させるかを決定します。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "ファイルをアップロード", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 463906b3e6cff..3457f35ee34ce 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -22148,8 +22148,6 @@ "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "查询（可选）", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "范围", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} 匹配一个数字、日期或 IP 地址范围。", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "源索引", - "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "至少需要一个源索引。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "策略类型值必填。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "确定如何将数据匹配到传入文档。", "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "上传文件", diff --git a/x-pack/test/api_integration/apis/management/index_management/create_enrich_policy.ts b/x-pack/test/api_integration/apis/management/index_management/create_enrich_policy.ts index e2bfa9cdb35ef..36f5e17e347d0 100644 --- a/x-pack/test/api_integration/apis/management/index_management/create_enrich_policy.ts +++ b/x-pack/test/api_integration/apis/management/index_management/create_enrich_policy.ts @@ -19,6 +19,9 @@ export default function ({ getService }: FtrProviderContext) { const INDEX_A_NAME = `index-${Math.random()}`; const INDEX_B_NAME = `index-${Math.random()}`; const POLICY_NAME = `policy-${Math.random()}`; + const DATA_STREAM_TEMPLATE = `data-stream-template`; + const DATA_STREAM_A_NAME = `data-stream-${Math.random()}`; + const DATA_STREAM_B_NAME = `data-stream-${Math.random()}`; before(async () => { try { @@ -56,6 +59,24 @@ export default function ({ getService }: FtrProviderContext) { log.debug('[Setup error] Error creating test index'); throw err; } + try { + await es.indices.putIndexTemplate({ + name: DATA_STREAM_TEMPLATE, + body: { + index_patterns: ['data-stream-*'], + data_stream: {}, + }, + }); + await es.indices.createDataStream({ + name: DATA_STREAM_A_NAME, + }); + await es.indices.createDataStream({ + name: DATA_STREAM_B_NAME, + }); + } catch (err) { + log.debug('[Setup error] Error creating test data stream'); + throw err; + } }); after(async () => { @@ -66,6 +87,14 @@ export default function ({ getService }: FtrProviderContext) { log.debug('[Cleanup error] Error deleting test index'); throw err; } + try { + await es.indices.deleteDataStream({ name: DATA_STREAM_A_NAME }); + await es.indices.deleteDataStream({ name: DATA_STREAM_B_NAME }); + await es.indices.deleteIndexTemplate({ name: DATA_STREAM_TEMPLATE }); + } catch (err) { + log.debug('[Cleanup error] Error deleting test data stream'); + throw err; + } }); it('Allows to create an enrich policy', async () => { @@ -92,11 +121,14 @@ export default function ({ getService }: FtrProviderContext) { .post(`${INTERNAL_API_BASE_PATH}/enrich_policies/get_fields_from_indices`) .set('kbn-xsrf', 'xxx') .set('x-elastic-internal-origin', 'xxx') - .send({ indices: [INDEX_A_NAME, INDEX_B_NAME] }) + .send({ indices: [INDEX_A_NAME, INDEX_B_NAME, DATA_STREAM_A_NAME, DATA_STREAM_B_NAME] }) .expect(200); expect(body).toStrictEqual({ - commonFields: [{ name: 'email', type: 'text', normalizedType: 'text' }], + commonFields: [ + { name: 'email', type: 'text', normalizedType: 'text' }, + { name: '@timestamp', type: 'date', normalizedType: 'date' }, + ], indices: [ { index: INDEX_A_NAME, @@ -112,6 +144,14 @@ export default function ({ getService }: FtrProviderContext) { { name: 'email', type: 'text', normalizedType: 'text' }, ], }, + { + index: DATA_STREAM_A_NAME, + fields: [{ name: '@timestamp', type: 'date', normalizedType: 'date' }], + }, + { + index: DATA_STREAM_B_NAME, + fields: [{ name: '@timestamp', type: 'date', normalizedType: 'date' }], + }, ], }); }); @@ -128,5 +168,20 @@ export default function ({ getService }: FtrProviderContext) { body.indices.every((value: string) => [INDEX_A_NAME, INDEX_B_NAME].includes(value)) ).toBe(true); }); + + it('Can retrieve matching data streams', async () => { + const { body } = await supertest + .post(`${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({ pattern: 'data-stream-' }) + .expect(200); + + expect( + body.dataStreams.every((value: string) => + [DATA_STREAM_A_NAME, DATA_STREAM_B_NAME].includes(value) + ) + ).toBe(true); + }); }); }