[8.x] Allow data Streams enrich policies (#200903) (#202055)

# Backport This will backport the following commits from `main` to `8.x`: - [Allow data Streams enrich policies (#200903)](#200903)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Sonia Sanz Vivas <[email protected]>
elastic · Nov 27, 2024 · 654ea79 · 654ea79
1 parent 454ad56
commit 654ea79
Show file tree

Hide file tree

Showing 13 changed files with 230 additions and 34 deletions.
diff --git a/...management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx b/...management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx
@@ -9,7 +9,11 @@ import React from 'react';
 import { act } from 'react-dom/test-utils';
 
 import { setupEnvironment } from '../helpers';
-import { getMatchingIndices, getFieldsFromIndices } from '../helpers/fixtures';
+import {
+  getMatchingIndices,
+  getFieldsFromIndices,
+  getMatchingDataStreams,
+} from '../helpers/fixtures';
 import { CreateEnrichPoliciesTestBed, setup } from './create_enrich_policy.helpers';
 import { getESPolicyCreationApiCall } from '../../../common/lib';
 
@@ -57,6 +61,7 @@ describe('Create enrich policy', () => {
       hasAllPrivileges: true,
       missingPrivileges: { cluster: [] },
     });
+    httpRequestsMockHelpers.setGetMatchingDataStreams(getMatchingDataStreams());
 
     await act(async () => {
       testBed = await setup(httpSetup);

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts
@@ -62,6 +62,9 @@ export const createTestEnrichPolicy = (name: string, type: EnrichPolicyType) =>
 export const getMatchingIndices = () => ({
   indices: ['test-1', 'test-2', 'test-3', 'test-4', 'test-5'],
 });
+export const getMatchingDataStreams = () => ({
+  dataStreams: ['test-6', 'test-7', 'test-8', 'test-9', 'test-10'],
+});
 
 export const getFieldsFromIndices = () => ({
   commonFields: [],

diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts
@@ -172,6 +172,13 @@ const registerHttpRequestMockHelpers = (
       response,
       error
     );
+  const setGetMatchingDataStreams = (response?: HttpResponse, error?: ResponseError) =>
+    mockResponse(
+      'POST',
+      `${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`,
+      response,
+      error
+    );
 
   const setGetFieldsFromIndices = (response?: HttpResponse, error?: ResponseError) =>
     mockResponse(
@@ -249,6 +256,7 @@ const registerHttpRequestMockHelpers = (
     setGetPrivilegesResponse,
     setCreateEnrichPolicy,
     setInferenceModels,
+    setGetMatchingDataStreams,
   };
 };
 

diff --git a/...index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx b/...index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx
@@ -94,18 +94,15 @@ export const configurationFormSchema: FormSchema = {
   },
 
   sourceIndices: {
-    label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel', {
-      defaultMessage: 'Source indices',
+    label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceLabel', {
+      defaultMessage: 'Source',
     }),
     validations: [
       {
         validator: fieldValidators.emptyField(
-          i18n.translate(
-            'xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError',
-            {
-              defaultMessage: 'At least one source index is required.',
-            }
-          )
+          i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceRequiredError', {
+            defaultMessage: 'At least one source is required.',
+          })
         ),
       },
     ],

diff --git a/...gement/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx b/...gement/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx
@@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n';
 import { uniq, isEmpty } from 'lodash';
 import { EuiFormRow, EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
 import type { EuiComboBoxProps } from '@elastic/eui';
-import { getMatchingIndices } from '../../../../services/api';
+import { getMatchingDataStreams, getMatchingIndices } from '../../../../services/api';
 import type { FieldHook } from '../../../../../shared_imports';
 import { getFieldValidityAndErrorMessage } from '../../../../../shared_imports';
 
@@ -25,23 +25,76 @@ interface Props {
   [key: string]: any;
 }
 
-const getIndexOptions = async (patternString: string) => {
-  const options: IOption[] = [];
+interface GetMatchingOptionsParams {
+  matches: string[];
+  optionsMessage: string;
+  noMatchingMessage: string;
+}
 
+const i18nTexts = {
+  indices: {
+    options: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', {
+      defaultMessage: 'Based on your indices',
+    }),
+    noMatches: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', {
+      defaultMessage: 'No indices match your search criteria.',
+    }),
+  },
+  dataStreams: {
+    options: i18n.translate(
+      'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.optionsLabel',
+      {
+        defaultMessage: 'Based on your data streams',
+      }
+    ),
+    noMatches: i18n.translate(
+      'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.noMatchingOption',
+      {
+        defaultMessage: 'No data streams match your search criteria.',
+      }
+    ),
+  },
+  sourcePlaceholder: i18n.translate(
+    'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.placeholder',
+    {
+      defaultMessage: 'Select source indices and data streams.',
+    }
+  ),
+};
+
+const getIndexOptions = async (patternString: string) => {
   if (!patternString) {
-    return options;
+    return [];
   }
 
-  const { data } = await getMatchingIndices(patternString);
-  const matchingIndices = data.indices;
+  const { data: indicesData } = await getMatchingIndices(patternString);
+  const { data: dataStreamsData } = await getMatchingDataStreams(patternString);
 
-  if (matchingIndices.length) {
-    const matchingOptions = uniq([...matchingIndices]);
+  const indices = getMatchingOptions({
+    matches: indicesData.indices,
+    optionsMessage: i18nTexts.indices.options,
+    noMatchingMessage: i18nTexts.indices.noMatches,
+  });
+  const dataStreams = getMatchingOptions({
+    matches: dataStreamsData.dataStreams,
+    optionsMessage: i18nTexts.dataStreams.options,
+    noMatchingMessage: i18nTexts.dataStreams.noMatches,
+  });
+
+  return [...indices, ...dataStreams];
+};
+
+const getMatchingOptions = ({
+  matches,
+  optionsMessage,
+  noMatchingMessage,
+}: GetMatchingOptionsParams) => {
+  const options: IOption[] = [];
+  if (matches.length) {
+    const matchingOptions = uniq([...matches]);
 
     options.push({
-      label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', {
-        defaultMessage: 'Based on your indices',
-      }),
+      label: optionsMessage,
       options: matchingOptions
         .map((match) => {
           return {
@@ -53,13 +106,10 @@ const getIndexOptions = async (patternString: string) => {
     });
   } else {
     options.push({
-      label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', {
-        defaultMessage: 'No indices match your search criteria.',
-      }),
+      label: noMatchingMessage,
       options: [],
     });
   }
-
   return options;
 };
 
@@ -71,7 +121,6 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => {
   const onSearchChange = useCallback(
     async (search: string) => {
       const indexPattern = isEmpty(search) ? '*' : search;
-
       setIsIndiciesLoading(true);
       setIndexOptions(await getIndexOptions(indexPattern));
       setIsIndiciesLoading(false);
@@ -98,6 +147,7 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => {
     >
       <EuiComboBox
         async
+        placeholder={i18nTexts.sourcePlaceholder}
         isLoading={isIndiciesLoading}
         options={indexOptions}
         noSuggestions={!indexOptions.length}

diff --git a/x-pack/plugins/index_management/public/application/services/api.ts b/x-pack/plugins/index_management/public/application/services/api.ts
@@ -400,6 +400,16 @@ export async function getMatchingIndices(pattern: string) {
   return result;
 }
 
+export async function getMatchingDataStreams(pattern: string) {
+  const result = sendRequest({
+    path: `${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`,
+    method: 'post',
+    body: JSON.stringify({ pattern }),
+  });
+
+  return result;
+}
+
 export async function getFieldsFromIndices(indices: string[]) {
   const result = sendRequest<FieldFromIndicesRequest>({
     path: `${INTERNAL_API_BASE_PATH}/enrich_policies/get_fields_from_indices`,

diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts
@@ -366,4 +366,33 @@ describe('Enrich policies API', () => {
       expect(searchMock).toHaveBeenCalled();
     });
   });
+
+  describe('Get matching indices - POST /api/index_management/enrich_policies/get_matching_data_streams', () => {
+    const getDataStreamsMock = router.getMockESApiFn('indices.getDataStream');
+
+    it('Return matching data streams', async () => {
+      const mockRequest: RequestMock = {
+        method: 'post',
+        path: addInternalBasePath('/enrich_policies/get_matching_data_streams'),
+        body: {
+          pattern: 'test',
+        },
+      };
+
+      getDataStreamsMock.mockResolvedValue({
+        body: {},
+        statusCode: 200,
+      });
+
+      const res = await router.runRequest(mockRequest);
+
+      expect(res).toEqual({
+        body: {
+          dataStreams: [],
+        },
+      });
+
+      expect(getDataStreamsMock).toHaveBeenCalledWith({ name: '*test*', expand_wildcards: 'open' });
+    });
+  });
 });
diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts
@@ -145,3 +145,15 @@ export async function getIndices(dataClient: IScopedClusterClient, pattern: stri
 
   return indices.buckets ? indices.buckets.map((bucket) => bucket.key) : [];
 }
+export async function getDataStreams(
+  dataClient: IScopedClusterClient,
+  pattern: string,
+  limit = 10
+) {
+  const response = await dataClient.asCurrentUser.indices.getDataStream({
+    name: pattern,
+    expand_wildcards: 'open',
+  });
+
+  return response.data_streams ? response.data_streams.map((dataStream) => dataStream.name) : [];
+}
diff --git a/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts b/x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts
@@ -13,7 +13,13 @@ import { RouteDependencies } from '../../../types';
 import { addInternalBasePath } from '..';
 import { enrichPoliciesActions } from '../../../lib/enrich_policies';
 import { serializeAsESPolicy } from '../../../../common/lib';
-import { normalizeFieldsList, getIndices, FieldCapsList, getCommonFields } from './helpers';
+import {
+  normalizeFieldsList,
+  getIndices,
+  FieldCapsList,
+  getCommonFields,
+  getDataStreams,
+} from './helpers';
 
 const validationSchema = schema.object({
   policy: schema.object({
@@ -105,6 +111,33 @@ export function registerCreateRoute({ router, lib: { handleEsError } }: RouteDep
       }
     }
   );
+  router.post(
+    {
+      path: addInternalBasePath('/enrich_policies/get_matching_data_streams'),
+      validate: { body: getMatchingIndicesSchema },
+    },
+    async (context, request, response) => {
+      let { pattern } = request.body;
+      const client = (await context.core).elasticsearch.client as IScopedClusterClient;
+
+      // Add wildcards to the search query to match the behavior of the
+      // index pattern search in the Kibana UI.
+      if (!pattern.startsWith('*')) {
+        pattern = `*${pattern}`;
+      }
+      if (!pattern.endsWith('*')) {
+        pattern = `${pattern}*`;
+      }
+
+      try {
+        const dataStreams = await getDataStreams(client, pattern);
+
+        return response.ok({ body: { dataStreams } });
+      } catch (error) {
+        return handleEsError({ error, response });
+      }
+    }
+  );
 
   router.post(
     {

diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json
@@ -22594,8 +22594,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "Requête (facultative)",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Plage",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} correspond à un nombre, une date ou une plage d'adresses IP.",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "Index source",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "Au moins un index source est requis.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "Une valeur est requise pour le type de politique.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "Détermine comment faire correspondre les données avec les documents entrants.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "Charger un fichier",

diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json
@@ -22565,8 +22565,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "クエリー（任意）",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Range",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type}は、番号、日付、またはIPアドレスの範囲と一致します。",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "ソースインデックス",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "ソースインデックスが最低1つ必要です。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "ポリシータイプ値が必要です。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "どのようにデータを受信ドキュメントに一致させるかを決定します。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "ファイルをアップロード",

diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json
@@ -22617,8 +22617,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "查询（可选）",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "范围",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} 匹配一个数字、日期或 IP 地址范围。",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "源索引",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "至少需要一个源索引。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "策略类型值必填。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "确定如何将数据匹配到传入文档。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "上传文件",