Allow data Streams enrich policies (#200903)

Closes [#187438](#187438)

## Summary

This PR introduces the data streams as source indices in the enrich
policies. For that, it uses the groups functionality, so the indices are
separated from the data streams.


https://github.com/user-attachments/assets/1716d57b-b8c7-417a-b75c-33378f3269ca

### Strings
I've changed some of the existing strings but left other as they were.
I'll be happy to also change those if folks feel that should be the
case.

I've added a hint text and changed the label for the box in the creation
step (marked in yellow)
<img width="400" alt="Screenshot 2024-11-20 at 12 35 32"
src="https://github.com/user-attachments/assets/794a7063-3f75-4f3d-996c-12ca8b9ed9b6">

I haven't change the label for the indices group (green) and I've
followed the same structure for the new group category (yellow)
<table>
  <tr>
    <td style="padding-right: 10px;">
<img alt="Screenshot 2024-11-20 at 12 37 51"
src="https://github.com/user-attachments/assets/89a554a6-57e4-4208-bde4-d4868dec11bc">
    </td>
    <td>
<img alt="Screenshot 2024-11-20 at 12 38 16"
src="https://github.com/user-attachments/assets/d82c7ae3-79c9-432e-aa3c-8ca02502e18f">
    </td>
  </tr>
</table>

I haven't change the 'Source indices' text in the Summary (green)
because in the request tab the correspondan field in the request is
indices (blue)
<table>
  <tr>
    <td style="padding-right: 10px;">
<img alt="Screenshot 2024-11-20 at 12 38 53"
src="https://github.com/user-attachments/assets/80623ded-11e3-4661-9fa8-ad84182a72a3">
    </td>
    <td>
<img alt="Screenshot 2024-11-20 at 12 39 19"
src="https://github.com/user-attachments/assets/f2ad6449-1a3a-448a-8598-c8c9eb41a717">
    </td>
  </tr>
</table>

I also haven't change the 'Source indices' text in the table or in the
detail flyout (green) since both indices and data streams are treated
like indices.
<table>
  <tr>
    <td style="padding-right: 10px;">
<img width="898" alt="Screenshot 2024-11-20 at 12 39 52"
src="https://github.com/user-attachments/assets/8165fe3f-0286-4ad1-a40d-44e19e4eef29">
    </td>
    <td>
<img width="892" alt="Screenshot 2024-11-20 at 12 40 25"
src="https://github.com/user-attachments/assets/bb046aa4-2a6c-40e0-bb32-ecb4a13adea9">
    </td>
  </tr>
</table>

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_node:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

x-pack/plugins/index_management/__jest__/client_integration/create_enrich_policy/create_enrich_policy.test.tsx

@@ -9,7 +9,11 @@ import React from 'react';
 import { act } from 'react-dom/test-utils';
 
 import { setupEnvironment } from '../helpers';
-import { getMatchingIndices, getFieldsFromIndices } from '../helpers/fixtures';
+import {
+  getMatchingIndices,
+  getFieldsFromIndices,
+  getMatchingDataStreams,
+} from '../helpers/fixtures';
 import { CreateEnrichPoliciesTestBed, setup } from './create_enrich_policy.helpers';
 import { getESPolicyCreationApiCall } from '../../../common/lib';
 
@@ -57,6 +61,7 @@ describe('Create enrich policy', () => {
       hasAllPrivileges: true,
       missingPrivileges: { cluster: [] },
     });
+    httpRequestsMockHelpers.setGetMatchingDataStreams(getMatchingDataStreams());
 
     await act(async () => {
       testBed = await setup(httpSetup);

x-pack/plugins/index_management/__jest__/client_integration/helpers/fixtures.ts

@@ -62,6 +62,9 @@ export const createTestEnrichPolicy = (name: string, type: EnrichPolicyType) =>
 export const getMatchingIndices = () => ({
   indices: ['test-1', 'test-2', 'test-3', 'test-4', 'test-5'],
 });
+export const getMatchingDataStreams = () => ({
+  dataStreams: ['test-6', 'test-7', 'test-8', 'test-9', 'test-10'],
+});
 
 export const getFieldsFromIndices = () => ({
   commonFields: [],

x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts

@@ -172,6 +172,13 @@ const registerHttpRequestMockHelpers = (
       response,
       error
     );
+  const setGetMatchingDataStreams = (response?: HttpResponse, error?: ResponseError) =>
+    mockResponse(
+      'POST',
+      `${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`,
+      response,
+      error
+    );
 
   const setGetFieldsFromIndices = (response?: HttpResponse, error?: ResponseError) =>
     mockResponse(
@@ -249,6 +256,7 @@ const registerHttpRequestMockHelpers = (
     setGetPrivilegesResponse,
     setCreateEnrichPolicy,
     setInferenceModels,
+    setGetMatchingDataStreams,
   };
 };
 

x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/configuration.tsx

@@ -94,18 +94,15 @@ export const configurationFormSchema: FormSchema = {
   },
 
   sourceIndices: {
-    label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel', {
-      defaultMessage: 'Source indices',
+    label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceLabel', {
+      defaultMessage: 'Source',
     }),
     validations: [
       {
         validator: fieldValidators.emptyField(
-          i18n.translate(
-            'xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError',
-            {
-              defaultMessage: 'At least one source index is required.',
-            }
-          )
+          i18n.translate('xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceRequiredError', {
+            defaultMessage: 'At least one source is required.',
+          })
         ),
       },
     ],

x-pack/plugins/index_management/public/application/sections/enrich_policy_create/steps/fields/indices_selector.tsx

@@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n';
 import { uniq, isEmpty } from 'lodash';
 import { EuiFormRow, EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
 import type { EuiComboBoxProps } from '@elastic/eui';
-import { getMatchingIndices } from '../../../../services/api';
+import { getMatchingDataStreams, getMatchingIndices } from '../../../../services/api';
 import type { FieldHook } from '../../../../../shared_imports';
 import { getFieldValidityAndErrorMessage } from '../../../../../shared_imports';
 
@@ -25,23 +25,76 @@ interface Props {
   [key: string]: any;
 }
 
-const getIndexOptions = async (patternString: string) => {
-  const options: IOption[] = [];
+interface GetMatchingOptionsParams {
+  matches: string[];
+  optionsMessage: string;
+  noMatchingMessage: string;
+}
 
+const i18nTexts = {
+  indices: {
+    options: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', {
+      defaultMessage: 'Based on your indices',
+    }),
+    noMatches: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', {
+      defaultMessage: 'No indices match your search criteria.',
+    }),
+  },
+  dataStreams: {
+    options: i18n.translate(
+      'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.optionsLabel',
+      {
+        defaultMessage: 'Based on your data streams',
+      }
+    ),
+    noMatches: i18n.translate(
+      'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.dataStream.noMatchingOption',
+      {
+        defaultMessage: 'No data streams match your search criteria.',
+      }
+    ),
+  },
+  sourcePlaceholder: i18n.translate(
+    'xpack.idxMgmt.enrichPolicyCreate.indicesSelector.placeholder',
+    {
+      defaultMessage: 'Select source indices and data streams.',
+    }
+  ),
+};
+
+const getIndexOptions = async (patternString: string) => {
   if (!patternString) {
-    return options;
+    return [];
   }
 
-  const { data } = await getMatchingIndices(patternString);
-  const matchingIndices = data.indices;
+  const { data: indicesData } = await getMatchingIndices(patternString);
+  const { data: dataStreamsData } = await getMatchingDataStreams(patternString);
 
-  if (matchingIndices.length) {
-    const matchingOptions = uniq([...matchingIndices]);
+  const indices = getMatchingOptions({
+    matches: indicesData.indices,
+    optionsMessage: i18nTexts.indices.options,
+    noMatchingMessage: i18nTexts.indices.noMatches,
+  });
+  const dataStreams = getMatchingOptions({
+    matches: dataStreamsData.dataStreams,
+    optionsMessage: i18nTexts.dataStreams.options,
+    noMatchingMessage: i18nTexts.dataStreams.noMatches,
+  });
+
+  return [...indices, ...dataStreams];
+};
+
+const getMatchingOptions = ({
+  matches,
+  optionsMessage,
+  noMatchingMessage,
+}: GetMatchingOptionsParams) => {
+  const options: IOption[] = [];
+  if (matches.length) {
+    const matchingOptions = uniq([...matches]);
 
     options.push({
-      label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.optionsLabel', {
-        defaultMessage: 'Based on your indices',
-      }),
+      label: optionsMessage,
       options: matchingOptions
         .map((match) => {
           return {
@@ -53,13 +106,10 @@ const getIndexOptions = async (patternString: string) => {
     });
   } else {
     options.push({
-      label: i18n.translate('xpack.idxMgmt.enrichPolicyCreate.indicesSelector.noMatchingOption', {
-        defaultMessage: 'No indices match your search criteria.',
-      }),
+      label: noMatchingMessage,
       options: [],
     });
   }
-
   return options;
 };
 
@@ -71,7 +121,6 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => {
   const onSearchChange = useCallback(
     async (search: string) => {
       const indexPattern = isEmpty(search) ? '*' : search;
-
       setIsIndiciesLoading(true);
       setIndexOptions(await getIndexOptions(indexPattern));
       setIsIndiciesLoading(false);
@@ -98,6 +147,7 @@ export const IndicesSelector = ({ field, euiFieldProps, ...rest }: Props) => {
     >
       <EuiComboBox
         async
+        placeholder={i18nTexts.sourcePlaceholder}
         isLoading={isIndiciesLoading}
         options={indexOptions}
         noSuggestions={!indexOptions.length}

x-pack/plugins/index_management/public/application/services/api.ts

@@ -400,6 +400,16 @@ export async function getMatchingIndices(pattern: string) {
   return result;
 }
 
+export async function getMatchingDataStreams(pattern: string) {
+  const result = sendRequest({
+    path: `${INTERNAL_API_BASE_PATH}/enrich_policies/get_matching_data_streams`,
+    method: 'post',
+    body: JSON.stringify({ pattern }),
+  });
+
+  return result;
+}
+
 export async function getFieldsFromIndices(indices: string[]) {
   const result = sendRequest<FieldFromIndicesRequest>({
     path: `${INTERNAL_API_BASE_PATH}/enrich_policies/get_fields_from_indices`,

x-pack/plugins/index_management/server/routes/api/enrich_policies/enrich_policies.test.ts

@@ -366,4 +366,33 @@ describe('Enrich policies API', () => {
       expect(searchMock).toHaveBeenCalled();
     });
   });
+
+  describe('Get matching indices - POST /api/index_management/enrich_policies/get_matching_data_streams', () => {
+    const getDataStreamsMock = router.getMockESApiFn('indices.getDataStream');
+
+    it('Return matching data streams', async () => {
+      const mockRequest: RequestMock = {
+        method: 'post',
+        path: addInternalBasePath('/enrich_policies/get_matching_data_streams'),
+        body: {
+          pattern: 'test',
+        },
+      };
+
+      getDataStreamsMock.mockResolvedValue({
+        body: {},
+        statusCode: 200,
+      });
+
+      const res = await router.runRequest(mockRequest);
+
+      expect(res).toEqual({
+        body: {
+          dataStreams: [],
+        },
+      });
+
+      expect(getDataStreamsMock).toHaveBeenCalledWith({ name: '*test*', expand_wildcards: 'open' });
+    });
+  });
 });

x-pack/plugins/index_management/server/routes/api/enrich_policies/helpers.ts

@@ -145,3 +145,15 @@ export async function getIndices(dataClient: IScopedClusterClient, pattern: stri
 
   return indices.buckets ? indices.buckets.map((bucket) => bucket.key) : [];
 }
+export async function getDataStreams(
+  dataClient: IScopedClusterClient,
+  pattern: string,
+  limit = 10
+) {
+  const response = await dataClient.asCurrentUser.indices.getDataStream({
+    name: pattern,
+    expand_wildcards: 'open',
+  });
+
+  return response.data_streams ? response.data_streams.map((dataStream) => dataStream.name) : [];
+}

x-pack/plugins/index_management/server/routes/api/enrich_policies/register_create_route.ts

@@ -13,7 +13,13 @@ import { RouteDependencies } from '../../../types';
 import { addInternalBasePath } from '..';
 import { enrichPoliciesActions } from '../../../lib/enrich_policies';
 import { serializeAsESPolicy } from '../../../../common/lib';
-import { normalizeFieldsList, getIndices, FieldCapsList, getCommonFields } from './helpers';
+import {
+  normalizeFieldsList,
+  getIndices,
+  FieldCapsList,
+  getCommonFields,
+  getDataStreams,
+} from './helpers';
 
 const validationSchema = schema.object({
   policy: schema.object({
@@ -105,6 +111,33 @@ export function registerCreateRoute({ router, lib: { handleEsError } }: RouteDep
       }
     }
   );
+  router.post(
+    {
+      path: addInternalBasePath('/enrich_policies/get_matching_data_streams'),
+      validate: { body: getMatchingIndicesSchema },
+    },
+    async (context, request, response) => {
+      let { pattern } = request.body;
+      const client = (await context.core).elasticsearch.client as IScopedClusterClient;
+
+      // Add wildcards to the search query to match the behavior of the
+      // index pattern search in the Kibana UI.
+      if (!pattern.startsWith('*')) {
+        pattern = `*${pattern}`;
+      }
+      if (!pattern.endsWith('*')) {
+        pattern = `${pattern}*`;
+      }
+
+      try {
+        const dataStreams = await getDataStreams(client, pattern);
+
+        return response.ok({ body: { dataStreams } });
+      } catch (error) {
+        return handleEsError({ error, response });
+      }
+    }
+  );
 
   router.post(
     {

x-pack/plugins/translations/translations/fr-FR.json

@@ -22567,8 +22567,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "Requête (facultative)",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Plage",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} correspond à un nombre, une date ou une plage d'adresses IP.",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "Index source",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "Au moins un index source est requis.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "Une valeur est requise pour le type de politique.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "Détermine comment faire correspondre les données avec les documents entrants.",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "Charger un fichier",

x-pack/plugins/translations/translations/ja-JP.json

@@ -22538,8 +22538,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "クエリー（任意）",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "Range",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type}は、番号、日付、またはIPアドレスの範囲と一致します。",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "ソースインデックス",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "ソースインデックスが最低1つ必要です。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "ポリシータイプ値が必要です。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "どのようにデータを受信ドキュメントに一致させるかを決定します。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "ファイルをアップロード",

x-pack/plugins/translations/translations/zh-CN.json

@@ -22148,8 +22148,6 @@
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.queryLabel": "查询（可选）",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeOption": "范围",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.rangeTypePopOver": "{type} 匹配一个数字、日期或 IP 地址范围。",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesLabel": "源索引",
-    "xpack.idxMgmt.enrichPolicyCreate.configurationStep.sourceIndicesRequiredError": "至少需要一个源索引。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeRequiredError": "策略类型值必填。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.typeTitlePopOver": "确定如何将数据匹配到传入文档。",
     "xpack.idxMgmt.enrichPolicyCreate.configurationStep.uploadFileLink": "上传文件",