Merge branch 'main' into bedrock-connector-oas

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -76,7 +76,7 @@ if [[ "${PUBLISH_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0
   "kibana_commit": "$BUILDKITE_COMMIT",
   "kibana_branch": "$BUILDKITE_BRANCH",
   "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG",
-  "elasticsearch_serverless_image_url: "$SOURCE_IMAGE",
+  "elasticsearch_serverless_image_url": "$SOURCE_IMAGE",
   "elasticsearch_serverless_commit": "TODO: this currently can't be decided",
   "elasticsearch_commit": "$ELASTIC_COMMIT_HASH",
   "created_at": "`date`",

src/plugins/data_views/common/constants.ts

@@ -57,3 +57,9 @@ export const PLUGIN_NAME = 'DataViews';
  * @public
  */
 export const FIELDS_FOR_WILDCARD_PATH = '/internal/data_views/_fields_for_wildcard';
+
+/**
+ * Existing indices path
+ * @public
+ */
+export const EXISTING_INDICES_PATH = '/internal/data_views/_existing_indices';

src/plugins/data_views/common/data_views/data_views.test.ts

@@ -662,36 +662,4 @@ describe('IndexPatterns', () => {
       expect(apiClient.getFieldsForWildcard.mock.calls[0][0].allowNoIndex).toBe(true);
     });
   });
-
-  describe('getExistingIndices', () => {
-    test('getExistingIndices returns the valid matched indices', async () => {
-      apiClient.getFieldsForWildcard = jest
-        .fn()
-        .mockResolvedValueOnce({ fields: ['length'] })
-        .mockResolvedValue({ fields: [] });
-      const patternList = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
-      expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2);
-      expect(patternList.length).toBe(1);
-    });
-
-    test('getExistingIndices checks the positive pattern if provided with a negative pattern', async () => {
-      const mockFn = jest.fn().mockResolvedValue({ fields: ['length'] });
-      apiClient.getFieldsForWildcard = mockFn;
-      const patternList = await indexPatterns.getExistingIndices(['-filebeat-*', 'filebeat-*']);
-      expect(mockFn.mock.calls[0][0].pattern).toEqual('filebeat-*');
-      expect(mockFn.mock.calls[1][0].pattern).toEqual('filebeat-*');
-      expect(patternList).toEqual(['-filebeat-*', 'filebeat-*']);
-    });
-
-    test('getExistingIndices handles an error', async () => {
-      apiClient.getFieldsForWildcard = jest
-        .fn()
-        .mockImplementationOnce(async () => {
-          throw new DataViewMissingIndices('Catch me if you can!');
-        })
-        .mockImplementation(() => Promise.resolve({ fields: ['length'] }));
-      const patternList = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
-      expect(patternList).toEqual(['filebeat-*']);
-    });
-  });
 });

src/plugins/data_views/common/data_views/data_views.ts

@@ -7,12 +7,10 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { defer, from } from 'rxjs';
 import type { PublicMethodsOf } from '@kbn/utility-types';
 import { castEsToKbnFieldTypeName } from '@kbn/field-types';
 import { FieldFormatsStartCommon, FORMATS_UI_SETTINGS } from '@kbn/field-formats-plugin/common';
 import { v4 as uuidv4 } from 'uuid';
-import { rateLimitingForkJoin } from './utils';
 import { PersistenceAPI } from '../types';
 
 import { createDataViewCache } from '.';
@@ -240,12 +238,6 @@ export interface DataViewsServicePublicMethods {
    * @param options - options for getting fields
    */
   getFieldsForWildcard: (options: GetFieldsOptions) => Promise<FieldSpec[]>;
-  /**
-   * Get existing index pattern list by providing string array index pattern list.
-   * @param indices - index pattern list
-   * @returns index pattern list of index patterns that match indices
-   */
-  getExistingIndices: (indices: string[]) => Promise<string[]>;
   /**
    * Get list of data view ids.
    * @param refresh - clear cache and fetch from server
@@ -524,41 +516,6 @@ export class DataViewsService {
     return fields;
   };
 
-  /**
-   * Get existing index pattern list by providing string array index pattern list.
-   * @param indices index pattern list
-   * @returns index pattern list
-   */
-  getExistingIndices = async (indices: string[]): Promise<string[]> => {
-    const indicesObs = indices.map((pattern) => {
-      // when checking a negative pattern, check if the positive pattern exists
-      const indexToQuery = pattern.trim().startsWith('-')
-        ? pattern.trim().substring(1)
-        : pattern.trim();
-      return defer(() =>
-        from(
-          this.getFieldsForWildcard({
-            // check one field to keep request fast/small
-            fields: ['_id'],
-            // true so no errors thrown in browser
-            allowNoIndex: true,
-            pattern: indexToQuery,
-          })
-        )
-      );
-    });
-
-    return new Promise<boolean[]>((resolve) => {
-      rateLimitingForkJoin(indicesObs, 3, []).subscribe((value) => {
-        resolve(value.map((v) => v.length > 0));
-      });
-    })
-      .then((allPatterns: boolean[]) =>
-        indices.filter((pattern, i, self) => self.indexOf(pattern) === i && allPatterns[i])
-      )
-      .catch(() => indices);
-  };
-
   /**
    * Get field list by providing an index patttern (or spec).
    * @param options options for getting field list

src/plugins/data_views/common/types.ts

@@ -319,6 +319,11 @@ export interface FieldsForWildcardResponse {
   indices: string[];
 }
 
+/**
+ * Existing Indices response
+ */
+export type ExistingIndicesResponse = string[];
+
 export interface IDataViewsApiClient {
   getFieldsForWildcard: (options: GetFieldsOptions) => Promise<FieldsForWildcardResponse>;
   hasUserDataView: () => Promise<boolean>;

src/plugins/data_views/public/data_views_service_public.ts

@@ -6,11 +6,15 @@
  * Side Public License, v 1.
  */
 
+import { HttpStart } from '@kbn/core/public';
 import { DataViewsService, MatchedItem } from '.';
 
 import { DataViewsServiceDeps } from '../common/data_views/data_views';
 import { HasDataService } from '../common';
 
+import { ExistingIndicesResponse } from '../common/types';
+import { EXISTING_INDICES_PATH } from '../common/constants';
+
 /**
  * Data Views public service dependencies
  * @public
@@ -32,6 +36,7 @@ export interface DataViewsServicePublicDeps extends DataViewsServiceDeps {
 
   getRollupsEnabled: () => boolean;
   scriptedFieldsEnabled: boolean;
+  http: HttpStart;
 }
 
 /**
@@ -48,6 +53,7 @@ export class DataViewsServicePublic extends DataViewsService {
   }) => Promise<MatchedItem[]>;
   public hasData: HasDataService;
   private rollupsEnabled: boolean = false;
+  private readonly http: HttpStart;
   public readonly scriptedFieldsEnabled: boolean;
 
   /**
@@ -62,9 +68,22 @@ export class DataViewsServicePublic extends DataViewsService {
     this.getIndices = deps.getIndices;
     this.rollupsEnabled = deps.getRollupsEnabled();
     this.scriptedFieldsEnabled = deps.scriptedFieldsEnabled;
+    this.http = deps.http;
   }
 
   getRollupsEnabled() {
     return this.rollupsEnabled;
   }
+
+  /**
+   * Get existing index pattern list by providing string array index pattern list.
+   * @param indices - index pattern list
+   * @returns index pattern list of index patterns that match indices
+   */
+  async getExistingIndices(indices: string[]): Promise<ExistingIndicesResponse> {
+    return this.http.get<ExistingIndicesResponse>(EXISTING_INDICES_PATH, {
+      query: { indices },
+      version: '1',
+    });
+  }
 }

src/plugins/data_views/public/plugin.ts

@@ -88,6 +88,7 @@ export class DataViewsPublicPlugin
       savedObjectsClient: new ContentMagementWrapper(contentManagement.client),
       apiClient: new DataViewsApiClient(http),
       fieldFormats,
+      http,
       onNotification: (toastInputFields, key) => {
         onNotifDebounced(key)(toastInputFields);
       },

src/plugins/data_views/public/types.ts

@@ -123,6 +123,12 @@ export interface DataViewsServicePublic extends DataViewsServicePublicMethods {
   }) => Promise<MatchedItem[]>;
   getRollupsEnabled: () => boolean;
   scriptedFieldsEnabled: boolean;
+  /**
+   * Get existing index pattern list by providing string array index pattern list.
+   * @param indices - index pattern list
+   * @returns index pattern list of index patterns that match indices
+   */
+  getExistingIndices: (indices: string[]) => Promise<string[]>;
 }
 
 export type DataViewsContract = DataViewsServicePublic;

src/plugins/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -9,6 +9,7 @@
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { IndexPatternsFetcher } from '.';
 import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
+import { DataViewMissingIndices } from '../../common';
 
 const rollupResponse = {
   foo: {
@@ -34,17 +35,16 @@ describe('Index Pattern Fetcher - server', () => {
   beforeEach(() => {
     jest.clearAllMocks();
     esClient = elasticsearchServiceMock.createElasticsearchClient();
+    esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse);
     indexPatterns = new IndexPatternsFetcher(esClient, false, true);
   });
   it('calls fieldcaps once', async () => {
-    esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse);
     indexPatterns = new IndexPatternsFetcher(esClient, true, true);
     await indexPatterns.getFieldsForWildcard({ pattern: patternList });
     expect(esClient.fieldCaps).toHaveBeenCalledTimes(1);
   });
 
   it('calls rollup api when given rollup data view', async () => {
-    esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse);
     esClient.rollup.getRollupIndexCaps.mockResponse(
       rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse
     );
@@ -58,7 +58,6 @@ describe('Index Pattern Fetcher - server', () => {
   });
 
   it("doesn't call rollup api when given rollup data view and rollups are disabled", async () => {
-    esClient.fieldCaps.mockResponse(response as unknown as estypes.FieldCapsResponse);
     esClient.rollup.getRollupIndexCaps.mockResponse(
       rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse
     );
@@ -70,4 +69,39 @@ describe('Index Pattern Fetcher - server', () => {
     });
     expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(0);
   });
+
+  describe('getExistingIndices', () => {
+    test('getExistingIndices returns the valid matched indices', async () => {
+      indexPatterns = new IndexPatternsFetcher(esClient, true, true);
+      indexPatterns.getFieldsForWildcard = jest
+        .fn()
+        .mockResolvedValueOnce({ indices: ['length'] })
+        .mockResolvedValue({ indices: [] });
+      const result = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
+      expect(indexPatterns.getFieldsForWildcard).toBeCalledTimes(2);
+      expect(result.length).toBe(1);
+    });
+
+    test('getExistingIndices checks the positive pattern if provided with a negative pattern', async () => {
+      indexPatterns = new IndexPatternsFetcher(esClient, true, true);
+      const mockFn = jest.fn().mockResolvedValue({ indices: ['length'] });
+      indexPatterns.getFieldsForWildcard = mockFn;
+      const result = await indexPatterns.getExistingIndices(['-filebeat-*', 'filebeat-*']);
+      expect(mockFn.mock.calls[0][0].pattern).toEqual('filebeat-*');
+      expect(mockFn.mock.calls[1][0].pattern).toEqual('filebeat-*');
+      expect(result).toEqual(['-filebeat-*', 'filebeat-*']);
+    });
+
+    test('getExistingIndices handles an error', async () => {
+      indexPatterns = new IndexPatternsFetcher(esClient, true, true);
+      indexPatterns.getFieldsForWildcard = jest
+        .fn()
+        .mockImplementationOnce(async () => {
+          throw new DataViewMissingIndices('Catch me if you can!');
+        })
+        .mockImplementation(() => Promise.resolve({ indices: ['length'] }));
+      const result = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
+      expect(result).toEqual(['filebeat-*']);
+    });
+  });
 });

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -9,6 +9,8 @@
 import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { ElasticsearchClient } from '@kbn/core/server';
 import { keyBy } from 'lodash';
+import { defer, from } from 'rxjs';
+import { rateLimitingForkJoin } from '../../common/data_views/utils';
 import type { QueryDslQueryContainer } from '../../common/types';
 
 import {
@@ -117,4 +119,37 @@ export class IndexPatternsFetcher {
     }
     return fieldCapsResponse;
   }
+
+  /**
+   * Get existing index pattern list by providing string array index pattern list.
+   * @param indices - index pattern list
+   * @returns index pattern list of index patterns that match indices
+   */
+  async getExistingIndices(indices: string[]): Promise<string[]> {
+    const indicesObs = indices.map((pattern) => {
+      // when checking a negative pattern, check if the positive pattern exists
+      const indexToQuery = pattern.trim().startsWith('-')
+        ? pattern.trim().substring(1)
+        : pattern.trim();
+      return defer(() =>
+        from(
+          this.getFieldsForWildcard({
+            // check one field to keep request fast/small
+            fields: ['_id'],
+            pattern: indexToQuery,
+          })
+        )
+      );
+    });
+
+    return new Promise<boolean[]>((resolve) => {
+      rateLimitingForkJoin(indicesObs, 3, { fields: [], indices: [] }).subscribe((value) => {
+        resolve(value.map((v) => v.indices.length > 0));
+      });
+    })
+      .then((allPatterns: boolean[]) =>
+        indices.filter((pattern, i, self) => self.indexOf(pattern) === i && allPatterns[i])
+      )
+      .catch(() => indices);
+  }
 }

src/plugins/data_views/server/rest_api_routes/internal/existing_indices.test.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { parseIndices } from './existing_indices';
+
+describe('parseIndices', () => {
+  it('should return an array of indices when given an array of indices', () => {
+    const indices = ['index1', 'index2', 'index3'];
+    const result = parseIndices(indices);
+    expect(result).toEqual(indices);
+  });
+
+  it('should return an array of indices when given a JSON-stringified array of indices', () => {
+    const indices = '["index1", "index2", "index3"]';
+    const result = parseIndices(indices);
+    expect(result).toEqual(['index1', 'index2', 'index3']);
+  });
+
+  it('should return an array with a single index when given a single index name', () => {
+    const index = 'index1';
+    const result = parseIndices(index);
+    expect(result).toEqual(['index1']);
+  });
+
+  it('should throw an error when given an invalid JSON-stringified array', () => {
+    const indices = '["index1", "index2"';
+    expect(() => {
+      parseIndices(indices);
+    }).toThrowError(
+      'indices should be an array of index aliases, a JSON-stringified array of index aliases, or a single index alias'
+    );
+  });
+
+  it('should throw an error when given a string with a comma but not JSON-stringified', () => {
+    const indices = 'index1,index2';
+    expect(() => {
+      parseIndices(indices);
+    }).toThrowError(
+      'indices should be an array of index aliases, a JSON-stringified array of index aliases, or a single index alias'
+    );
+  });
+
+  it('should return an empty array when given an empty array', () => {
+    const indices: string[] = [];
+    const result = parseIndices(indices);
+    expect(result).toEqual([]);
+  });
+});