diff --git a/x-pack/plugins/serverless_search/common/types/index.ts b/x-pack/plugins/serverless_search/common/types/index.ts index 13c8fc566defd..c4dac1508374e 100644 --- a/x-pack/plugins/serverless_search/common/types/index.ts +++ b/x-pack/plugins/serverless_search/common/types/index.ts @@ -11,3 +11,12 @@ export interface CreateAPIKeyArgs { name: string; role_descriptors?: Record; } + +export interface IndexData { + name: string; + count: number; +} + +export interface FetchIndicesResult { + indices: IndexData[]; +} diff --git a/x-pack/plugins/serverless_search/common/utils/is_not_nullish.ts b/x-pack/plugins/serverless_search/common/utils/is_not_nullish.ts new file mode 100644 index 0000000000000..d492dad5d52c2 --- /dev/null +++ b/x-pack/plugins/serverless_search/common/utils/is_not_nullish.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function isNotNullish(value: T | null | undefined): value is T { + return value !== null && value !== undefined; +} diff --git a/x-pack/plugins/serverless_search/public/application/components/indexing_api.tsx b/x-pack/plugins/serverless_search/public/application/components/indexing_api.tsx index adef923902c59..eb34e345be142 100644 --- a/x-pack/plugins/serverless_search/public/application/components/indexing_api.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/indexing_api.tsx @@ -5,23 +5,33 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { + EuiCallOut, + EuiComboBox, + EuiComboBoxOptionOption, EuiFlexGroup, EuiFlexItem, + EuiFormRow, EuiLink, EuiPageTemplate, EuiSpacer, + EuiStat, EuiText, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; +import { useQuery } from '@tanstack/react-query'; +import { IndexData, FetchIndicesResult } from '../../../common/types'; +import { FETCH_INDICES_PATH } from '../routes'; +import { API_KEY_PLACEHOLDER, ELASTICSEARCH_URL_PLACEHOLDER } from '../constants'; +import { useKibanaServices } from '../hooks/use_kibana'; import { CodeBox } from './code_box'; import { javascriptDefinition } from './languages/javascript'; import { languageDefinitions } from './languages/languages'; -import { LanguageDefinition } from './languages/types'; +import { LanguageDefinition, LanguageDefinitionSnippetArguments } from './languages/types'; import { OverviewPanel } from './overview_panels/overview_panel'; import { LanguageClientPanel } from './overview_panels/language_client_panel'; @@ -48,9 +58,88 @@ const NoIndicesContent = () => ( ); +interface IndicesContentProps { + indices: IndexData[]; + isLoading: boolean; + onChange: (selectedOptions: Array>) => void; + selectedIndex?: IndexData; + setSearchValue: (searchValue?: string) => void; +} +const IndicesContent = ({ + indices, + isLoading, + onChange, + selectedIndex, + setSearchValue, +}: IndicesContentProps) => { + const toOption = (index: IndexData) => ({ label: index.name, value: index }); + const options: Array> = indices.map(toOption); + return ( + <> + + + + + + + + + + + + + ); +}; + export const ElasticsearchIndexingApi = () => { + const { cloud, http } = useKibanaServices(); const [selectedLanguage, setSelectedLanguage] = useState(javascriptDefinition); + const [indexSearchQuery, setIndexSearchQuery] = useState(undefined); + const [selectedIndex, setSelectedIndex] = useState(undefined); + const elasticsearchURL = useMemo(() => { + return cloud?.elasticsearchUrl ?? ELASTICSEARCH_URL_PLACEHOLDER; + }, [cloud]); + const { data, isLoading, isError } = useQuery({ + queryKey: ['indices', { searchQuery: indexSearchQuery }], + queryFn: async () => { + const query = { + search_query: indexSearchQuery || null, + }; + const result = await http.get(FETCH_INDICES_PATH, { query }); + return result; + }, + }); + + const codeSnippetArguments: LanguageDefinitionSnippetArguments = { + url: elasticsearchURL, + apiKey: API_KEY_PLACEHOLDER, + indexName: selectedIndex?.name, + }; + const showNoIndices = !isLoading && data?.indices?.length === 0 && indexSearchQuery === undefined; return ( @@ -67,6 +156,17 @@ export const ElasticsearchIndexingApi = () => { )} bottomBorder="extended" /> + {isError && ( + + + + )} { } + links={ + showNoIndices + ? undefined + : [ + { + label: i18n.translate( + 'xpack.serverlessSearch.content.indexingApi.ingestDocsLink', + { defaultMessage: 'Ingestion documentation' } + ), + href: '#', // TODO: get doc links ? + }, + ] + } > - + {showNoIndices ? ( + + ) : ( + { + setSelectedIndex(options?.[0]?.value); + }} + setSearchValue={setIndexSearchQuery} + selectedIndex={selectedIndex} + /> + )} diff --git a/x-pack/plugins/serverless_search/public/application/components/languages/console.ts b/x-pack/plugins/serverless_search/public/application/components/languages/console.ts index 08571e6d87173..c2fceaae4f85b 100644 --- a/x-pack/plugins/serverless_search/public/application/components/languages/console.ts +++ b/x-pack/plugins/serverless_search/public/application/components/languages/console.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { INDEX_NAME_PLACEHOLDER } from '../../constants'; import { LanguageDefinition } from './types'; export const consoleDefinition: Partial = { @@ -29,4 +30,8 @@ export const consoleDefinition: Partial = { {"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} { "index" : { "_index" : "books" } } {"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311}`, + ingestDataIndex: ({ indexName }) => `POST _bulk?pretty + { "index" : { "_index" : "${indexName ?? INDEX_NAME_PLACEHOLDER}" } } + {"name": "foo", "title": "bar"} +`, }; diff --git a/x-pack/plugins/serverless_search/public/application/components/languages/curl.ts b/x-pack/plugins/serverless_search/public/application/components/languages/curl.ts index 41c18a9470dcb..cc98b35c87696 100644 --- a/x-pack/plugins/serverless_search/public/application/components/languages/curl.ts +++ b/x-pack/plugins/serverless_search/public/application/components/languages/curl.ts @@ -43,6 +43,13 @@ export API_KEY="${apiKey}"`, { "index" : { "_index" : "books" } } {"name": "The Handmaid'"'"'s Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} '`, + ingestDataIndex: ({ apiKey, url, indexName }) => `curl -X POST ${url}/_bulk?pretty \\ + -H "Authorization: ApiKey ${apiKey}" \\ + -H "Content-Type: application/json" \\ + -d' +{ "index" : { "_index" : "${indexName ?? 'index_name'}" } } +{"name": "foo", "title": "bar" } +`, installClient: `# if cURL is not already installed on your system # then install it with the package manager of your choice diff --git a/x-pack/plugins/serverless_search/public/application/components/languages/javascript.ts b/x-pack/plugins/serverless_search/public/application/components/languages/javascript.ts index 09187c68670ee..9a3978081e404 100644 --- a/x-pack/plugins/serverless_search/public/application/components/languages/javascript.ts +++ b/x-pack/plugins/serverless_search/public/application/components/languages/javascript.ts @@ -59,6 +59,30 @@ bytes: 293, aborted: false } */`, + ingestDataIndex: ({ + apiKey, + url, + indexName, + }) => `const { Client } = require('@elastic/elasticsearch'); +const client = new Client({ + node: '${url}', + auth: { + apiKey: '${apiKey}' + } +}); +const dataset = [ + {'name': 'foo', 'title': 'bar'}, +]; + +// Index with the bulk helper +const result = await client.helpers.bulk({ + datasource: dataset, + onDocument (doc) { + return { index: { _index: '${indexName ?? 'index_name'}' }}; + } +}); +console.log(result); +`, installClient: 'npm install @elastic/elasticsearch@8', name: i18n.translate('xpack.serverlessSearch.languages.javascript', { defaultMessage: 'JavaScript / Node.js', diff --git a/x-pack/plugins/serverless_search/public/application/components/languages/ruby.ts b/x-pack/plugins/serverless_search/public/application/components/languages/ruby.ts index c13ae7d5ced66..b6b8ce3c24428 100644 --- a/x-pack/plugins/serverless_search/public/application/components/languages/ruby.ts +++ b/x-pack/plugins/serverless_search/public/application/components/languages/ruby.ts @@ -7,6 +7,7 @@ import { i18n } from '@kbn/i18n'; import { docLinks } from '../../../../common/doc_links'; +import { INDEX_NAME_PLACEHOLDER } from '../../constants'; import { LanguageDefinition, Languages } from './types'; export const rubyDefinition: LanguageDefinition = { @@ -31,6 +32,18 @@ export const rubyDefinition: LanguageDefinition = { { index: { _index: 'books', data: {name: "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} } } ] client.bulk(body: documents)`, + ingestDataIndex: ({ apiKey, url, indexName }) => `client = ElasticsearchServerless::Client.new( + api_key: '${apiKey}', + url: '${url}' +) + +documents = [ + { index: { _index: '${ + indexName ?? INDEX_NAME_PLACEHOLDER + }', data: {name: "foo", "title": "bar"} } }, +] +client.bulk(body: documents) +`, installClient: `# Requires Ruby version 3.0 or higher # From the project's root directory:$ gem build elasticsearch-serverless.gemspec diff --git a/x-pack/plugins/serverless_search/public/application/components/languages/types.ts b/x-pack/plugins/serverless_search/public/application/components/languages/types.ts index 5e2fe63681389..7849b800fc1a0 100644 --- a/x-pack/plugins/serverless_search/public/application/components/languages/types.ts +++ b/x-pack/plugins/serverless_search/public/application/components/languages/types.ts @@ -21,6 +21,7 @@ export enum Languages { export interface LanguageDefinitionSnippetArguments { url: string; apiKey: string; + indexName?: string; } type CodeSnippet = string | ((args: LanguageDefinitionSnippetArguments) => string); @@ -33,6 +34,7 @@ export interface LanguageDefinition { iconType: string; id: Languages; ingestData: CodeSnippet; + ingestDataIndex: CodeSnippet; installClient: string; languageStyling?: string; name: string; diff --git a/x-pack/plugins/serverless_search/public/application/components/overview.tsx b/x-pack/plugins/serverless_search/public/application/components/overview.tsx index 59c0a0f2eca88..a683f64820785 100644 --- a/x-pack/plugins/serverless_search/public/application/components/overview.tsx +++ b/x-pack/plugins/serverless_search/public/application/components/overview.tsx @@ -23,6 +23,7 @@ import React, { useMemo, useState } from 'react'; import { docLinks } from '../../../common/doc_links'; import { PLUGIN_ID } from '../../../common'; import { useKibanaServices } from '../hooks/use_kibana'; +import { API_KEY_PLACEHOLDER, ELASTICSEARCH_URL_PLACEHOLDER } from '../constants'; import { CodeBox } from './code_box'; import { javascriptDefinition } from './languages/javascript'; import { languageDefinitions } from './languages/languages'; @@ -35,9 +36,6 @@ import { SelectClientPanel } from './overview_panels/select_client'; import { ApiKeyPanel } from './api_key/api_key'; import { LanguageClientPanel } from './overview_panels/language_client_panel'; -const ELASTICSEARCH_URL_PLACEHOLDER = 'https://your_deployment_url'; -const API_KEY_PLACEHOLDER = 'your_api_key'; - export const ElasticsearchOverview = () => { const [selectedLanguage, setSelectedLanguage] = useState(javascriptDefinition); diff --git a/x-pack/plugins/serverless_search/public/application/constants.ts b/x-pack/plugins/serverless_search/public/application/constants.ts new file mode 100644 index 0000000000000..b0b122fa01b5c --- /dev/null +++ b/x-pack/plugins/serverless_search/public/application/constants.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const API_KEY_PLACEHOLDER = 'your_api_key'; +export const ELASTICSEARCH_URL_PLACEHOLDER = 'https://your_deployment_url'; +export const INDEX_NAME_PLACEHOLDER = 'index_name'; diff --git a/x-pack/plugins/serverless_search/public/application/routes.ts b/x-pack/plugins/serverless_search/public/application/routes.ts index 1a933d8f01717..bace5c55d54e2 100644 --- a/x-pack/plugins/serverless_search/public/application/routes.ts +++ b/x-pack/plugins/serverless_search/public/application/routes.ts @@ -9,3 +9,4 @@ export const MANAGEMENT_API_KEYS = '/app/management/security/api_keys'; // Server Routes export const CREATE_API_KEY_PATH = '/internal/security/api_key'; +export const FETCH_INDICES_PATH = '/internal/serverless_search/indices'; diff --git a/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.test.ts b/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.test.ts new file mode 100644 index 0000000000000..da2a229340c98 --- /dev/null +++ b/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ByteSizeValue } from '@kbn/config-schema'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { fetchIndices } from './fetch_indices'; + +describe('fetch indices lib functions', () => { + const mockClient = { + indices: { + get: jest.fn(), + stats: jest.fn(), + }, + security: { + hasPrivileges: jest.fn(), + }, + asInternalUser: {}, + }; + + const regularIndexResponse = { + 'search-regular-index': { + aliases: {}, + }, + }; + + const regularIndexStatsResponse = { + indices: { + 'search-regular-index': { + health: 'green', + size: new ByteSizeValue(108000).toString(), + status: 'open', + total: { + docs: { + count: 100, + deleted: 0, + }, + store: { + size_in_bytes: 108000, + }, + }, + uuid: '83a81e7e-5955-4255-b008-5d6961203f57', + }, + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('fetchIndices', () => { + it('should return regular index', async () => { + mockClient.indices.get.mockImplementation(() => ({ + ...regularIndexResponse, + })); + mockClient.indices.stats.mockImplementation(() => regularIndexStatsResponse); + + await expect( + fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search') + ).resolves.toEqual([{ count: 100, name: 'search-regular-index' }]); + expect(mockClient.indices.get).toHaveBeenCalledWith({ + expand_wildcards: ['open'], + features: ['aliases', 'settings'], + index: '*search*', + }); + + expect(mockClient.indices.stats).toHaveBeenCalledWith({ + index: ['search-regular-index'], + metric: ['docs'], + }); + }); + + it('should not return hidden indices', async () => { + mockClient.indices.get.mockImplementation(() => ({ + ...regularIndexResponse, + ['search-regular-index']: { + ...regularIndexResponse['search-regular-index'], + ...{ settings: { index: { hidden: 'true' } } }, + }, + })); + mockClient.indices.stats.mockImplementation(() => regularIndexStatsResponse); + + await expect( + fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20) + ).resolves.toEqual([]); + expect(mockClient.indices.get).toHaveBeenCalledWith({ + expand_wildcards: ['open'], + features: ['aliases', 'settings'], + index: '*', + }); + + expect(mockClient.indices.stats).not.toHaveBeenCalled(); + }); + + it('should handle index missing in stats call', async () => { + const missingStatsResponse = { + indices: { + some_other_index: { ...regularIndexStatsResponse.indices['search-regular-index'] }, + }, + }; + + mockClient.indices.get.mockImplementationOnce(() => regularIndexResponse); + mockClient.indices.stats.mockImplementationOnce(() => missingStatsResponse); + // simulates when an index has been deleted after get indices call + // deleted index won't be present in the indices stats call response + await expect( + fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search') + ).resolves.toEqual([{ count: 0, name: 'search-regular-index' }]); + }); + + it('should return empty array when no index found', async () => { + mockClient.indices.get.mockImplementationOnce(() => ({})); + await expect( + fetchIndices(mockClient as unknown as ElasticsearchClient, 0, 20, 'search') + ).resolves.toEqual([]); + expect(mockClient.indices.stats).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.ts b/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.ts new file mode 100644 index 0000000000000..9560bbe7f9bb3 --- /dev/null +++ b/x-pack/plugins/serverless_search/server/lib/indices/fetch_indices.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { isNotNullish } from '../../../common/utils/is_not_nullish'; +import { isHidden, isClosed } from '../../utils/index_utils'; + +export async function fetchIndices( + client: ElasticsearchClient, + from: number, + size: number, + searchQuery?: string +) { + const indexPattern = searchQuery ? `*${searchQuery}*` : '*'; + const indexMatches = await client.indices.get({ + expand_wildcards: ['open'], + // for better performance only compute settings of indices but not mappings + features: ['aliases', 'settings'], + index: indexPattern, + }); + const indexNames = Object.keys(indexMatches).filter( + (indexName) => + indexMatches[indexName] && + !isHidden(indexMatches[indexName]) && + !isClosed(indexMatches[indexName]) + ); + const indexNameSlice = indexNames.slice(from, from + size).filter(isNotNullish); + if (indexNameSlice.length === 0) { + return []; + } + const indexCounts = await fetchIndexCounts(client, indexNameSlice); + return indexNameSlice.map((name) => ({ + name, + count: indexCounts[name]?.total?.docs?.count ?? 0, + })); +} + +const fetchIndexCounts = async ( + client: ElasticsearchClient, + indicesNames: string[] +): Promise> => { + if (indicesNames.length === 0) { + return {}; + } + const indexCounts: Record = {}; + // batch calls in batches of 100 to prevent loading too much onto ES + for (let i = 0; i < indicesNames.length; i += 100) { + const stats = await client.indices.stats({ + index: indicesNames.slice(i, i + 100), + metric: ['docs'], + }); + Object.assign(indexCounts, stats.indices); + } + return indexCounts; +}; diff --git a/x-pack/plugins/serverless_search/server/plugin.ts b/x-pack/plugins/serverless_search/server/plugin.ts index caa53bcf0adb0..760e59f82199c 100644 --- a/x-pack/plugins/serverless_search/server/plugin.ts +++ b/x-pack/plugins/serverless_search/server/plugin.ts @@ -8,6 +8,7 @@ import { IRouter, Logger, PluginInitializerContext, Plugin, CoreSetup } from '@kbn/core/server'; import { SecurityPluginStart } from '@kbn/security-plugin/server'; import { registerApiKeyRoutes } from './routes/api_key_routes'; +import { registerIndicesRoutes } from './routes/indices_routes'; import { ServerlessSearchConfig } from './config'; import { ServerlessSearchPluginSetup, ServerlessSearchPluginStart } from './types'; @@ -41,6 +42,7 @@ export class ServerlessSearchPlugin const dependencies = { logger: this.logger, router, security: this.security }; registerApiKeyRoutes(dependencies); + registerIndicesRoutes(dependencies); }); return {}; } diff --git a/x-pack/plugins/serverless_search/server/routes/indices_routes.ts b/x-pack/plugins/serverless_search/server/routes/indices_routes.ts new file mode 100644 index 0000000000000..75f2d737ccc90 --- /dev/null +++ b/x-pack/plugins/serverless_search/server/routes/indices_routes.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +import { fetchIndices } from '../lib/indices/fetch_indices'; +import { RouteDependencies } from '../plugin'; + +export const registerIndicesRoutes = ({ router, security }: RouteDependencies) => { + router.get( + { + path: '/internal/serverless_search/indices', + validate: { + query: schema.object({ + from: schema.number({ defaultValue: 0, min: 0 }), + search_query: schema.maybe(schema.string()), + size: schema.number({ defaultValue: 20, min: 0 }), + }), + }, + }, + async (context, request, response) => { + const client = (await context.core).elasticsearch.client.asCurrentUser; + const user = security.authc.getCurrentUser(request); + + if (!user) { + return response.customError({ + statusCode: 502, + body: 'Could not retrieve current user, security plugin is not ready', + }); + } + + const { from, size, search_query: searchQuery } = request.query; + + const indices = await fetchIndices(client, from, size, searchQuery); + return response.ok({ + body: { + indices, + }, + headers: { 'content-type': 'application/json' }, + }); + } + ); +}; diff --git a/x-pack/plugins/serverless_search/server/utils/index_utils.ts b/x-pack/plugins/serverless_search/server/utils/index_utils.ts new file mode 100644 index 0000000000000..043bb80d7f8d0 --- /dev/null +++ b/x-pack/plugins/serverless_search/server/utils/index_utils.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +export function isHidden(index: IndicesIndexState): boolean { + return index.settings?.index?.hidden === true || index.settings?.index?.hidden === 'true'; +} + +export function isClosed(index: IndicesIndexState): boolean { + return ( + index.settings?.index?.verified_before_close === true || + index.settings?.index?.verified_before_close === 'true' + ); +} diff --git a/x-pack/plugins/serverless_search/tsconfig.json b/x-pack/plugins/serverless_search/tsconfig.json index c9cd5562ff562..e62623958886c 100644 --- a/x-pack/plugins/serverless_search/tsconfig.json +++ b/x-pack/plugins/serverless_search/tsconfig.json @@ -27,5 +27,6 @@ "@kbn/security-plugin", "@kbn/cloud-plugin", "@kbn/share-plugin", + "@kbn/core-elasticsearch-server", ] }