diff --git a/x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts b/x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts index 58111119601d9..2eadfdb039cae 100644 --- a/x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts +++ b/x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts @@ -6,8 +6,12 @@ */ import { z } from '@kbn/zod'; +import { BooleanFromString } from '@kbn/zod-helpers'; export const getEntityDefinitionQuerySchema = z.object({ page: z.optional(z.coerce.number()), perPage: z.optional(z.coerce.number()), + includeState: z.optional(BooleanFromString).default(false), }); + +export type GetEntityDefinitionQuerySchema = z.infer; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/client/index.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/client/index.ts index a2578e1ee09e6..90562264851ce 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/client/index.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/client/index.ts @@ -6,6 +6,7 @@ */ import type { IScopedClusterClient, SavedObjectsClientContract } from '@kbn/core/server'; +import { EntityDefinition } from '@kbn/entities-schema'; import { findEntityDefinitions } from '../entities/find_entity_definition'; import type { EntityDefinitionWithState } from '../entities/types'; @@ -16,7 +17,7 @@ export class EntityManagerClient { ) {} findEntityDefinitions({ page, perPage }: { page?: number; perPage?: number } = {}): Promise< - EntityDefinitionWithState[] + EntityDefinition[] | EntityDefinitionWithState[] > { return findEntityDefinitions({ esClient: this.esClient.asCurrentUser, diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts index 0ea681676e9b1..6932394a14e69 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts @@ -5,9 +5,10 @@ * 2.0. */ -import { compact } from 'lodash'; +import { compact, forEach, reduce } from 'lodash'; import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; +import { NodesIngestTotal } from '@elastic/elasticsearch/lib/api/types'; import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects'; import { generateHistoryTransformId, @@ -19,7 +20,7 @@ import { generateLatestIndexTemplateId, } from './helpers/generate_component_id'; import { BUILT_IN_ID_PREFIX } from './built_in'; -import { EntityDefinitionWithState } from './types'; +import { EntityDefinitionState, EntityDefinitionWithState } from './types'; import { isBackfillEnabled } from './helpers/is_backfill_enabled'; export async function findEntityDefinitions({ @@ -29,6 +30,7 @@ export async function findEntityDefinitions({ id, page = 1, perPage = 10, + includeState = false, }: { soClient: SavedObjectsClientContract; esClient: ElasticsearchClient; @@ -36,7 +38,8 @@ export async function findEntityDefinitions({ id?: string; page?: number; perPage?: number; -}): Promise { + includeState?: boolean; +}): Promise { const filter = compact([ typeof builtIn === 'boolean' ? `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${BUILT_IN_ID_PREFIX}*)` @@ -50,6 +53,10 @@ export async function findEntityDefinitions({ perPage, }); + if (!includeState) { + return response.saved_objects.map(({ attributes }) => attributes); + } + return Promise.all( response.saved_objects.map(async ({ attributes }) => { const state = await getEntityDefinitionState(esClient, attributes); @@ -62,15 +69,18 @@ export async function findEntityDefinitionById({ id, esClient, soClient, + includeState = false, }: { id: string; esClient: ElasticsearchClient; soClient: SavedObjectsClientContract; + includeState?: boolean; }) { const [definition] = await findEntityDefinitions({ esClient, soClient, id, + includeState, perPage: 1, }); @@ -80,43 +90,126 @@ export async function findEntityDefinitionById({ async function getEntityDefinitionState( esClient: ElasticsearchClient, definition: EntityDefinition -) { - const historyIngestPipelineId = generateHistoryIngestPipelineId(definition); - const latestIngestPipelineId = generateLatestIngestPipelineId(definition); +): Promise { + const [ingestPipelines, transforms, indexTemplates] = await Promise.all([ + getIngestPipelineState({ definition, esClient }), + getTransformState({ definition, esClient }), + getIndexTemplatesState({ definition, esClient }), + ]); + + const installed = + ingestPipelines.every((pipeline) => pipeline.installed) && + transforms.every((transform) => transform.installed) && + indexTemplates.every((template) => template.installed); + const running = transforms.every((transform) => transform.running); + + return { + installed, + running, + components: { transforms, ingestPipelines, indexTemplates }, + }; +} + +async function getTransformState({ + definition, + esClient, +}: { + definition: EntityDefinition; + esClient: ElasticsearchClient; +}) { const transformIds = [ generateHistoryTransformId(definition), generateLatestTransformId(definition), ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []), ]; - const [ingestPipelines, indexTemplatesInstalled, transforms] = await Promise.all([ - esClient.ingest.getPipeline( - { - id: `${historyIngestPipelineId},${latestIngestPipelineId}`, - }, - { ignore: [404] } - ), - esClient.indices.existsIndexTemplate({ - name: `${ - (generateLatestIndexTemplateId(definition), generateHistoryIndexTemplateId(definition)) - }`, - }), - esClient.transform.getTransformStats({ - transform_id: transformIds, + + const transformStats = await Promise.all( + transformIds.map((id) => esClient.transform.getTransformStats({ transform_id: id })) + ).then((results) => results.map(({ transforms }) => transforms).flat()); + + return transformIds.map((id) => { + const stats = transformStats.find((transform) => transform.id === id); + if (!stats) { + return { id, installed: false, running: false }; + } + + return { + id, + installed: true, + running: stats.state === 'started' || stats.state === 'indexing', + stats, + }; + }); +} + +async function getIngestPipelineState({ + definition, + esClient, +}: { + definition: EntityDefinition; + esClient: ElasticsearchClient; +}) { + const ingestPipelineIds = [ + generateHistoryIngestPipelineId(definition), + generateLatestIngestPipelineId(definition), + ]; + const [ingestPipelines, ingestPipelinesStats] = await Promise.all([ + esClient.ingest.getPipeline({ id: ingestPipelineIds.join(',') }, { ignore: [404] }), + esClient.nodes.stats({ + metric: 'ingest', + filter_path: ingestPipelineIds.map((id) => `nodes.*.ingest.pipelines.${id}`), }), ]); - const ingestPipelinesInstalled = !!( - ingestPipelines[historyIngestPipelineId] && ingestPipelines[latestIngestPipelineId] + const ingestStatsByPipeline = reduce( + ingestPipelinesStats.nodes, + (pipelines, { ingest }) => { + forEach(ingest?.pipelines, (value: NodesIngestTotal, key: string) => { + if (!pipelines[key]) { + pipelines[key] = { count: 0, failed: 0 }; + } + pipelines[key].count += value.count ?? 0; + pipelines[key].failed += value.failed ?? 0; + }); + return pipelines; + }, + {} as Record ); - const transformsInstalled = transforms.count === transformIds.length; - const transformsRunning = - transformsInstalled && - transforms.transforms.every( - (transform) => transform.state === 'started' || transform.state === 'indexing' - ); - return { - installed: ingestPipelinesInstalled && transformsInstalled && indexTemplatesInstalled, - running: transformsRunning, - }; + return ingestPipelineIds.map((id) => ({ + id, + installed: !!ingestPipelines[id], + stats: ingestStatsByPipeline[id], + })); +} + +async function getIndexTemplatesState({ + definition, + esClient, +}: { + definition: EntityDefinition; + esClient: ElasticsearchClient; +}) { + const indexTemplatesIds = [ + generateLatestIndexTemplateId(definition), + generateHistoryIndexTemplateId(definition), + ]; + const templates = await Promise.all( + indexTemplatesIds.map((id) => + esClient.indices + .getIndexTemplate({ name: id }, { ignore: [404] }) + .then(({ index_templates: indexTemplates }) => indexTemplates?.[0]) + ) + ).then(compact); + return indexTemplatesIds.map((id) => { + const template = templates.find(({ name }) => name === id); + if (!template) { + return { id, installed: false }; + } + return { + id, + installed: true, + stats: template.index_template, + }; + }); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts index e09496762d921..5cee21dc43a07 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts @@ -354,6 +354,7 @@ describe('install_entity_definition', () => { version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 }); const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValueOnce({ @@ -391,6 +392,7 @@ describe('install_entity_definition', () => { version: semver.inc(mockEntityDefinition.version, 'major') ?? '0.0.0', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 }); const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValueOnce({ @@ -426,6 +428,7 @@ describe('install_entity_definition', () => { it('should reinstall when failed installation', async () => { const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.transform.getTransformStats.mockResolvedValue({ transforms: [], count: 0 }); const soClient = savedObjectsClientMock.create(); soClient.find.mockResolvedValueOnce({ diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts index 54e1a4cffcc39..7d6dee4fb2ced 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.ts @@ -39,8 +39,8 @@ import { generateEntitiesLatestIndexTemplateConfig } from './templates/entities_ import { generateEntitiesHistoryIndexTemplateConfig } from './templates/entities_history_template'; import { EntityIdConflict } from './errors/entity_id_conflict_error'; import { EntityDefinitionNotFound } from './errors/entity_not_found'; -import { EntityDefinitionWithState } from './types'; import { mergeEntityDefinitionUpdate } from './helpers/merge_definition_update'; +import { EntityDefinitionWithState } from './types'; import { stopTransforms } from './stop_transforms'; import { deleteTransforms } from './delete_transforms'; @@ -136,6 +136,7 @@ export async function installBuiltInEntityDefinitions({ esClient, soClient, id: builtInDefinition.id, + includeState: true, }); if (!installedDefinition) { @@ -148,7 +149,12 @@ export async function installBuiltInEntityDefinitions({ } // verify existing installation - if (!shouldReinstallBuiltinDefinition(installedDefinition, builtInDefinition)) { + if ( + !shouldReinstallBuiltinDefinition( + installedDefinition as EntityDefinitionWithState, + builtInDefinition + ) + ) { return installedDefinition; } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts index 2cb4eb43791c2..cf0ef5e61342d 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/types.ts @@ -5,12 +5,43 @@ * 2.0. */ +import { + IndicesIndexTemplate, + TransformGetTransformStatsTransformStats, +} from '@elastic/elasticsearch/lib/api/types'; import { EntityDefinition } from '@kbn/entities-schema'; +interface TransformState { + id: string; + installed: boolean; + running: boolean; + stats?: TransformGetTransformStatsTransformStats; +} + +interface IngestPipelineState { + id: string; + installed: boolean; + stats?: { count: number; failed: number }; +} + +interface IndexTemplateState { + id: string; + installed: boolean; + stats?: IndicesIndexTemplate; +} + // state is the *live* state of the definition. since a definition // is composed of several elasticsearch components that can be // modified or deleted outside of the entity manager apis, this can // be used to verify the actual installation is complete and running -export type EntityDefinitionWithState = EntityDefinition & { - state: { installed: boolean; running: boolean }; -}; +export interface EntityDefinitionState { + installed: boolean; + running: boolean; + components: { + transforms: TransformState[]; + ingestPipelines: IngestPipelineState[]; + indexTemplates: IndexTemplateState[]; + }; +} + +export type EntityDefinitionWithState = EntityDefinition & { state: EntityDefinitionState }; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts index 0503bdf770818..5bd9154ec9daf 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts @@ -70,12 +70,24 @@ export class EntityClient { }); } - async getEntityDefinitions({ page = 1, perPage = 10 }: { page?: number; perPage?: number }) { + async getEntityDefinitions({ + id, + page = 1, + perPage = 10, + includeState = false, + }: { + id?: string; + page?: number; + perPage?: number; + includeState?: boolean; + }) { const definitions = await findEntityDefinitions({ esClient: this.options.esClient, soClient: this.options.soClient, page, perPage, + id, + includeState, }); return { definitions }; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/utils.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/utils.ts index d1d76e147efb0..aec8ffa940437 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/lib/utils.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/utils.ts @@ -19,7 +19,7 @@ export const getClientsFromAPIKey = ({ server: EntityManagerServerSetup; }): { esClient: ElasticsearchClient; soClient: SavedObjectsClientContract } => { const fakeRequest = getFakeKibanaRequest({ id: apiKey.id, api_key: apiKey.apiKey }); - const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asCurrentUser; + const esClient = server.core.elasticsearch.client.asScoped(fakeRequest).asSecondaryAuthUser; const soClient = server.core.savedObjects.getScopedClient(fakeRequest); return { esClient, soClient }; }; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts index 101fdde95c9dc..2677b78042620 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts @@ -99,7 +99,7 @@ export class EntityManagerServerPlugin request: KibanaRequest; coreStart: CoreStart; }) { - const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + const esClient = coreStart.elasticsearch.client.asScoped(request).asSecondaryAuthUser; const soClient = coreStart.savedObjects.getScopedClient(request); return new EntityClient({ esClient, soClient, logger: this.logger }); } diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts index cadd4e1dc4dec..d0e2a572cb6f5 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/enablement/check.ts @@ -17,6 +17,7 @@ import { checkIfEntityDiscoveryAPIKeyIsValid, readEntityDiscoveryAPIKey } from ' import { builtInDefinitions } from '../../lib/entities/built_in'; import { findEntityDefinitions } from '../../lib/entities/find_entity_definition'; import { getClientsFromAPIKey } from '../../lib/utils'; +import { EntityDefinitionWithState } from '../../lib/entities/types'; import { createEntityManagerServerRoute } from '../create_entity_manager_server_route'; /** @@ -68,9 +69,13 @@ export const checkEntityDiscoveryEnabledRoute = createEntityManagerServerRoute({ esClient, soClient, id: builtInDefinition.id, + includeState: true, }); - return { installedDefinition: definitions[0], builtInDefinition }; + return { + installedDefinition: definitions[0] as EntityDefinitionWithState, + builtInDefinition, + }; }) ).then((results) => results.reduce( diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts index 2c268aa315560..8ec2489136fb1 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { getEntityDefinitionQuerySchema } from '@kbn/entities-schema'; import { z } from '@kbn/zod'; import { createEntityManagerServerRoute } from '../create_entity_manager_server_route'; @@ -17,6 +16,12 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ * tags: * - definitions * parameters: + * - in: path + * name: id + * description: The entity definition ID + * schema: + * $ref: '#/components/schemas/deleteEntityDefinitionParamsSchema/properties/id' + * required: false * - in: query * name: page * schema: @@ -25,6 +30,10 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ * name: perPage * schema: * $ref: '#/components/schemas/getEntityDefinitionQuerySchema/properties/perPage' + * - in: query + * name: includeState + * schema: + * $ref: '#/components/schemas/getEntityDefinitionQuerySchema/properties/includeState' * responses: * 200: * description: OK @@ -38,27 +47,21 @@ import { createEntityManagerServerRoute } from '../create_entity_manager_server_ * items: * allOf: * - $ref: '#/components/schemas/entityDefinitionSchema' - * - type: object - * properties: - * state: - * type: object - * properties: - * installed: - * type: boolean - * running: - * type: boolean */ export const getEntityDefinitionRoute = createEntityManagerServerRoute({ - endpoint: 'GET /internal/entities/definition', + endpoint: 'GET /internal/entities/definition/{id?}', params: z.object({ query: getEntityDefinitionQuerySchema, + path: z.object({ id: z.optional(z.string()) }), }), handler: async ({ request, response, params, logger, getScopedClient }) => { try { const client = await getScopedClient({ request }); const result = await client.getEntityDefinitions({ - page: params?.query?.page, - perPage: params?.query?.perPage, + id: params.path?.id, + page: params.query.page, + perPage: params.query.perPage, + includeState: params.query.includeState, }); return response.ok({ body: result }); diff --git a/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts b/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts index 822ab2e7ce24a..8eb99ca1fe371 100644 --- a/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts +++ b/x-pack/test/api_integration/apis/entity_manager/helpers/request.ts @@ -16,11 +16,12 @@ export interface Auth { export const getInstalledDefinitions = async ( supertest: Agent, - params: { auth?: Auth; id?: string } = {} + params: { auth?: Auth; id?: string; includeState?: boolean } = {} ): Promise<{ definitions: EntityDefinitionWithState[] }> => { - const { auth, id } = params; + const { auth, id, includeState = true } = params; let req = supertest .get(`/internal/entities/definition${id ? `/${id}` : ''}`) + .query({ includeState }) .set('kbn-xsrf', 'xxx'); if (auth) { req = req.auth(auth.username, auth.password);