From 5fdcb3fed6d23bf1a5fae61fd977124d3f47e2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 13 May 2024 11:56:18 +0200 Subject: [PATCH] [Obs AI Assistant] E2E test for contextual insights (#182715) This adds an e2e test for the AI Assistant Contextual Insights on the APM Error details view. Follow-up to: https://github.com/elastic/kibana/pull/182572 --- .../components/insight/insight_base.tsx | 7 +- .../common/bootstrap_apm_synthtrace.ts | 2 +- .../test/apm_api_integration/common/config.ts | 4 +- .../common/config.ts | 23 ++- .../common/create_synthtrace_client.ts | 32 ++++ .../common/config.ts | 4 + .../common/ui/index.ts | 52 +++--- .../tests/contextual_insights/index.spec.ts | 160 ++++++++++++++++++ 8 files changed, 241 insertions(+), 43 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_api_integration/common/create_synthtrace_client.ts create mode 100644 x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight_base.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight_base.tsx index 62310b2aed555..7e54d7671237c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight_base.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/components/insight/insight_base.tsx @@ -138,7 +138,12 @@ export function InsightBase({ onToggle={onToggle} > - + {children} diff --git a/x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts b/x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts index 6bb8da1f8ee58..11d30c2d077e2 100644 --- a/x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts +++ b/x-pack/test/apm_api_integration/common/bootstrap_apm_synthtrace.ts @@ -14,7 +14,7 @@ import url from 'url'; import { kbnTestConfig } from '@kbn/test'; import { InheritedFtrProviderContext } from './ftr_provider_context'; -export async function bootstrapApmSynthtrace( +export async function getApmSynthtraceEsClient( context: InheritedFtrProviderContext, kibanaClient: ApmSynthtraceKibanaClient ) { diff --git a/x-pack/test/apm_api_integration/common/config.ts b/x-pack/test/apm_api_integration/common/config.ts index 37278fa510bc9..8557ddf57c2a2 100644 --- a/x-pack/test/apm_api_integration/common/config.ts +++ b/x-pack/test/apm_api_integration/common/config.ts @@ -21,7 +21,7 @@ import { format, UrlObject } from 'url'; import { MachineLearningAPIProvider } from '../../functional/services/ml/api'; import { APMFtrConfigName } from '../configs'; import { createApmApiClient } from './apm_api_supertest'; -import { bootstrapApmSynthtrace, getApmSynthtraceKibanaClient } from './bootstrap_apm_synthtrace'; +import { getApmSynthtraceEsClient, getApmSynthtraceKibanaClient } from './bootstrap_apm_synthtrace'; import { FtrProviderContext, InheritedFtrProviderContext, @@ -118,7 +118,7 @@ export function createTestConfig( apmFtrConfig: () => config, registry: RegistryProvider, apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => { - return bootstrapApmSynthtrace(context, synthtraceKibanaClient); + return getApmSynthtraceEsClient(context, synthtraceKibanaClient); }, logSynthtraceEsClient: (context: InheritedFtrProviderContext) => new LogsSynthtraceEsClient({ diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 3551c6cb290ce..351d43f4e30b2 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -9,7 +9,8 @@ import { Config, FtrConfigProviderContext } from '@kbn/test'; import supertest from 'supertest'; import { format, UrlObject } from 'url'; import { ObservabilityAIAssistantFtrConfigName } from '../configs'; -import { InheritedServices } from './ftr_provider_context'; +import { getApmSynthtraceEsClient } from './create_synthtrace_client'; +import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context'; import { createObservabilityAIAssistantApiClient, ObservabilityAIAssistantAPIClient, @@ -21,12 +22,8 @@ export interface ObservabilityAIAssistantFtrConfig { kibanaConfig?: Record; } -async function getObservabilityAIAssistantAPIClient({ kibanaServer }: { kibanaServer: UrlObject }) { - const url = format({ - ...kibanaServer, - }); - - return createObservabilityAIAssistantApiClient(supertest(url)); +async function getObservabilityAIAssistantAPIClient(kibanaServerUrl: string) { + return createObservabilityAIAssistantApiClient(supertest(kibanaServerUrl)); } export type CreateTestConfig = ReturnType; @@ -59,20 +56,20 @@ export function createObservabilityAIAssistantAPIConfig({ const services = config.get('services') as InheritedServices; const servers = config.get('servers'); const kibanaServer = servers.kibana as UrlObject; + const kibanaServerUrl = format(kibanaServer); + const apmSynthtraceKibanaClient = services.apmSynthtraceKibanaClient(); const createTest: Omit = { ...config.getAll(), servers, services: { ...services, + apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => + getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient), observabilityAIAssistantAPIClient: async () => { return { - readUser: await getObservabilityAIAssistantAPIClient({ - kibanaServer, - }), - writeUser: await getObservabilityAIAssistantAPIClient({ - kibanaServer, - }), + readUser: await getObservabilityAIAssistantAPIClient(kibanaServerUrl), + writeUser: await getObservabilityAIAssistantAPIClient(kibanaServerUrl), }; }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/create_synthtrace_client.ts b/x-pack/test/observability_ai_assistant_api_integration/common/create_synthtrace_client.ts new file mode 100644 index 0000000000000..5e2497a0342a7 --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/common/create_synthtrace_client.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + ApmSynthtraceEsClient, + ApmSynthtraceKibanaClient, + createLogger, + LogLevel, +} from '@kbn/apm-synthtrace'; +import { InheritedFtrProviderContext } from './ftr_provider_context'; + +export async function getApmSynthtraceEsClient( + context: InheritedFtrProviderContext, + kibanaClient: ApmSynthtraceKibanaClient +) { + const es = context.getService('es'); + + const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion(); + await kibanaClient.installApmPackage(kibanaVersion); + + const esClient = new ApmSynthtraceEsClient({ + client: es, + logger: createLogger(LogLevel.info), + version: kibanaVersion, + refreshAfterIndex: true, + }); + + return esClient; +} diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index 980c4cbf60a4d..35a12f10861c4 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -10,6 +10,7 @@ import { merge } from 'lodash'; import supertest from 'supertest'; import { format, UrlObject } from 'url'; import type { EBTHelpersContract } from '@kbn/analytics-ftr-helpers-plugin/common/types'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { KibanaEBTServerProvider, KibanaEBTUIProvider, @@ -39,6 +40,9 @@ export interface TestConfig extends CreateTestAPI { >; kibana_ebt_server: (context: InheritedFtrProviderContext) => EBTHelpersContract; kibana_ebt_ui: (context: InheritedFtrProviderContext) => EBTHelpersContract; + apmSynthtraceEsClient: ( + context: InheritedFtrProviderContext + ) => Promise; }; } diff --git a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts index 9f41a06394b1e..17fd5f89c7d17 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts @@ -44,6 +44,10 @@ const pages = { apiKeyInput: 'secrets.apiKey-input', saveButton: 'create-connector-flyout-save-btn', }, + contextualInsights: { + button: 'obsAiAssistantInsightButton', + text: 'obsAiAssistantInsightResponse', + }, }; export async function ObservabilityAIAssistantUIProvider({ @@ -55,40 +59,36 @@ export async function ObservabilityAIAssistantUIProvider({ const security = getService('security'); const pageObjects = getPageObjects(['common']); - const roleName = 'observability-ai-assistant-functional-test-role'; + const roleDefinition: Role = { + name: 'observability-ai-assistant-functional-test-role', + elasticsearch: { + cluster: [], + indices: [], + run_as: [], + }, + kibana: [ + { + spaces: ['*'], + base: [], + feature: { + actions: ['all'], + [APM_SERVER_FEATURE_ID]: ['all'], + [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID]: ['all'], + }, + }, + ], + }; return { pages, auth: { login: async () => { await browser.navigateTo(deployment.getHostPort()); - - const roleDefinition: Role = { - name: roleName, - elasticsearch: { - cluster: [], - indices: [], - run_as: [], - }, - kibana: [ - { - spaces: ['*'], - base: [], - feature: { - actions: ['all'], - [APM_SERVER_FEATURE_ID]: ['all'], - [OBSERVABILITY_AI_ASSISTANT_FEATURE_ID]: ['all'], - }, - }, - ], - }; - - await security.role.create(roleName, roleDefinition); - - await security.testUser.setRoles([roleName, 'apm_user']); // performs a page reload + await security.role.create(roleDefinition.name, roleDefinition); + await security.testUser.setRoles([roleDefinition.name, 'apm_user', 'viewer']); // performs a page reload }, logout: async () => { - await security.role.delete(roleName); + await security.role.delete(roleDefinition.name); await security.testUser.restoreDefaults(); }, }, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts new file mode 100644 index 0000000000000..b1edc25053cc8 --- /dev/null +++ b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import moment from 'moment'; +import OpenAI from 'openai'; +import { + createLlmProxy, + LlmProxy, +} from '../../../observability_ai_assistant_api_integration/common/create_llm_proxy'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ApiTest({ getService, getPageObjects }: FtrProviderContext) { + const ui = getService('observabilityAIAssistantUI'); + const testSubjects = getService('testSubjects'); + const supertest = getService('supertest'); + const retry = getService('retry'); + const log = getService('log'); + const browser = getService('browser'); + const deployment = getService('deployment'); + const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); + const { common } = getPageObjects(['header', 'common']); + + async function createSynthtraceErrors() { + const start = moment().subtract(5, 'minutes').valueOf(); + const end = moment().valueOf(); + const serviceName = 'opbeans-go'; + + const serviceInstance = apm + .service({ name: serviceName, environment: 'production', agentName: 'go' }) + .instance('instance-a'); + + const interval = '1m'; + const documents = [ + timerange(start, end) + .interval(interval) + .rate(50) + .generator((timestamp) => + serviceInstance + .transaction({ transactionName: 'GET /banana' }) + .errors( + serviceInstance + .error({ message: 'Some exception', type: 'exception' }) + .timestamp(timestamp) + ) + .duration(10) + .timestamp(timestamp) + .failure() + ), + ]; + + await apmSynthtraceEsClient.index(documents); + } + + async function createConnector(proxy: LlmProxy) { + await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'foo', + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + defaultModel: 'gpt-4', + }, + secrets: { apiKey: 'myApiKey' }, + connector_type_id: '.gen-ai', + }) + .expect(200); + } + + async function deleteConnectors() { + const connectors = await supertest.get('/api/actions/connectors').expect(200); + const promises = connectors.body.map((connector: { id: string }) => { + return supertest + .delete(`/api/actions/connector/${connector.id}`) + .set('kbn-xsrf', 'foo') + .expect(204); + }); + + return Promise.all(promises); + } + + async function navigateToError() { + await common.navigateToApp('apm'); + await browser.get(`${deployment.getHostPort()}/app/apm/services/opbeans-go/errors/`); + await testSubjects.click('errorGroupId'); + } + + describe('Contextual insights for APM errors', () => { + before(async () => { + await Promise.all([ + deleteConnectors(), // cleanup previous connectors + apmSynthtraceEsClient.clean(), // cleanup previous synthtrace data + ]); + + await Promise.all([ + createSynthtraceErrors(), // create synthtrace + ui.auth.login(), // login + ]); + }); + + after(async () => { + await Promise.all([ + deleteConnectors(), // cleanup previous connectors + apmSynthtraceEsClient.clean(), // cleanup synthtrace data + ui.auth.logout(), // logout + ]); + }); + + describe('when there are no connectors', () => { + it('should not show the contextual insight component', async () => { + await navigateToError(); + await testSubjects.missingOrFail(ui.pages.contextualInsights.button); + }); + }); + + describe('when there are connectors', () => { + let proxy: LlmProxy; + + before(async () => { + proxy = await createLlmProxy(log); + await createConnector(proxy); + }); + + after(async () => { + proxy.close(); + }); + + it('should show the contextual insight component on the APM error details page', async () => { + await navigateToError(); + + proxy + .intercept( + 'conversation', + (body) => !isFunctionTitleRequest(body), + 'This error is nothing to worry about. Have a nice day!' + ) + .complete(); + + await testSubjects.click(ui.pages.contextualInsights.button); + + await retry.try(async () => { + const llmResponse = await testSubjects.getVisibleText(ui.pages.contextualInsights.text); + expect(llmResponse).to.contain('This error is nothing to worry about. Have a nice day!'); + }); + }); + }); + }); +} + +function isFunctionTitleRequest(body: string) { + const parsedBody = JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming; + return parsedBody.functions?.find((fn) => fn.name === 'title_conversation') !== undefined; +}