diff --git a/x-pack/plugins/search_playground/public/components/chat.tsx b/x-pack/plugins/search_playground/public/components/chat.tsx index 6cf82dd1ed2e1..de0a296e162b0 100644 --- a/x-pack/plugins/search_playground/public/components/chat.tsx +++ b/x-pack/plugins/search_playground/public/components/chat.tsx @@ -157,6 +157,7 @@ export const Chat = () => { iconType="sparkles" disabled={isToolBarActionsDisabled} onClick={regenerateMessages} + data-test-subj="regenerateActionButton" > { iconType="refresh" disabled={isToolBarActionsDisabled} onClick={handleClearChat} + data-test-subj="clearChatActionButton" > = ({ selectedIndicesCount } defaultMessage: 'Model settings', }), children: , + dataTestId: 'summarizationAccordion', }, { id: useGeneratedHtmlId({ prefix: 'sourcesAccordion' }), @@ -50,13 +51,14 @@ export const ChatSidebar: React.FC = ({ selectedIndicesCount } ), children: , + dataTestId: 'sourcesAccordion', }, ]; const [openAccordionId, setOpenAccordionId] = useState(accordions[0].id); return ( - {accordions.map(({ id, title, extraAction, children }, index) => ( + {accordions.map(({ id, title, extraAction, children, dataTestId }, index) => ( = ({ selectedIndicesCount } buttonProps={{ paddingSize: 'l' }} forceState={openAccordionId === id ? 'open' : 'closed'} onToggle={() => setOpenAccordionId(openAccordionId === id ? '' : id)} + data-test-subj={dataTestId} > {children} diff --git a/x-pack/plugins/search_playground/public/components/sources_panel/indices_list.tsx b/x-pack/plugins/search_playground/public/components/sources_panel/indices_list.tsx index 7af94bbdce814..0cf410072fbd2 100644 --- a/x-pack/plugins/search_playground/public/components/sources_panel/indices_list.tsx +++ b/x-pack/plugins/search_playground/public/components/sources_panel/indices_list.tsx @@ -32,6 +32,7 @@ export const IndicesList: React.FC = ({ indices, onRemoveClick color="primary" label={index} size="s" + data-test-subj="indicesInAccordian" extraAction={{ alwaysShow: true, 'aria-label': i18n.translate('xpack.searchPlayground.sources.indices.removeIndex', { diff --git a/x-pack/test/functional/apps/search_playground/playground_overview.ess.ts b/x-pack/test/functional/apps/search_playground/playground_overview.ess.ts index 48f6d5e13cf88..0afa2979a6c15 100644 --- a/x-pack/test/functional/apps/search_playground/playground_overview.ess.ts +++ b/x-pack/test/functional/apps/search_playground/playground_overview.ess.ts @@ -5,10 +5,16 @@ * 2.0. */ +import type OpenAI from 'openai'; import { FtrProviderContext } from '../../ftr_provider_context'; import { createOpenAIConnector } from './utils/create_openai_connector'; import { MachineLearningCommonAPIProvider } from '../../services/ml/common_api'; +import { + createLlmProxy, + LlmProxy, +} from '../../../observability_ai_assistant_api_integration/common/create_llm_proxy'; + const indexName = 'basic_index'; const esArchiveIndex = 'test/api_integration/fixtures/es_archiver/index_patterns/basic_index'; @@ -18,76 +24,166 @@ export default function (ftrContext: FtrProviderContext) { const commonAPI = MachineLearningCommonAPIProvider(ftrContext); const supertest = getService('supertest'); const esArchiver = getService('esArchiver'); + + const log = getService('log'); + const browser = getService('browser'); + const createIndex = async () => await esArchiver.load(esArchiveIndex); + + let proxy: LlmProxy; let removeOpenAIConnector: () => Promise; const createConnector = async () => { removeOpenAIConnector = await createOpenAIConnector({ supertest, requestHeader: commonAPI.getCommonRequestHeader(), + proxy, }); }; - describe('Playground Overview', () => { + describe('Playground', () => { before(async () => { + proxy = await createLlmProxy(log); await pageObjects.common.navigateToApp('enterpriseSearchApplications/playground'); }); after(async () => { await esArchiver.unload(esArchiveIndex); - await removeOpenAIConnector?.(); + proxy.close(); }); - describe('start chat page', () => { - it('playground app is loaded', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundStartChatPageComponentsToExist(); + describe('setup Page', () => { + it('is loaded successfully', async () => { await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundHeaderComponentsToExist(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundHeaderComponentsToDisabled(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundStartChatPageComponentsToExist(); }); - it('show no index callout', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectNoIndexCalloutExists(); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectCreateIndexButtonToExists(); - }); - - it('hide no index callout when index added', async () => { - await createIndex(); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); - }); - - it('show add connector button', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectAddConnectorButtonExists(); + describe('with gen ai connectors', () => { + before(async () => { + await createConnector(); + await browser.refresh(); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await browser.refresh(); + }); + it('hide gen ai panel', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnector(); + }); }); - it('click add connector button opens connector flyout', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectOpenConnectorPagePlayground(); + describe('without gen ai connectors', () => { + it('should display the set up connectors button', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectAddConnectorButtonExists(); + }); + + it('creates a connector successfully', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectOpenConnectorPagePlayground(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnectorAfterCreatingConnector( + createConnector + ); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await browser.refresh(); + }); }); - it('hide gen ai panel when connector exists', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnector( - createConnector - ); + describe('without any indices', () => { + it('show no index callout', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectNoIndexCalloutExists(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectCreateIndexButtonToExists(); + }); + + it('hide no index callout when index added', async () => { + await createIndex(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); + }); + + after(async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.removeIndexFromComboBox(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); + }); }); - it('show chat page', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectToStartChatPage(); + describe('with existing indices', () => { + before(async () => { + await createConnector(); + await createIndex(); + await browser.refresh(); + }); + + it('dropdown shows up', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectIndicesInDropdown(); + }); + + it('can select index from dropdown and navigate to chat window', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectToSelectIndicesAndStartButtonEnabled( + indexName + ); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); + }); }); }); describe('chat page', () => { - it('chat works', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWorks(); + before(async () => { + await createConnector(); + await createIndex(); + await browser.refresh(); + await pageObjects.searchPlayground.PlaygroundChatPage.navigateToChatPage(indexName); }); - - it('open view code', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectOpenViewCode(); + it('loads successfully', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWindowLoaded(); }); - it('show fields and code in view query', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectViewQueryHasFields(); + describe('chat', () => { + it('works', async () => { + const conversationInterceptor = proxy.intercept( + 'conversation', + (body) => + (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).tools?.find( + (fn) => fn.function.name === 'title_conversation' + ) === undefined + ); + + await pageObjects.searchPlayground.PlaygroundChatPage.sendQuestion(); + + const conversationSimulator = await conversationInterceptor.waitForIntercept(); + + await conversationSimulator.next('My response'); + + await conversationSimulator.complete(); + + await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWorks(); + await pageObjects.searchPlayground.PlaygroundChatPage.expectTokenTooltipExists(); + }); + + it('open view code', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectOpenViewCode(); + }); + + it('show fields and code in view query', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectViewQueryHasFields(); + }); + + it('show edit context', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectEditContextOpens(); + }); }); - it('show edit context', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectEditContextOpens(); + after(async () => { + await removeOpenAIConnector?.(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); }); }); }); diff --git a/x-pack/test/functional/apps/search_playground/utils/create_openai_connector.ts b/x-pack/test/functional/apps/search_playground/utils/create_openai_connector.ts index 864e424664785..ed8c81eda0491 100644 --- a/x-pack/test/functional/apps/search_playground/utils/create_openai_connector.ts +++ b/x-pack/test/functional/apps/search_playground/utils/create_openai_connector.ts @@ -6,20 +6,23 @@ */ import type SuperTest from 'supertest'; +import { LlmProxy } from '../../../../observability_ai_assistant_api_integration/common/create_llm_proxy'; export async function createOpenAIConnector({ supertest, requestHeader = {}, apiKeyHeader = {}, + proxy, }: { supertest: SuperTest.Agent; requestHeader?: Record; apiKeyHeader?: Record; + proxy: LlmProxy; }): Promise<() => Promise> { const config = { apiProvider: 'OpenAI', defaultModel: 'gpt-4', - apiUrl: 'http://localhost:3002', + apiUrl: `http://localhost:${proxy.getPort()}`, }; const connector: { id: string } | undefined = ( @@ -28,7 +31,7 @@ export async function createOpenAIConnector({ .set(requestHeader) .set(apiKeyHeader) .send({ - name: 'test Open AI', + name: 'myConnector', connector_type_id: '.gen-ai', config, secrets: { diff --git a/x-pack/test/functional/page_objects/search_playground_page.ts b/x-pack/test/functional/page_objects/search_playground_page.ts index 35e0a8cc253d3..89bffc8bbd841 100644 --- a/x-pack/test/functional/page_objects/search_playground_page.ts +++ b/x-pack/test/functional/page_objects/search_playground_page.ts @@ -17,6 +17,7 @@ export function SearchPlaygroundPageProvider({ getService }: FtrProviderContext) PlaygroundStartChatPage: { async expectPlaygroundStartChatPageComponentsToExist() { await testSubjects.existOrFail('startChatPage'); + await testSubjects.existOrFail('connectToLLMChatPanel'); await testSubjects.existOrFail('selectIndicesChatPanel'); await testSubjects.existOrFail('startChatButton'); }, @@ -26,8 +27,10 @@ export function SearchPlaygroundPageProvider({ getService }: FtrProviderContext) await testSubjects.existOrFail('playground-documentation-link'); }, - async expectCreateIndexButtonToMissed() { - await testSubjects.missingOrFail('createIndexButton'); + async expectPlaygroundHeaderComponentsToDisabled() { + expect(await testSubjects.isEnabled('editContextActionButton')).to.be(false); + expect(await testSubjects.isEnabled('viewQueryActionButton')).to.be(false); + expect(await testSubjects.isEnabled('viewCodeActionButton')).to.be(false); }, async expectCreateIndexButtonToExists() { @@ -45,8 +48,23 @@ export function SearchPlaygroundPageProvider({ getService }: FtrProviderContext) await comboBox.setCustom('selectIndicesComboBox', indexName); }, - async expectNoIndicesFieldsWarningExists() { - await testSubjects.existOrFail('NoIndicesFieldsMessage'); + async expectIndicesInDropdown() { + await testSubjects.existOrFail('selectIndicesComboBox'); + }, + + async removeIndexFromComboBox() { + await testSubjects.click('removeIndexButton'); + }, + + async expectToSelectIndicesAndStartButtonEnabled(indexName: string) { + await comboBox.setCustom('selectIndicesComboBox', indexName); + expect(await testSubjects.isEnabled('startChatButton')).to.be(true); + expect(await testSubjects.isEnabled('editContextActionButton')).to.be(true); + expect(await testSubjects.isEnabled('viewQueryActionButton')).to.be(true); + expect(await testSubjects.isEnabled('viewCodeActionButton')).to.be(true); + + await testSubjects.click('startChatButton'); + await testSubjects.existOrFail('chatPage'); }, async expectAddConnectorButtonExists() { @@ -58,24 +76,66 @@ export function SearchPlaygroundPageProvider({ getService }: FtrProviderContext) await testSubjects.existOrFail('create-connector-flyout'); }, - async expectHideGenAIPanelConnector(createConnector: () => Promise) { + async expectHideGenAIPanelConnectorAfterCreatingConnector( + createConnector: () => Promise + ) { await createConnector(); await browser.refresh(); await testSubjects.missingOrFail('connectToLLMChatPanel'); }, - async expectToStartChatPage() { - expect(await testSubjects.isEnabled('startChatButton')).to.be(true); - await testSubjects.click('startChatButton'); - await testSubjects.existOrFail('chatPage'); + async expectHideGenAIPanelConnector() { + await testSubjects.missingOrFail('connectToLLMChatPanel'); }, }, PlaygroundChatPage: { - async expectChatWorks() { + async navigateToChatPage(indexName: string) { + await comboBox.setCustom('selectIndicesComboBox', indexName); + await testSubjects.click('startChatButton'); + }, + + async expectChatWindowLoaded() { + expect(await testSubjects.isEnabled('editContextActionButton')).to.be(true); + expect(await testSubjects.isEnabled('viewQueryActionButton')).to.be(true); + expect(await testSubjects.isEnabled('viewCodeActionButton')).to.be(true); + + expect(await testSubjects.isEnabled('regenerateActionButton')).to.be(false); + expect(await testSubjects.isEnabled('clearChatActionButton')).to.be(false); + expect(await testSubjects.isEnabled('sendQuestionButton')).to.be(false); + await testSubjects.existOrFail('questionInput'); + const model = await testSubjects.find('summarizationModelSelect'); + const defaultModel = await model.getVisibleText(); + + expect(defaultModel).to.equal('OpenAI GPT-3.5 Turbo'); + expect(defaultModel).not.to.be.empty(); + + expect( + await (await testSubjects.find('manageConnectorsLink')).getAttribute('href') + ).to.contain('/app/management/insightsAndAlerting/triggersActionsConnectors/connectors/'); + + await testSubjects.click('sourcesAccordion'); + + expect(await testSubjects.findAll('indicesInAccordian')).to.have.length(1); + }, + + async sendQuestion() { await testSubjects.setValue('questionInput', 'test question'); await testSubjects.click('sendQuestionButton'); - await testSubjects.existOrFail('userMessage'); + }, + + async expectChatWorks() { + const userMessageElement = await testSubjects.find('userMessage'); + const userMessage = await userMessageElement.getVisibleText(); + expect(userMessage).to.contain('test question'); + + const assistantMessageElement = await testSubjects.find('assistant-message'); + const assistantMessage = await assistantMessageElement.getVisibleText(); + expect(assistantMessage).to.contain('My response'); + }, + + async expectTokenTooltipExists() { + await testSubjects.existOrFail('token-tooltip-button'); }, async expectOpenViewCode() { diff --git a/x-pack/test_serverless/functional/test_suites/search/index.ts b/x-pack/test_serverless/functional/test_suites/search/index.ts index 8c3cfd83e04e9..79728fb4663e6 100644 --- a/x-pack/test_serverless/functional/test_suites/search/index.ts +++ b/x-pack/test_serverless/functional/test_suites/search/index.ts @@ -20,7 +20,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./advanced_settings')); loadTestFile(require.resolve('./rules/rule_details')); loadTestFile(require.resolve('./console_notebooks')); - loadTestFile(require.resolve('./playground_overview')); + loadTestFile(require.resolve('./search_playground/playground_overview')); loadTestFile(require.resolve('./ml')); loadTestFile(require.resolve('./search_homepage')); diff --git a/x-pack/test_serverless/functional/test_suites/search/playground_overview.ts b/x-pack/test_serverless/functional/test_suites/search/playground_overview.ts deleted file mode 100644 index 40a36362a585e..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/search/playground_overview.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type SuperTest from 'supertest'; -import { testHasEmbeddedConsole } from './embedded_console'; -import { FtrProviderContext } from '../../ftr_provider_context'; -import { RoleCredentials } from '../../../shared/services'; - -const indexName = 'basic_index'; -const esArchiveIndex = 'test/api_integration/fixtures/es_archiver/index_patterns/basic_index'; -async function createOpenAIConnector({ - supertest, - requestHeader = {}, - apiKeyHeader = {}, -}: { - supertest: SuperTest.Agent; - requestHeader?: Record; - apiKeyHeader?: Record; -}): Promise<() => Promise> { - const config = { - apiProvider: 'OpenAI', - defaultModel: 'gpt-4', - apiUrl: 'http://localhost:3002', - }; - - const connector: { id: string } | undefined = ( - await supertest - .post('/api/actions/connector') - .set(requestHeader) - .set(apiKeyHeader) - .send({ - name: 'test Open AI', - connector_type_id: '.gen-ai', - config, - secrets: { - apiKey: 'genAiApiKey', - }, - }) - .expect(200) - ).body; - - return async () => { - if (connector) { - await supertest - .delete(`/api/actions/connector/${connector.id}`) - .set(requestHeader) - .set(apiKeyHeader) - .expect(204); - } - }; -} - -export default function ({ getPageObjects, getService }: FtrProviderContext) { - const pageObjects = getPageObjects(['svlCommonPage', 'svlCommonNavigation', 'searchPlayground']); - const svlCommonApi = getService('svlCommonApi'); - const svlUserManager = getService('svlUserManager'); - const supertestWithoutAuth = getService('supertestWithoutAuth'); - const esArchiver = getService('esArchiver'); - const createIndex = async () => await esArchiver.load(esArchiveIndex); - let roleAuthc: RoleCredentials; - - describe('Serverless Playground Overview', function () { - // see details: https://github.com/elastic/kibana/issues/183893 - this.tags(['failsOnMKI']); - - let removeOpenAIConnector: () => Promise; - let createConnector: () => Promise; - - before(async () => { - await pageObjects.svlCommonPage.login(); - await pageObjects.svlCommonNavigation.sidenav.clickLink({ - deepLinkId: 'searchPlayground', - }); - - const requestHeader = svlCommonApi.getInternalRequestHeader(); - roleAuthc = await svlUserManager.createApiKeyForRole('admin'); - createConnector = async () => { - removeOpenAIConnector = await createOpenAIConnector({ - supertest: supertestWithoutAuth, - requestHeader, - apiKeyHeader: roleAuthc.apiKeyHeader, - }); - }; - }); - - after(async () => { - await removeOpenAIConnector?.(); - await esArchiver.unload(esArchiveIndex); - await svlUserManager.invalidateApiKeyForRole(roleAuthc); - await pageObjects.svlCommonPage.forceLogout(); - }); - - describe('start chat page', () => { - it('playground app is loaded', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundStartChatPageComponentsToExist(); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundHeaderComponentsToExist(); - }); - - it('show no index callout', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectNoIndexCalloutExists(); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectCreateIndexButtonToMissed(); - }); - - it('hide no index callout when index added', async () => { - await createIndex(); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); - }); - - it('show add connector button', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectAddConnectorButtonExists(); - }); - - it('click add connector button opens connector flyout', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectOpenConnectorPagePlayground(); - }); - - it('hide gen ai panel when connector exists', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnector( - createConnector - ); - }); - - it('show chat page', async () => { - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); - await pageObjects.searchPlayground.PlaygroundStartChatPage.expectToStartChatPage(); - }); - }); - - describe('chat page', () => { - it('chat works', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWorks(); - }); - - it('open view code', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectOpenViewCode(); - }); - - it('show fields and code in view query', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectViewQueryHasFields(); - }); - - it('show edit context', async () => { - await pageObjects.searchPlayground.PlaygroundChatPage.expectEditContextOpens(); - }); - }); - - it('has embedded console', async () => { - await testHasEmbeddedConsole(pageObjects); - }); - }); -} diff --git a/x-pack/test_serverless/functional/test_suites/search/search_playground/playground_overview.ts b/x-pack/test_serverless/functional/test_suites/search/search_playground/playground_overview.ts new file mode 100644 index 0000000000000..e114d339ec242 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/search/search_playground/playground_overview.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type OpenAI from 'openai'; +import { testHasEmbeddedConsole } from '../embedded_console'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials } from '../../../../shared/services'; +import { createOpenAIConnector } from './utils/create_openai_connector'; +import { createLlmProxy, LlmProxy } from './utils/create_llm_proxy'; + +const indexName = 'basic_index'; +const esArchiveIndex = 'test/api_integration/fixtures/es_archiver/index_patterns/basic_index'; + +export default function ({ getPageObjects, getService }: FtrProviderContext) { + const pageObjects = getPageObjects(['svlCommonPage', 'svlCommonNavigation', 'searchPlayground']); + const svlCommonApi = getService('svlCommonApi'); + const svlUserManager = getService('svlUserManager'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const esArchiver = getService('esArchiver'); + const log = getService('log'); + const browser = getService('browser'); + const createIndex = async () => await esArchiver.load(esArchiveIndex); + let roleAuthc: RoleCredentials; + + describe('Serverless Playground Overview', function () { + // see details: https://github.com/elastic/kibana/issues/183893 + this.tags(['failsOnMKI']); + + let removeOpenAIConnector: () => Promise; + let createConnector: () => Promise; + let proxy: LlmProxy; + + before(async () => { + proxy = await createLlmProxy(log); + await pageObjects.svlCommonPage.login(); + await pageObjects.svlCommonNavigation.sidenav.clickLink({ + deepLinkId: 'searchPlayground', + }); + + const requestHeader = svlCommonApi.getInternalRequestHeader(); + roleAuthc = await svlUserManager.createApiKeyForRole('admin'); + createConnector = async () => { + removeOpenAIConnector = await createOpenAIConnector({ + supertest: supertestWithoutAuth, + requestHeader, + apiKeyHeader: roleAuthc.apiKeyHeader, + proxy, + }); + }; + }); + + after(async () => { + // await removeOpenAIConnector?.(); + await esArchiver.unload(esArchiveIndex); + proxy.close(); + await svlUserManager.invalidateApiKeyForRole(roleAuthc); + await pageObjects.svlCommonPage.forceLogout(); + }); + + describe('setup Page', () => { + it('is loaded successfully', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundHeaderComponentsToExist(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundHeaderComponentsToDisabled(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectPlaygroundStartChatPageComponentsToExist(); + }); + + describe('with gen ai connectors', () => { + before(async () => { + await createConnector(); + await browser.refresh(); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await browser.refresh(); + }); + it('hide gen ai panel', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnector(); + }); + }); + + describe('without gen ai connectors', () => { + it('should display the set up connectors button', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectAddConnectorButtonExists(); + }); + + it('creates a connector successfully', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectOpenConnectorPagePlayground(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectHideGenAIPanelConnectorAfterCreatingConnector( + createConnector + ); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await browser.refresh(); + }); + }); + + describe('without any indices', () => { + it('show no index callout', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectNoIndexCalloutExists(); + }); + + it('hide no index callout when index added', async () => { + await createIndex(); + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectSelectIndex(indexName); + }); + + after(async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.removeIndexFromComboBox(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); + }); + }); + + describe('with existing indices', () => { + before(async () => { + await createConnector(); + await createIndex(); + await browser.refresh(); + }); + + it('dropdown shows up', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectIndicesInDropdown(); + }); + + it('can select index from dropdown and navigate to chat window', async () => { + await pageObjects.searchPlayground.PlaygroundStartChatPage.expectToSelectIndicesAndStartButtonEnabled( + indexName + ); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); + }); + }); + }); + + describe('chat page', () => { + before(async () => { + await createConnector(); + await createIndex(); + await browser.refresh(); + await pageObjects.searchPlayground.PlaygroundChatPage.navigateToChatPage(indexName); + }); + it('loads successfully', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWindowLoaded(); + }); + + describe('chat', () => { + it('works', async () => { + const conversationInterceptor = proxy.intercept( + 'conversation', + (body) => + (JSON.parse(body) as OpenAI.Chat.ChatCompletionCreateParamsNonStreaming).tools?.find( + (fn) => fn.function.name === 'title_conversation' + ) === undefined + ); + + await pageObjects.searchPlayground.PlaygroundChatPage.sendQuestion(); + + const conversationSimulator = await conversationInterceptor.waitForIntercept(); + + await conversationSimulator.next('My response'); + + await conversationSimulator.complete(); + + await pageObjects.searchPlayground.PlaygroundChatPage.expectChatWorks(); + await pageObjects.searchPlayground.PlaygroundChatPage.expectTokenTooltipExists(); + }); + + it('open view code', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectOpenViewCode(); + }); + + it('show fields and code in view query', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectViewQueryHasFields(); + }); + + it('show edit context', async () => { + await pageObjects.searchPlayground.PlaygroundChatPage.expectEditContextOpens(); + }); + }); + + after(async () => { + await removeOpenAIConnector?.(); + await esArchiver.unload(esArchiveIndex); + await browser.refresh(); + }); + }); + + it('has embedded console', async () => { + await testHasEmbeddedConsole(pageObjects); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_llm_proxy.ts b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_llm_proxy.ts new file mode 100644 index 0000000000000..4952135c3d623 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_llm_proxy.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import getPort from 'get-port'; +import http, { type Server } from 'http'; +import { once, pull } from 'lodash'; +import { createOpenAiChunk } from './create_openai_chunk'; + +type Request = http.IncomingMessage; +type Response = http.ServerResponse & { req: http.IncomingMessage }; + +type RequestHandler = (request: Request, response: Response, body: string) => void; + +interface RequestInterceptor { + name: string; + when: (body: string) => boolean; +} + +export interface LlmResponseSimulator { + body: string; + status: (code: number) => Promise; + next: ( + msg: + | string + | { + content?: string; + function_call?: { name: string; arguments: string }; + } + ) => Promise; + error: (error: any) => Promise; + complete: () => Promise; + rawWrite: (chunk: string) => Promise; + rawEnd: () => Promise; +} + +export class LlmProxy { + server: Server; + + interceptors: Array = []; + + constructor(private readonly port: number, private readonly log: ToolingLog) { + this.server = http + .createServer() + .on('request', async (request, response) => { + this.log.info(`LLM request received`); + + const interceptors = this.interceptors.concat(); + const body = await getRequestBody(request); + + while (interceptors.length) { + const interceptor = interceptors.shift()!; + + if (interceptor.when(body)) { + pull(this.interceptors, interceptor); + interceptor.handle(request, response, body); + return; + } + } + + response.writeHead(500, 'No interceptors found to handle request: ' + request.url); + response.end(); + }) + .on('error', (error) => { + this.log.error(`LLM proxy encountered an error: ${error}`); + }) + .listen(port); + } + + getPort() { + return this.port; + } + + clear() { + this.interceptors.length = 0; + } + + close() { + this.server.close(); + } + + waitForAllInterceptorsSettled() { + return Promise.all(this.interceptors); + } + + intercept< + TResponseChunks extends Array> | string | undefined = undefined + >( + name: string, + when: RequestInterceptor['when'], + responseChunks?: TResponseChunks + ): TResponseChunks extends undefined + ? { + waitForIntercept: () => Promise; + } + : { + complete: () => Promise; + } { + const waitForInterceptPromise = Promise.race([ + new Promise((outerResolve) => { + this.interceptors.push({ + name, + when, + handle: (request, response, body) => { + this.log.info(`LLM request intercepted by "${name}"`); + + function write(chunk: string) { + return new Promise((resolve) => response.write(chunk, () => resolve())); + } + function end() { + return new Promise((resolve) => response.end(resolve)); + } + + const simulator: LlmResponseSimulator = { + body, + status: once(async (status: number) => { + response.writeHead(status, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }); + }), + next: (msg) => { + const chunk = createOpenAiChunk(msg); + return write(`data: ${JSON.stringify(chunk)}\n\n`); + }, + rawWrite: (chunk: string) => { + return write(chunk); + }, + rawEnd: async () => { + await end(); + }, + complete: async () => { + await write('data: [DONE]\n\n'); + await end(); + }, + error: async (error) => { + await write(`data: ${JSON.stringify({ error })}\n\n`); + await end(); + }, + }; + + outerResolve(simulator); + }, + }); + }), + new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Interceptor "${name}" timed out after 5000ms`)), 5000); + }), + ]); + + if (responseChunks === undefined) { + return { waitForIntercept: () => waitForInterceptPromise } as any; + } + + const parsedChunks = Array.isArray(responseChunks) + ? responseChunks + : responseChunks.split(' ').map((token, i) => (i === 0 ? token : ` ${token}`)); + + return { + complete: async () => { + const simulator = await waitForInterceptPromise; + for (const chunk of parsedChunks) { + await simulator.next(chunk); + } + await simulator.complete(); + }, + } as any; + } +} + +export async function createLlmProxy(log: ToolingLog) { + const port = await getPort({ port: getPort.makeRange(9000, 9100) }); + + return new LlmProxy(port, log); +} + +async function getRequestBody(request: http.IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let data = ''; + + request.on('data', (chunk) => { + data += chunk.toString(); + }); + + request.on('close', () => { + resolve(data); + }); + + request.on('error', (error) => { + reject(error); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_chunk.ts b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_chunk.ts new file mode 100644 index 0000000000000..3d7c64537ee5f --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_chunk.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CreateChatCompletionResponseChunk } from '@kbn/observability-ai-assistant-plugin/server/service/client/adapters/process_openai_stream'; +import { v4 } from 'uuid'; + +export function createOpenAiChunk( + msg: string | { content?: string; function_call?: { name: string; arguments?: string } } +): CreateChatCompletionResponseChunk { + msg = typeof msg === 'string' ? { content: msg } : msg; + + return { + id: v4(), + object: 'chat.completion.chunk', + created: 0, + model: 'gpt-4', + choices: [ + { + delta: msg, + index: 0, + finish_reason: null, + }, + ], + }; +} diff --git a/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_connector.ts b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_connector.ts new file mode 100644 index 0000000000000..c2188aec10fb2 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/search/search_playground/utils/create_openai_connector.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; +import { LlmProxy } from './create_llm_proxy'; + +export async function createOpenAIConnector({ + supertest, + requestHeader = {}, + apiKeyHeader = {}, + proxy, +}: { + supertest: SuperTest.Agent; + requestHeader?: Record; + apiKeyHeader?: Record; + proxy: LlmProxy; +}): Promise<() => Promise> { + const config = { + apiProvider: 'OpenAI', + defaultModel: 'gpt-4', + apiUrl: `http://localhost:${proxy.getPort()}`, + }; + + const connector: { id: string } | undefined = ( + await supertest + .post('/api/actions/connector') + .set(requestHeader) + .set(apiKeyHeader) + .send({ + name: 'myConnector', + connector_type_id: '.gen-ai', + config, + secrets: { + apiKey: 'genAiApiKey', + }, + }) + .expect(200) + ).body; + + return async () => { + if (connector) { + await supertest + .delete(`/api/actions/connector/${connector.id}`) + .set(requestHeader) + .set(apiKeyHeader) + .expect(204); + } + }; +} diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index 95b99a8a08f20..ebc11815da754 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -104,5 +104,6 @@ "@kbn/reporting-server", "@kbn/config-schema", "@kbn/features-plugin", + "@kbn/observability-ai-assistant-plugin", ] }