From e34876aa6809a5ea79a836ce61e94192a4769d59 Mon Sep 17 00:00:00 2001 From: Viduni Wickramarachchi Date: Wed, 16 Oct 2024 10:42:09 -0400 Subject: [PATCH] [Obs AI Assistant] Pass function responses when copying conversation (#195635) Closes https://github.com/elastic/kibana/issues/181216 ## Summary ### Problem Function call arguments and responses are serialized separately. Therefore, when a conversation is copied, arguments and responses (`content` and `data`) appear as strings instead of JSON objects. This makes it harder to debug. ### Solution Deserialize the arguments and responses and include it in the copied conversation object. Example of original copied conversation:
Click to expand JSON ```json { "title": "", "messages": [ { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "system", "content": "You are a helpful assistant for Elastic Observability...." } }, { "@timestamp": "2024-10-09T13:58:26.881Z", "message": { "role": "user", "content": "Give me examples of questions I can ask here." } }, { "@timestamp": "2024-10-09T13:58:26.965Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:27.063Z", "message": { "role": "user", "data": "{\"scores\":[],\"suggestions\":[]}", "name": "context", "content": "{\"screen_description\":\"The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\\n\\nThe user is viewing the Overview page which shows a summary of the following apps: {\\\"universal_profiling\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"uptime\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"heartbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_metrics\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"metrics-*,metricbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"alert\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"apm\\\":{\\\"hasData\\\":false,\\\"indices\\\":{\\\"transaction\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"span\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"error\\\":\\\"logs-apm*,apm-*,logs-*.otel-*\\\",\\\"metric\\\":\\\"metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"onboarding\\\":\\\"apm-*\\\",\\\"sourcemap\\\":\\\"apm-*\\\"},\\\"status\\\":\\\"success\\\"},\\\"ux\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_logs\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\\\",\\\"status\\\":\\\"success\\\"}}\",\"learnings\":[]}" } }, { "@timestamp": "2024-10-09T13:58:35.140Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "Sure, here are some examples of questions you can ask:\n\n1. \"What is the average response time for my services?\"\n2. \"Show me the error rate for my services.\"\n3. \"Are there any anomalies in my system?\"\n4. \"What are the top 5 services by transaction volume?\"\n5. \"Show me the logs for a specific service.\"\n6. \"Are there any alerts in my system?\"\n7. \"What is the CPU usage of my hosts?\"\n8. \"Show me the network traffic in my system.\"\n9. \"What is the disk usage of my hosts?\"\n10. \"Show me the memory usage of my containers.\"\n\nPlease note that the actual questions you can ask depend on the data you have in your system." } }, { "@timestamp": "2024-10-09T13:58:41.651Z", "message": { "role": "user", "content": "What are the top 5 services by transaction volume" } }, { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:41.784Z", "message": { "role": "user", "data": "{\"scores\":[],\"suggestions\":[]}", "name": "context", "content": "{\"screen_description\":\"The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\\n\\nThe user is viewing the Overview page which shows a summary of the following apps: {\\\"universal_profiling\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"uptime\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"heartbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_metrics\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"metrics-*,metricbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"alert\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"apm\\\":{\\\"hasData\\\":false,\\\"indices\\\":{\\\"transaction\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"span\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"error\\\":\\\"logs-apm*,apm-*,logs-*.otel-*\\\",\\\"metric\\\":\\\"metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"onboarding\\\":\\\"apm-*\\\",\\\"sourcemap\\\":\\\"apm-*\\\"},\\\"status\\\":\\\"success\\\"},\\\"ux\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_logs\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\\\",\\\"status\\\":\\\"success\\\"}}\",\"learnings\":[]}" } }, { "@timestamp": "2024-10-09T13:58:43.370Z", "message": { "role": "assistant", "function_call": { "name": "get_dataset_info", "arguments": "{\n \"index\": \"traces-apm*,apm-*,traces-*.otel-*\"\n}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:43.379Z", "message": { "role": "user", "name": "get_dataset_info", "content": "{\"indices\":[],\"fields\":[]}" } }, { "@timestamp": "2024-10-09T13:58:43.998Z", "message": { "role": "assistant", "function_call": { "name": "query", "arguments": "{}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:47.572Z", "message": { "role": "user", "data": "{\"keywords\":[\"STATS\",\"SORT\",\"LIMIT\",\"COUNT\"],\"requestedDocumentation\":{\"STATS\":\"# STATS ... BY\\n\\nThe `STATS ... BY` command groups rows based on a common value and calculates one or more aggregated values over these grouped rows.\\n\\n## Syntax\\n\\n```esql\\nSTATS [column1 =] expression1[, ..., [columnN =] expressionN] [BY grouping_expression1[, ..., grouping_expressionN]]\\n```\\n\\n### Parameters\\n\\n#### columnX\\n\\nThe name for the aggregated value in the output. If not provided...."}}", "name": "query", "content": "{}" } }, { "@timestamp": "2024-10-09T13:58:51.481Z", "message": { "role": "assistant", "function_call": { "name": "execute_query", "arguments": "{\"query\":\"FROM traces-apm*,apm-*,traces-*.otel-* | STATS COUNT(*) BY service.name | SORT COUNT(*) DESC | LIMIT 5\"}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:51.619Z", "message": { "role": "user", "name": "execute_query", "content": "{\"message\":\"The query failed to execute\",\"error\":{\"name\":\"ResponseError\",\"message\":\"verification_exception\\n\\tRoot causes:\\n\\t\\tverification_exception: Found 1 problem\\nline 1:1: Unknown index [traces-apm*,apm-*,traces-*.otel-*]\"}}" } }, { "@timestamp": "2024-10-09T13:58:56.600Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "I'm sorry, but it seems like there are no indices matching the pattern \"traces-apm*,apm-*,traces-*.otel-*\" in your system. Therefore, I'm unable to retrieve the top 5 services by transaction volume. Please ensure that your APM data is being ingested correctly into Elasticsearch." } } ] } ```

Copied conversation after deserializing:
Click to expand JSON ```json { "title": "", "messages": [ { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "system", "content": "You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems..." } }, { "@timestamp": "2024-10-09T13:58:26.881Z", "message": { "role": "user", "content": "Give me examples of questions I can ask here." } }, { "@timestamp": "2024-10-09T13:58:26.965Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:27.063Z", "message": { "role": "user", "data": { "scores": [], "suggestions": [] }, "name": "context", "content": { "screen_description": "The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\n\nThe user is viewing the Overview page which shows a summary of the following apps: {\"universal_profiling\":{\"hasData\":false,\"status\":\"success\"},\"uptime\":{\"hasData\":false,\"indices\":\"heartbeat-*\",\"status\":\"success\"},\"infra_metrics\":{\"hasData\":false,\"indices\":\"metrics-*,metricbeat-*\",\"status\":\"success\"},\"alert\":{\"hasData\":false,\"status\":\"success\"},\"apm\":{\"hasData\":false,\"indices\":{\"transaction\":\"traces-apm*,apm-*,traces-*.otel-*\",\"span\":\"traces-apm*,apm-*,traces-*.otel-*\",\"error\":\"logs-apm*,apm-*,logs-*.otel-*\",\"metric\":\"metrics-apm*,apm-*,metrics-*.otel-*\",\"onboarding\":\"apm-*\",\"sourcemap\":\"apm-*\"},\"status\":\"success\"},\"ux\":{\"hasData\":false,\"indices\":\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\",\"status\":\"success\"},\"infra_logs\":{\"hasData\":false,\"indices\":\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\",\"status\":\"success\"}}", "learnings": [] } } }, { "@timestamp": "2024-10-09T13:58:35.140Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "Sure, here are some examples of questions..." } }, { "@timestamp": "2024-10-09T13:58:41.651Z", "message": { "role": "user", "content": "What are the top 5 services by transaction volume" } }, { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:41.784Z", "message": { "role": "user", "data": { "scores": [], "suggestions": [] }, "name": "context", "content": { "screen_description": "The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\n\nThe user is viewing the Overview page which shows a summary of the following apps: {\"universal_profiling\":{\"hasData\":false,\"status\":\"success\"},\"uptime\":{\"hasData\":false,\"indices\":\"heartbeat-*\",\"status\":\"success\"},\"infra_metrics\":{\"hasData\":false,\"indices\":\"metrics-*,metricbeat-*\",\"status\":\"success\"},\"alert\":{\"hasData\":false,\"status\":\"success\"},\"apm\":{\"hasData\":false,\"indices\":{\"transaction\":\"traces-apm*,apm-*,traces-*.otel-*\",\"span\":\"traces-apm*,apm-*,traces-*.otel-*\",\"error\":\"logs-apm*,apm-*,logs-*.otel-*\",\"metric\":\"metrics-apm*,apm-*,metrics-*.otel-*\",\"onboarding\":\"apm-*\",\"sourcemap\":\"apm-*\"},\"status\":\"success\"},\"ux\":{\"hasData\":false,\"indices\":\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\",\"status\":\"success\"},\"infra_logs\":{\"hasData\":false,\"indices\":\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\",\"status\":\"success\"}}", "learnings": [] } } }, { "@timestamp": "2024-10-09T13:58:43.370Z", "message": { "role": "assistant", "function_call": { "name": "get_dataset_info", "arguments": { "index": "traces-apm*,apm-*,traces-*.otel-*" }, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:43.379Z", "message": { "role": "user", "name": "get_dataset_info", "content": { "indices": [], "fields": [] } } }, { "@timestamp": "2024-10-09T13:58:43.998Z", "message": { "role": "assistant", "function_call": { "name": "query", "arguments": {}, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:47.572Z", "message": { "role": "user", "data": { "keywords": ["STATS", "SORT", "LIMIT", "COUNT"], "requestedDocumentation": { "STATS": "# STATS ... BY\n\nThe `STATS ... BY`...", "SORT": "# SORT\n\nThe SORT command...", "LIMIT": "# LIMIT\n\nThe LIMIT command...", "COUNT": "# COUNT\n\n....", "SYNTAX": "# ES|QL Syntax Guide....", "OVERVIEW": "## ES|QL Overview\n\n### ES|QL....", "OPERATORS": "# ES|QL Operators\n\nThis document...." } }, "name": "query", "content": {} } }, { "@timestamp": "2024-10-09T13:58:51.481Z", "message": { "role": "assistant", "function_call": { "name": "execute_query", "arguments": { "query": "FROM traces-apm*,apm-*,traces-*.otel-* | STATS COUNT(*) BY service.name | SORT COUNT(*) DESC | LIMIT 5" }, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:51.619Z", "message": { "role": "user", "name": "execute_query", "content": { "message": "The query failed to execute", "error": { "name": "ResponseError", "message": "verification_exception\n\tRoot causes:\n\t\tverification_exception: Found 1 problem\nline 1:1: Unknown index [traces-apm*,apm-*,traces-*.otel-*]" } } } }, { "@timestamp": "2024-10-09T13:58:56.600Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "I'm sorry, but it seems like there are no indices matching the pattern \"traces-apm*,apm-*,traces-*.otel-*\" in your system. Therefore, I'm unable to retrieve the top 5 services by transaction volume. Please ensure that your APM data is being ingested correctly into Elasticsearch." } } ] } ```
--- .../kbn-ai-assistant/src/chat/chat_body.tsx | 5 +- .../src/utils/deserialize_message.test.ts | 118 ++++++++++++++++++ .../src/utils/deserialize_message.ts | 35 ++++++ 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts create mode 100644 x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts diff --git a/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx b/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx index c3989f6971fff..5b80a34e0bf7b 100644 --- a/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx +++ b/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx @@ -45,6 +45,7 @@ import { SimulatedFunctionCallingCallout } from './simulated_function_calling_ca import { WelcomeMessage } from './welcome_message'; import { useLicense } from '../hooks/use_license'; import { PromptEditor } from '../prompt_editor/prompt_editor'; +import { deserializeMessage } from '../utils/deserialize_message'; const fullHeightClassName = css` height: 100%; @@ -226,9 +227,11 @@ export function ChatBody({ }); const handleCopyConversation = () => { + const deserializedMessages = (conversation.value?.messages ?? messages).map(deserializeMessage); + const content = JSON.stringify({ title: initialTitle, - messages: conversation.value?.messages ?? messages, + messages: deserializedMessages, }); navigator.clipboard?.writeText(content || ''); diff --git a/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts new file mode 100644 index 0000000000000..b2c067a3e9f10 --- /dev/null +++ b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; +import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; +import { deserializeMessage } from './deserialize_message'; +import { safeJsonParse } from './safe_json_parse'; + +jest.mock('lodash', () => ({ + cloneDeep: jest.fn(), +})); + +jest.mock('./safe_json_parse', () => ({ + safeJsonParse: jest.fn((value) => { + try { + return JSON.parse(value); + } catch { + return value; + } + }), +})); + +describe('deserializeMessage', () => { + const baseMessage: Message = { + '@timestamp': '2024-10-15T00:00:00Z', + message: { + role: MessageRole.User, + content: 'This is a message', + }, + }; + + beforeEach(() => { + (cloneDeep as jest.Mock).mockImplementation((obj) => JSON.parse(JSON.stringify(obj))); + }); + + it('should clone the original message', () => { + const message = { ...baseMessage }; + deserializeMessage(message); + + expect(cloneDeep).toHaveBeenCalledWith(message); + }); + + it('should deserialize function_call.arguments if it is a string', () => { + const messageWithFunctionCall: Message = { + ...baseMessage, + message: { + ...baseMessage.message, + function_call: { + name: 'testFunction', + arguments: '{"key": "value"}', + trigger: MessageRole.Assistant, + }, + }, + }; + + const result = deserializeMessage(messageWithFunctionCall); + + expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}'); + expect(result.message.function_call!.arguments).toEqual({ key: 'value' }); + }); + + it('should deserialize message.content if it is a string', () => { + const messageWithContent: Message = { + ...baseMessage, + message: { + ...baseMessage.message, + name: 'testMessage', + content: '{"key": "value"}', + }, + }; + + const result = deserializeMessage(messageWithContent); + + expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}'); + expect(result.message.content).toEqual({ key: 'value' }); + }); + + it('should deserialize message.data if it is a string', () => { + const messageWithData: Message = { + ...baseMessage, + message: { + ...baseMessage.message, + name: 'testMessage', + data: '{"key": "value"}', + }, + }; + + const result = deserializeMessage(messageWithData); + + expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}'); + expect(result.message.data).toEqual({ key: 'value' }); + }); + + it('should return the copied message as is if no deserialization is needed', () => { + const messageWithoutSerialization: Message = { + ...baseMessage, + message: { + ...baseMessage.message, + function_call: { + name: 'testFunction', + arguments: '', + trigger: MessageRole.Assistant, + }, + content: '', + }, + }; + + const result = deserializeMessage(messageWithoutSerialization); + + expect(result.message.function_call!.name).toEqual('testFunction'); + expect(result.message.function_call!.arguments).toEqual(''); + expect(result.message.content).toEqual(''); + }); +}); diff --git a/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts new file mode 100644 index 0000000000000..445e6330981a9 --- /dev/null +++ b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; +import type { Message } from '@kbn/observability-ai-assistant-plugin/common'; +import { safeJsonParse } from './safe_json_parse'; + +export const deserializeMessage = (message: Message): Message => { + const copiedMessage = cloneDeep(message); + + if ( + copiedMessage.message.function_call?.arguments && + typeof copiedMessage.message.function_call?.arguments === 'string' + ) { + copiedMessage.message.function_call.arguments = safeJsonParse( + copiedMessage.message.function_call.arguments ?? '{}' + ); + } + + if (copiedMessage.message.name) { + if (copiedMessage.message.content && typeof copiedMessage.message.content === 'string') { + copiedMessage.message.content = safeJsonParse(copiedMessage.message.content); + } + + if (copiedMessage.message.data && typeof copiedMessage.message.data === 'string') { + copiedMessage.message.data = safeJsonParse(copiedMessage.message.data); + } + } + + return copiedMessage; +};