[Obs AI Assistant] Pass function responses when copying conversation (e…

…lastic#195635) Closes elastic#181216 ## Summary ### Problem Function call arguments and responses are serialized separately. Therefore, when a conversation is copied, arguments and responses (`content` and `data`) appear as strings instead of JSON objects. This makes it harder to debug. ### Solution Deserialize the arguments and responses and include it in the copied conversation object. Example of original copied conversation: <details> <summary>Click to expand JSON</summary> ```json { "title": "", "messages": [ { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "system", "content": "You are a helpful assistant for Elastic Observability...." } }, { "@timestamp": "2024-10-09T13:58:26.881Z", "message": { "role": "user", "content": "Give me examples of questions I can ask here." } }, { "@timestamp": "2024-10-09T13:58:26.965Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:27.063Z", "message": { "role": "user", "data": "{\"scores\":[],\"suggestions\":[]}", "name": "context", "content": "{\"screen_description\":\"The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\\n\\nThe user is viewing the Overview page which shows a summary of the following apps: {\\\"universal_profiling\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"uptime\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"heartbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_metrics\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"metrics-*,metricbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"alert\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"apm\\\":{\\\"hasData\\\":false,\\\"indices\\\":{\\\"transaction\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"span\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"error\\\":\\\"logs-apm*,apm-*,logs-*.otel-*\\\",\\\"metric\\\":\\\"metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"onboarding\\\":\\\"apm-*\\\",\\\"sourcemap\\\":\\\"apm-*\\\"},\\\"status\\\":\\\"success\\\"},\\\"ux\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_logs\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\\\",\\\"status\\\":\\\"success\\\"}}\",\"learnings\":[]}" } }, { "@timestamp": "2024-10-09T13:58:35.140Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "Sure, here are some examples of questions you can ask:\n\n1. \"What is the average response time for my services?\"\n2. \"Show me the error rate for my services.\"\n3. \"Are there any anomalies in my system?\"\n4. \"What are the top 5 services by transaction volume?\"\n5. \"Show me the logs for a specific service.\"\n6. \"Are there any alerts in my system?\"\n7. \"What is the CPU usage of my hosts?\"\n8. \"Show me the network traffic in my system.\"\n9. \"What is the disk usage of my hosts?\"\n10. \"Show me the memory usage of my containers.\"\n\nPlease note that the actual questions you can ask depend on the data you have in your system." } }, { "@timestamp": "2024-10-09T13:58:41.651Z", "message": { "role": "user", "content": "What are the top 5 services by transaction volume" } }, { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:41.784Z", "message": { "role": "user", "data": "{\"scores\":[],\"suggestions\":[]}", "name": "context", "content": "{\"screen_description\":\"The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\\n\\nThe user is viewing the Overview page which shows a summary of the following apps: {\\\"universal_profiling\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"uptime\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"heartbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_metrics\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"metrics-*,metricbeat-*\\\",\\\"status\\\":\\\"success\\\"},\\\"alert\\\":{\\\"hasData\\\":false,\\\"status\\\":\\\"success\\\"},\\\"apm\\\":{\\\"hasData\\\":false,\\\"indices\\\":{\\\"transaction\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"span\\\":\\\"traces-apm*,apm-*,traces-*.otel-*\\\",\\\"error\\\":\\\"logs-apm*,apm-*,logs-*.otel-*\\\",\\\"metric\\\":\\\"metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"onboarding\\\":\\\"apm-*\\\",\\\"sourcemap\\\":\\\"apm-*\\\"},\\\"status\\\":\\\"success\\\"},\\\"ux\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\\\",\\\"status\\\":\\\"success\\\"},\\\"infra_logs\\\":{\\\"hasData\\\":false,\\\"indices\\\":\\\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\\\",\\\"status\\\":\\\"success\\\"}}\",\"learnings\":[]}" } }, { "@timestamp": "2024-10-09T13:58:43.370Z", "message": { "role": "assistant", "function_call": { "name": "get_dataset_info", "arguments": "{\n \"index\": \"traces-apm*,apm-*,traces-*.otel-*\"\n}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:43.379Z", "message": { "role": "user", "name": "get_dataset_info", "content": "{\"indices\":[],\"fields\":[]}" } }, { "@timestamp": "2024-10-09T13:58:43.998Z", "message": { "role": "assistant", "function_call": { "name": "query", "arguments": "{}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:47.572Z", "message": { "role": "user", "data": "{\"keywords\":[\"STATS\",\"SORT\",\"LIMIT\",\"COUNT\"],\"requestedDocumentation\":{\"STATS\":\"# STATS ... BY\\n\\nThe `STATS ... BY` command groups rows based on a common value and calculates one or more aggregated values over these grouped rows.\\n\\n## Syntax\\n\\n```esql\\nSTATS [column1 =] expression1[, ..., [columnN =] expressionN] [BY grouping_expression1[, ..., grouping_expressionN]]\\n```\\n\\n### Parameters\\n\\n#### columnX\\n\\nThe name for the aggregated value in the output. If not provided...."}}", "name": "query", "content": "{}" } }, { "@timestamp": "2024-10-09T13:58:51.481Z", "message": { "role": "assistant", "function_call": { "name": "execute_query", "arguments": "{\"query\":\"FROM traces-apm*,apm-*,traces-*.otel-* | STATS COUNT(*) BY service.name | SORT COUNT(*) DESC | LIMIT 5\"}", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:51.619Z", "message": { "role": "user", "name": "execute_query", "content": "{\"message\":\"The query failed to execute\",\"error\":{\"name\":\"ResponseError\",\"message\":\"verification_exception\\n\\tRoot causes:\\n\\t\\tverification_exception: Found 1 problem\\nline 1:1: Unknown index [traces-apm*,apm-*,traces-*.otel-*]\"}}" } }, { "@timestamp": "2024-10-09T13:58:56.600Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "I'm sorry, but it seems like there are no indices matching the pattern \"traces-apm*,apm-*,traces-*.otel-*\" in your system. Therefore, I'm unable to retrieve the top 5 services by transaction volume. Please ensure that your APM data is being ingested correctly into Elasticsearch." } } ] } ``` </details> <br/> Copied conversation after deserializing: <details> <summary>Click to expand JSON</summary> ```json { "title": "", "messages": [ { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "system", "content": "You are a helpful assistant for Elastic Observability. Your goal is to help the Elastic Observability users to quickly assess what is happening in their observed systems..." } }, { "@timestamp": "2024-10-09T13:58:26.881Z", "message": { "role": "user", "content": "Give me examples of questions I can ask here." } }, { "@timestamp": "2024-10-09T13:58:26.965Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:27.063Z", "message": { "role": "user", "data": { "scores": [], "suggestions": [] }, "name": "context", "content": { "screen_description": "The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\n\nThe user is viewing the Overview page which shows a summary of the following apps: {\"universal_profiling\":{\"hasData\":false,\"status\":\"success\"},\"uptime\":{\"hasData\":false,\"indices\":\"heartbeat-*\",\"status\":\"success\"},\"infra_metrics\":{\"hasData\":false,\"indices\":\"metrics-*,metricbeat-*\",\"status\":\"success\"},\"alert\":{\"hasData\":false,\"status\":\"success\"},\"apm\":{\"hasData\":false,\"indices\":{\"transaction\":\"traces-apm*,apm-*,traces-*.otel-*\",\"span\":\"traces-apm*,apm-*,traces-*.otel-*\",\"error\":\"logs-apm*,apm-*,logs-*.otel-*\",\"metric\":\"metrics-apm*,apm-*,metrics-*.otel-*\",\"onboarding\":\"apm-*\",\"sourcemap\":\"apm-*\"},\"status\":\"success\"},\"ux\":{\"hasData\":false,\"indices\":\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\",\"status\":\"success\"},\"infra_logs\":{\"hasData\":false,\"indices\":\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\",\"status\":\"success\"}}", "learnings": [] } } }, { "@timestamp": "2024-10-09T13:58:35.140Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "Sure, here are some examples of questions..." } }, { "@timestamp": "2024-10-09T13:58:41.651Z", "message": { "role": "user", "content": "What are the top 5 services by transaction volume" } }, { "@timestamp": "2024-10-09T13:58:41.723Z", "message": { "role": "assistant", "function_call": { "name": "context", "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:41.784Z", "message": { "role": "user", "data": { "scores": [], "suggestions": [] }, "name": "context", "content": { "screen_description": "The user is looking at http://localhost:5601/kyq/app/observability/overview?rangeFrom=now-15m&rangeTo=now. The current time range is 2024-10-09T13:40:00.288Z - 2024-10-09T13:55:00.288Z.\n\nThe user is viewing the Overview page which shows a summary of the following apps: {\"universal_profiling\":{\"hasData\":false,\"status\":\"success\"},\"uptime\":{\"hasData\":false,\"indices\":\"heartbeat-*\",\"status\":\"success\"},\"infra_metrics\":{\"hasData\":false,\"indices\":\"metrics-*,metricbeat-*\",\"status\":\"success\"},\"alert\":{\"hasData\":false,\"status\":\"success\"},\"apm\":{\"hasData\":false,\"indices\":{\"transaction\":\"traces-apm*,apm-*,traces-*.otel-*\",\"span\":\"traces-apm*,apm-*,traces-*.otel-*\",\"error\":\"logs-apm*,apm-*,logs-*.otel-*\",\"metric\":\"metrics-apm*,apm-*,metrics-*.otel-*\",\"onboarding\":\"apm-*\",\"sourcemap\":\"apm-*\"},\"status\":\"success\"},\"ux\":{\"hasData\":false,\"indices\":\"traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*\",\"status\":\"success\"},\"infra_logs\":{\"hasData\":false,\"indices\":\"logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*\",\"status\":\"success\"}}", "learnings": [] } } }, { "@timestamp": "2024-10-09T13:58:43.370Z", "message": { "role": "assistant", "function_call": { "name": "get_dataset_info", "arguments": { "index": "traces-apm*,apm-*,traces-*.otel-*" }, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:43.379Z", "message": { "role": "user", "name": "get_dataset_info", "content": { "indices": [], "fields": [] } } }, { "@timestamp": "2024-10-09T13:58:43.998Z", "message": { "role": "assistant", "function_call": { "name": "query", "arguments": {}, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:47.572Z", "message": { "role": "user", "data": { "keywords": ["STATS", "SORT", "LIMIT", "COUNT"], "requestedDocumentation": { "STATS": "# STATS ... BY\n\nThe `STATS ... BY`...", "SORT": "# SORT\n\nThe SORT command...", "LIMIT": "# LIMIT\n\nThe LIMIT command...", "COUNT": "# COUNT\n\n....", "SYNTAX": "# ES|QL Syntax Guide....", "OVERVIEW": "## ES|QL Overview\n\n### ES|QL....", "OPERATORS": "# ES|QL Operators\n\nThis document...." } }, "name": "query", "content": {} } }, { "@timestamp": "2024-10-09T13:58:51.481Z", "message": { "role": "assistant", "function_call": { "name": "execute_query", "arguments": { "query": "FROM traces-apm*,apm-*,traces-*.otel-* | STATS COUNT(*) BY service.name | SORT COUNT(*) DESC | LIMIT 5" }, "trigger": "assistant" }, "content": "" } }, { "@timestamp": "2024-10-09T13:58:51.619Z", "message": { "role": "user", "name": "execute_query", "content": { "message": "The query failed to execute", "error": { "name": "ResponseError", "message": "verification_exception\n\tRoot causes:\n\t\tverification_exception: Found 1 problem\nline 1:1: Unknown index [traces-apm*,apm-*,traces-*.otel-*]" } } } }, { "@timestamp": "2024-10-09T13:58:56.600Z", "message": { "role": "assistant", "function_call": { "name": "", "arguments": "", "trigger": "assistant" }, "content": "I'm sorry, but it seems like there are no indices matching the pattern \"traces-apm*,apm-*,traces-*.otel-*\" in your system. Therefore, I'm unable to retrieve the top 5 services by transaction volume. Please ensure that your APM data is being ingested correctly into Elasticsearch." } } ] } ``` </details>
ashokaditya · Oct 16, 2024 · e34876a · e34876a
1 parent 241a05a
commit e34876a
Show file tree

Hide file tree

Showing 3 changed files with 157 additions and 1 deletion.
diff --git a/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx b/x-pack/packages/kbn-ai-assistant/src/chat/chat_body.tsx
@@ -45,6 +45,7 @@ import { SimulatedFunctionCallingCallout } from './simulated_function_calling_ca
 import { WelcomeMessage } from './welcome_message';
 import { useLicense } from '../hooks/use_license';
 import { PromptEditor } from '../prompt_editor/prompt_editor';
+import { deserializeMessage } from '../utils/deserialize_message';
 
 const fullHeightClassName = css`
   height: 100%;
@@ -226,9 +227,11 @@ export function ChatBody({
   });
 
   const handleCopyConversation = () => {
+    const deserializedMessages = (conversation.value?.messages ?? messages).map(deserializeMessage);
+
     const content = JSON.stringify({
       title: initialTitle,
-      messages: conversation.value?.messages ?? messages,
+      messages: deserializedMessages,
     });
 
     navigator.clipboard?.writeText(content || '');

diff --git a/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.test.ts
@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common';
+import { deserializeMessage } from './deserialize_message';
+import { safeJsonParse } from './safe_json_parse';
+
+jest.mock('lodash', () => ({
+  cloneDeep: jest.fn(),
+}));
+
+jest.mock('./safe_json_parse', () => ({
+  safeJsonParse: jest.fn((value) => {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return value;
+    }
+  }),
+}));
+
+describe('deserializeMessage', () => {
+  const baseMessage: Message = {
+    '@timestamp': '2024-10-15T00:00:00Z',
+    message: {
+      role: MessageRole.User,
+      content: 'This is a message',
+    },
+  };
+
+  beforeEach(() => {
+    (cloneDeep as jest.Mock).mockImplementation((obj) => JSON.parse(JSON.stringify(obj)));
+  });
+
+  it('should clone the original message', () => {
+    const message = { ...baseMessage };
+    deserializeMessage(message);
+
+    expect(cloneDeep).toHaveBeenCalledWith(message);
+  });
+
+  it('should deserialize function_call.arguments if it is a string', () => {
+    const messageWithFunctionCall: Message = {
+      ...baseMessage,
+      message: {
+        ...baseMessage.message,
+        function_call: {
+          name: 'testFunction',
+          arguments: '{"key": "value"}',
+          trigger: MessageRole.Assistant,
+        },
+      },
+    };
+
+    const result = deserializeMessage(messageWithFunctionCall);
+
+    expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}');
+    expect(result.message.function_call!.arguments).toEqual({ key: 'value' });
+  });
+
+  it('should deserialize message.content if it is a string', () => {
+    const messageWithContent: Message = {
+      ...baseMessage,
+      message: {
+        ...baseMessage.message,
+        name: 'testMessage',
+        content: '{"key": "value"}',
+      },
+    };
+
+    const result = deserializeMessage(messageWithContent);
+
+    expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}');
+    expect(result.message.content).toEqual({ key: 'value' });
+  });
+
+  it('should deserialize message.data if it is a string', () => {
+    const messageWithData: Message = {
+      ...baseMessage,
+      message: {
+        ...baseMessage.message,
+        name: 'testMessage',
+        data: '{"key": "value"}',
+      },
+    };
+
+    const result = deserializeMessage(messageWithData);
+
+    expect(safeJsonParse).toHaveBeenCalledWith('{"key": "value"}');
+    expect(result.message.data).toEqual({ key: 'value' });
+  });
+
+  it('should return the copied message as is if no deserialization is needed', () => {
+    const messageWithoutSerialization: Message = {
+      ...baseMessage,
+      message: {
+        ...baseMessage.message,
+        function_call: {
+          name: 'testFunction',
+          arguments: '',
+          trigger: MessageRole.Assistant,
+        },
+        content: '',
+      },
+    };
+
+    const result = deserializeMessage(messageWithoutSerialization);
+
+    expect(result.message.function_call!.name).toEqual('testFunction');
+    expect(result.message.function_call!.arguments).toEqual('');
+    expect(result.message.content).toEqual('');
+  });
+});
diff --git a/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts b/x-pack/packages/kbn-ai-assistant/src/utils/deserialize_message.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+import type { Message } from '@kbn/observability-ai-assistant-plugin/common';
+import { safeJsonParse } from './safe_json_parse';
+
+export const deserializeMessage = (message: Message): Message => {
+  const copiedMessage = cloneDeep(message);
+
+  if (
+    copiedMessage.message.function_call?.arguments &&
+    typeof copiedMessage.message.function_call?.arguments === 'string'
+  ) {
+    copiedMessage.message.function_call.arguments = safeJsonParse(
+      copiedMessage.message.function_call.arguments ?? '{}'
+    );
+  }
+
+  if (copiedMessage.message.name) {
+    if (copiedMessage.message.content && typeof copiedMessage.message.content === 'string') {
+      copiedMessage.message.content = safeJsonParse(copiedMessage.message.content);
+    }
+
+    if (copiedMessage.message.data && typeof copiedMessage.message.data === 'string') {
+      copiedMessage.message.data = safeJsonParse(copiedMessage.message.data);
+    }
+  }
+
+  return copiedMessage;
+};