diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/connect/_components/ConnectEvaluationModal/List/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/connect/_components/ConnectEvaluationModal/List/index.tsx index 91507bd4d..841f46b7b 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/connect/_components/ConnectEvaluationModal/List/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/connect/_components/ConnectEvaluationModal/List/index.tsx @@ -23,7 +23,7 @@ export default function EvaluationList({ onSearchChange, }: EvaluationListProps) { return ( -
+
{ const documents = result.unwrap() expect(documents).toHaveLength(2) - const paths = documents.map((d) => d.path).sort() - expect(paths).toEqual(['bar', 'foo']) + expect(new Set(documents.map((d) => d.path))).toEqual( + new Set(['foo', 'bar']), + ) expect(documents.every((d) => d.commitId === commit1Id)).toBe(true) }) diff --git a/packages/core/src/services/chains/run.test.ts b/packages/core/src/services/chains/run.test.ts index bee346ec8..71e5f2105 100644 --- a/packages/core/src/services/chains/run.test.ts +++ b/packages/core/src/services/chains/run.test.ts @@ -2,7 +2,7 @@ import { Chain, ContentType, MessageRole } from '@latitude-data/compiler' import { v4 as uuid } from 'uuid' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { Workspace } from '../../browser' +import { objectToString, Workspace } from '../../browser' import { LogSources, Providers } from '../../constants' import * as factories from '../../tests/factories' import * as aiModule from '../ai' @@ -159,7 +159,7 @@ describe('runChain', () => { expect.objectContaining({ documentLogUuid: expect.any(String), object: { name: 'John', age: 30 }, - text: '{"name":"John","age":30}', + text: objectToString({ name: 'John', age: 30 }), usage: { totalTokens: 15 }, }), ) @@ -377,7 +377,7 @@ describe('runChain', () => { expect.objectContaining({ documentLogUuid: expect.any(String), object: { name: 'John', age: 30 }, - text: '{"name":"John","age":30}', + text: objectToString({ name: 'John', age: 30 }), usage: { totalTokens: 15 }, }), ) @@ -463,7 +463,10 @@ describe('runChain', () => { { name: 'John', age: 30 }, { name: 'Jane', age: 25 }, ], - text: '[{"name":"John","age":30},{"name":"Jane","age":25}]', + text: objectToString([ + { name: 'John', age: 30 }, + { name: 'Jane', age: 25 }, + ]), usage: { totalTokens: 20 }, }), ) diff --git a/packages/core/src/services/providerLogs/formatForEvaluation.test.ts b/packages/core/src/services/providerLogs/formatForEvaluation.test.ts new file mode 100644 index 000000000..c6c493d81 --- /dev/null +++ b/packages/core/src/services/providerLogs/formatForEvaluation.test.ts @@ -0,0 +1,423 @@ +import { ContentType, MessageRole } from '@latitude-data/compiler' +import { describe, expect, it } from 'vitest' + +import { ProviderLog, ProviderLogDto } from '../../browser' +import { formatContext, formatConversation } from './formatForEvaluation' + +describe('formatConversation', () => { + it('should format a ProviderLogDto with response correctly', () => { + // @ts-expect-error + const providerLogDto: ProviderLogDto = { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + { role: MessageRole.assistant, content: 'Hi there', toolCalls: [] }, + ], + response: 'How can I help you?', + toolCalls: [], + } + + const result = formatConversation(providerLogDto) + + expect(result).toEqual({ + all: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + { role: MessageRole.assistant, content: 'Hi there', toolCalls: [] }, + { + role: MessageRole.assistant, + content: 'How can I help you?', + toolCalls: [], + }, + ], + first: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + last: { + role: MessageRole.assistant, + content: 'How can I help you?', + toolCalls: [], + }, + user: { + all: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + ], + first: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + last: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Hello' }], + }, + }, + system: { + all: [], + first: null, + last: null, + }, + assistant: { + all: [ + { role: MessageRole.assistant, content: 'Hi there', toolCalls: [] }, + { + role: MessageRole.assistant, + content: 'How can I help you?', + toolCalls: [], + }, + ], + first: { + role: MessageRole.assistant, + content: 'Hi there', + toolCalls: [], + }, + last: { + role: MessageRole.assistant, + content: 'How can I help you?', + toolCalls: [], + }, + }, + }) + }) + + it('should format a ProviderLog with responseText correctly', () => { + // @ts-expect-error + const providerLog: ProviderLog = { + messages: [ + { role: MessageRole.system, content: 'You are an AI assistant' }, + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + ], + responseText: 'The weather is sunny today.', + toolCalls: [], + } + + const result = formatConversation(providerLog) + + expect(result).toEqual({ + all: [ + { + role: MessageRole.system, + content: 'You are an AI assistant', + }, + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + ], + first: { + role: MessageRole.system, + content: 'You are an AI assistant', + }, + last: { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + user: { + all: [ + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + ], + first: { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + last: { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + }, + system: { + all: [ + { + role: MessageRole.system, + content: 'You are an AI assistant', + }, + ], + first: { + role: MessageRole.system, + content: 'You are an AI assistant', + }, + last: { + role: MessageRole.system, + content: 'You are an AI assistant', + }, + }, + assistant: { + all: [ + { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + ], + first: { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + last: { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + }, + }) + }) + + it('should format a ProviderLog with responseObject correctly', () => { + const obj = { key: 'value', number: 42 } + const objStr = JSON.stringify(obj, null, 2) + const providerLog: ProviderLog = { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Give me a JSON object' }], + }, + ], + // @ts-expect-error + responseObject: obj, + toolCalls: [], + } + + const result = formatConversation(providerLog) + + expect(result).toEqual({ + all: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Give me a JSON object' }], + }, + { + role: MessageRole.assistant, + content: objStr, + toolCalls: [], + }, + ], + first: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Give me a JSON object' }], + }, + last: { + role: MessageRole.assistant, + content: objStr, + toolCalls: [], + }, + user: { + all: [ + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: 'Give me a JSON object' }, + ], + }, + ], + first: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Give me a JSON object' }], + }, + last: { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Give me a JSON object' }], + }, + }, + system: { + all: [], + first: null, + last: null, + }, + assistant: { + all: [ + { + role: MessageRole.assistant, + content: objStr, + toolCalls: [], + }, + ], + first: { + role: MessageRole.assistant, + content: objStr, + toolCalls: [], + }, + last: { + role: MessageRole.assistant, + content: objStr, + toolCalls: [], + }, + }, + }) + }) + + it('should handle empty messages array', () => { + // @ts-expect-error + const providerLog: ProviderLog = { + messages: [], + responseText: 'Hello!', + toolCalls: [], + } + + const result = formatConversation(providerLog) + + expect(result).toEqual({ + all: [{ role: MessageRole.assistant, content: 'Hello!', toolCalls: [] }], + first: { role: MessageRole.assistant, content: 'Hello!', toolCalls: [] }, + last: { role: MessageRole.assistant, content: 'Hello!', toolCalls: [] }, + user: { + all: [], + first: null, + last: null, + }, + system: { + all: [], + first: null, + last: null, + }, + assistant: { + all: [ + { role: MessageRole.assistant, content: 'Hello!', toolCalls: [] }, + ], + first: { + role: MessageRole.assistant, + content: 'Hello!', + toolCalls: [], + }, + last: { role: MessageRole.assistant, content: 'Hello!', toolCalls: [] }, + }, + }) + }) +}) + +describe('formatContext', () => { + it('should format a conversation with text messages correctly', () => { + // @ts-expect-error + const providerLog: ProviderLog = { + messages: [ + { role: MessageRole.system, content: 'You are an AI assistant' }, + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: "What's the weather like?" }, + ], + }, + { + role: MessageRole.assistant, + content: 'The weather is sunny today.', + toolCalls: [], + }, + ], + responseText: 'Is there anything else I can help you with?', + toolCalls: [], + } + + const result = formatContext(providerLog) + + expect(result).toBe( + 'System:\nYou are an AI assistant\n\n' + + "User:\nWhat's the weather like?\n\n" + + 'Assistant:\nThe weather is sunny today.', + ) + }) + + it('should handle image content in messages', () => { + // @ts-expect-error + const providerLog: ProviderLog = { + messages: [ + { + role: MessageRole.user, + content: [ + { type: ContentType.text, text: 'What can you see in this image?' }, + { + type: ContentType.image, + image: 'https://example.com/image.jpg', + }, + ], + }, + { + role: MessageRole.assistant, + content: 'I see a beautiful landscape.', + toolCalls: [], + }, + ], + responseText: 'Would you like me to describe it in more detail?', + toolCalls: [], + } + + const result = formatContext(providerLog) + + expect(result).toBe( + 'User:\nWhat can you see in this image?\n\n' + + 'Assistant:\nI see a beautiful landscape.', + ) + }) + + it('should handle empty messages array', () => { + // @ts-expect-error + const providerLog: ProviderLog = { + messages: [], + responseText: 'Hello! How can I assist you today?', + toolCalls: [], + } + + const result = formatContext(providerLog) + + expect(result).toBe('') + }) + + it('should handle messages with string content', () => { + // @ts-expect-error + const providerLogDto: ProviderLogDto = { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Tell me a joke' }], + }, + { + role: MessageRole.assistant, + content: 'Why did the chicken cross the road?', + toolCalls: [], + }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: "I don't know, why?" }], + }, + ], + response: 'To get to the other side!', + toolCalls: [], + } + + const result = formatContext(providerLogDto) + + expect(result).toBe( + 'User:\nTell me a joke\n\n' + + 'Assistant:\nWhy did the chicken cross the road?\n\n' + + "User:\nI don't know, why?", + ) + }) +}) diff --git a/packages/core/src/services/providerLogs/formatForEvaluation.ts b/packages/core/src/services/providerLogs/formatForEvaluation.ts index a1defb37a..a3e08397a 100644 --- a/packages/core/src/services/providerLogs/formatForEvaluation.ts +++ b/packages/core/src/services/providerLogs/formatForEvaluation.ts @@ -33,8 +33,33 @@ export function formatConversation(providerLog: ProviderLogDto | ProviderLog) { return formatMessages(messages) } -export function formatContext(providerLog: ProviderLog | ProviderLogDto) { - return formatMessages(providerLog.messages) +export function formatContext( + providerLog: ProviderLog | ProviderLogDto, +): string { + const messages = providerLog.messages || [] + let formattedConversation = '' + + messages.forEach((message) => { + const speaker = message.role.charAt(0).toUpperCase() + message.role.slice(1) + let content = '' + if (typeof message.content === 'string') { + content = message.content + } else if ( + Array.isArray(message.content) && + 'text' in message.content[0]! + ) { + content = message.content[0].text + } else if ( + Array.isArray(message.content) && + 'image' in message.content[0]! + ) { + content = '[IMAGE]' + } + + formattedConversation += `${speaker}:\n${content}\n\n` + }) + + return formattedConversation.trim() } function formatMessages(messages: Message[]) {