feature: batch evaluations backend #2

This commit implements the full cycle of a batch evaluation. Now users
can run batch evaluations on top of their prompts with dataset data.

apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts

@@ -2,7 +2,6 @@ import { zValidator } from '@hono/zod-validator'
 import { LogSources } from '@latitude-data/core/browser'
 import { runDocumentAtCommit } from '@latitude-data/core/services/commits/runDocumentAtCommit'
 import { pipeToStream } from '$/common/pipeToStream'
-import { queues } from '$/jobs'
 import { Factory } from 'hono/factory'
 import { streamSSE } from 'hono/streaming'
 import { z } from 'zod'
@@ -21,8 +20,6 @@ export const runHandler = factory.createHandlers(
   zValidator('json', runSchema),
   async (c) => {
     return streamSSE(c, async (stream) => {
-      const startTime = Date.now()
-
       const { projectId, commitUuid } = c.req.param()
       const { documentPath, parameters, source } = c.req.valid('json')
 
@@ -44,17 +41,6 @@ export const runHandler = factory.createHandlers(
       }).then((r) => r.unwrap())
 
       await pipeToStream(stream, result.stream)
-
-      queues.defaultQueue.jobs.enqueueCreateDocumentLogJob({
-        commit,
-        data: {
-          uuid: result.documentLogUuid,
-          documentUuid: document.documentUuid,
-          resolvedContent: result.resolvedContent,
-          parameters,
-          duration: Date.now() - startTime,
-        },
-      })
     })
   },
 )

apps/web/src/actions/evaluations/runBatch.test.ts

@@ -10,7 +10,7 @@ import {
 import * as factories from '@latitude-data/core/factories'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-import { runBatchAction } from './runBatch'
+import { runBatchEvaluationAction } from './runBatch'
 
 const mocks = vi.hoisted(() => ({
   getSession: vi.fn(),
@@ -40,7 +40,7 @@ vi.mock('@latitude-data/jobs', () => ({
 describe('runBatchAction', () => {
   describe('unauthorized', () => {
     it('errors when the user is not authenticated', async () => {
-      const [_, error] = await runBatchAction({
+      const [_, error] = await runBatchEvaluationAction({
         datasetId: 1,
         projectId: 1,
         documentUuid: 'doc-uuid',
@@ -95,7 +95,7 @@ describe('runBatchAction', () => {
     })
 
     it('successfully enqueues a batch evaluation job', async () => {
-      const [result, error] = await runBatchAction({
+      const [result, error] = await runBatchEvaluationAction({
         datasetId: dataset.id,
         projectId: project.id,
         documentUuid: document.documentUuid,
@@ -128,7 +128,7 @@ describe('runBatchAction', () => {
     })
 
     it('handles optional parameters', async () => {
-      const [result, error] = await runBatchAction({
+      const [result, error] = await runBatchEvaluationAction({
         datasetId: dataset.id,
         projectId: project.id,
         documentUuid: document.documentUuid,
@@ -156,7 +156,7 @@ describe('runBatchAction', () => {
     })
 
     it('handles errors when resources are not found', async () => {
-      const [_, error] = await runBatchAction({
+      const [_, error] = await runBatchEvaluationAction({
         datasetId: 999999,
         projectId: project.id,
         documentUuid: document.documentUuid,
@@ -176,7 +176,7 @@ describe('runBatchAction', () => {
         name: 'Test Evaluation 2',
       })
 
-      const [result, error] = await runBatchAction({
+      const [result, error] = await runBatchEvaluationAction({
         datasetId: dataset.id,
         projectId: project.id,
         documentUuid: document.documentUuid,

apps/web/src/actions/evaluations/runBatch.ts

@@ -1,3 +1,5 @@
+'use server'
+
 import {
   DatasetsRepository,
   DocumentVersionsRepository,
@@ -9,16 +11,16 @@ import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
-export const runBatchAction = authProcedure
+export const runBatchEvaluationAction = authProcedure
   .createServerAction()
   .input(
     z.object({
       datasetId: z.number(),
       projectId: z.number(),
       documentUuid: z.string(),
       commitUuid: z.string(),
-      fromLine: z.number(),
-      toLine: z.number(),
+      fromLine: z.number().optional(),
+      toLine: z.number().optional(),
       parameters: z.record(z.number()).optional(),
       evaluationIds: z.array(z.number()),
     }),

apps/web/src/actions/prompts/run.ts

@@ -14,7 +14,7 @@ export const runPromptAction = authProcedure
   .input(
     z.object({
       prompt: z.string(),
-      parameters: z.object({ messages: z.string(), last_message: z.string() }),
+      parameters: z.record(z.any()),
     }),
   )
   .handler(async ({ ctx, input }) => {

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx

@@ -24,10 +24,18 @@ import {
 } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat'
 import { readStreamableValue } from 'ai/rsc'
 
-export const EVALUATION_PARAMETERS = ['messages', 'last_message']
+export const EVALUATION_PARAMETERS = [
+  'messages',
+  'context',
+  'response',
+  'prompt',
+  'parameters',
+  'cost',
+  'latency',
+  'config',
+]
 
 export type Parameters = (typeof EVALUATION_PARAMETERS)[number]
-export type Inputs = { [key in Parameters]: string }
 
 export default function Chat({
   clearChat,
@@ -36,7 +44,7 @@ export default function Chat({
 }: {
   clearChat: () => void
   evaluation: EvaluationDto
-  parameters: Inputs
+  parameters: Record<string, string>
 }) {
   const [error, setError] = useState<Error | undefined>()
   const [tokens, setTokens] = useState<number>(0)
@@ -79,7 +87,7 @@ export default function Chat({
 
     const [data, error] = await runPromptAction({
       prompt: evaluation.metadata.prompt,
-      parameters: parameters as { messages: string; last_message: string },
+      parameters,
     })
     if (error) {
       setError(error)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx

@@ -1,50 +1,24 @@
 'use client'
 
 import { useCallback, useEffect, useMemo, useState } from 'react'
-import { capitalize } from 'lodash-es'
 
-import {
-  ConversationMetadata,
-  Message,
-  MessageContent,
-  TextContent,
-} from '@latitude-data/compiler'
+import { ConversationMetadata } from '@latitude-data/compiler'
 import { EvaluationDto } from '@latitude-data/core/browser'
-import { Badge, Icon, Text, TextArea } from '@latitude-data/web-ui'
+import {
+  formatContext,
+  formatConversation,
+} from '@latitude-data/core/services/providerLogs/formatForEvaluation'
+import { Badge, Icon, Input, Text } from '@latitude-data/web-ui'
+import { convertParams } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground'
 import { ROUTES } from '$/services/routes'
 import useProviderLogs from '$/stores/providerLogs'
 import Link from 'next/link'
 import { useSearchParams } from 'next/navigation'
 
 import { Header } from '../Header'
-import Chat, { EVALUATION_PARAMETERS, Inputs } from './Chat'
+import Chat, { EVALUATION_PARAMETERS } from './Chat'
 import Preview from './Preview'
 
-function convertMessage(message: Message) {
-  if (typeof message.content === 'string') {
-    return `${capitalize(message.role)}: \n ${message.content}`
-  } else {
-    const content = message.content[0] as MessageContent
-    if (content.type === 'text') {
-      return `${capitalize(message.role)}: \n ${(content as TextContent).text}`
-    } else {
-      return `${capitalize(message.role)}: <${content.type} message>`
-    }
-  }
-}
-
-function convertMessages(messages: Message[]) {
-  return messages.map((message) => convertMessage(message)).join('\n')
-}
-
-function convertParams(inputs: Inputs) {
-  return Object.fromEntries(
-    Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
-    }),
-  )
-}
-
 export default function Playground({
   evaluation,
   metadata,
@@ -53,7 +27,7 @@ export default function Playground({
   metadata: ConversationMetadata
 }) {
   const [mode, setMode] = useState<'preview' | 'chat'>('preview')
-  const [inputs, setInputs] = useState<Inputs>(
+  const [inputs, setInputs] = useState<Record<string, string>>(
     Object.fromEntries(
       EVALUATION_PARAMETERS.map((param: string) => [param, '']),
     ),
@@ -74,8 +48,14 @@ export default function Playground({
   useEffect(() => {
     if (providerLog) {
       setInputs({
-        messages: convertMessages(providerLog.messages),
-        last_message: `Assistant: ${providerLog.responseText}`,
+        messages: JSON.stringify(formatConversation(providerLog)),
+        context: JSON.stringify(formatContext(providerLog)),
+        response: providerLog.responseText,
+        prompt: '',
+        parameters: '',
+        config: '',
+        duration: '',
+        cost: '',
       })
     }
   }, [setInput, providerLog])
@@ -106,7 +86,7 @@ export default function Playground({
               >
                 <Badge variant='accent'>&#123;&#123;{param}&#125;&#125;</Badge>
                 <div className='flex flex-grow w-full'>
-                  <TextArea
+                  <Input
                     value={value}
                     onChange={(e) => setInput(param, e.currentTarget.value)}
                   />

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx

@@ -10,12 +10,14 @@ import { Header } from '../Header'
 import Chat from './Chat'
 import Preview from './Preview'
 
-function convertParams(
-  inputs: Record<string, string>,
-): Record<string, unknown> {
+export function convertParams(inputs: Record<string, string>) {
   return Object.fromEntries(
     Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
+      try {
+        return [key, JSON.parse(value)]
+      } catch (e) {
+        return [key, value]
+      }
     }),
   )
 }

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx

@@ -0,0 +1,51 @@
+'use client'
+
+import { Button, TableWithHeader } from '@latitude-data/web-ui'
+import { runBatchEvaluationAction } from '$/actions/evaluations/runBatch'
+import useLatitudeAction from '$/hooks/useLatitudeAction'
+import { ROUTES } from '$/services/routes'
+import Link from 'next/link'
+
+export function Actions({
+  projectId,
+  commitUuid,
+  documentUuid,
+}: {
+  projectId: string
+  commitUuid: string
+  documentUuid: string
+}) {
+  const href = ROUTES.projects
+    .detail({ id: Number(projectId) })
+    .commits.detail({ uuid: commitUuid })
+    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
+  const { execute: executeBatchEvaluation } = useLatitudeAction(
+    runBatchEvaluationAction,
+  )
+
+  return (
+    <>
+      <Link href={href}>
+        <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
+      </Link>
+      <Button
+        fancy
+        onClick={() =>
+          executeBatchEvaluation({
+            projectId: Number(projectId),
+            commitUuid,
+            documentUuid,
+            evaluationIds: [1],
+            datasetId: 1,
+            parameters: {
+              nombre_usuario: 0,
+              nombre: 2,
+            },
+          })
+        }
+      >
+        Execute batch evaluation
+      </Button>
+    </>
+  )
+}

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx

@@ -2,9 +2,8 @@ import { ReactNode } from 'react'
 
 import { TableWithHeader } from '@latitude-data/web-ui'
 import { getEvaluationsByDocumentUuidCached } from '$/app/(private)/_data-access'
-import { ROUTES } from '$/services/routes'
-import Link from 'next/link'
 
+import { Actions } from './_components/Actions'
 import EvaluationsLayoutClient from './_components/Layout'
 
 export default async function EvaluationsLayout({
@@ -15,20 +14,17 @@ export default async function EvaluationsLayout({
   params: { projectId: string; commitUuid: string; documentUuid: string }
 }) {
   const evaluations = await getEvaluationsByDocumentUuidCached(documentUuid)
-  const href = ROUTES.projects
-    .detail({ id: Number(projectId) })
-    .commits.detail({ uuid: commitUuid })
-    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
-
   return (
     <div className='w-full p-6'>
       {children}
       <TableWithHeader
         title='Evaluations'
         actions={
-          <Link href={href}>
-            <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
-          </Link>
+          <Actions
+            projectId={projectId}
+            commitUuid={commitUuid}
+            documentUuid={documentUuid}
+          />
         }
         table={<EvaluationsLayoutClient evaluations={evaluations} />}
       />

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx

@@ -137,7 +137,7 @@ export function DocumentLogMetadata({
           trigger={
             <div className='flex flex-row items-center gap-x-1'>
               <Text.H5 color='foregroundMuted'>
-                {formatCostInMillicents(documentLog.cost_in_millicents ?? 0)}
+                {formatCostInMillicents(documentLog.costInMillicents ?? 0)}
               </Text.H5>
               <Icon name='info' className='text-muted-foreground' />
             </div>

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx

@@ -84,7 +84,7 @@ export const DocumentLogsTable = ({
             </TableCell>
             <TableCell>
               <Text.H4 noWrap>
-                {formatCostInMillicents(documentLog.cost_in_millicents || 0)}
+                {formatCostInMillicents(documentLog.costInMillicents || 0)}
               </Text.H4>
             </TableCell>
           </TableRow>

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx

@@ -1,3 +1,5 @@
+'use server'
+
 import { ReactNode } from 'react'
 
 import {

packages/core/public/uploads/workspaces/1/datasets/batchtest.csv

@@ -0,0 +1,7 @@
+Username, Identifier,First name,Last name
+booker12,9012,Rachel,Booker
+grey07,2070,Laura,Grey
+johnson81,4081,Craig,Johnson
+jenkins46,9346,Mary,Jenkins
+smith79,5079,Jamie,Smith
+