feature: batch evaluations backend #2

This commit implements the full cycle of a batch evaluation. Now users can run batch evaluations on top of their prompts with dataset data.
latitude-dev · Sep 11, 2024 · 7d3804d · 7d3804d
1 parent 50707ee
commit 7d3804d
Show file tree

Hide file tree

Showing 43 changed files with 913 additions and 287 deletions.
diff --git a/...teway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts b/...teway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts
@@ -2,7 +2,6 @@ import { zValidator } from '@hono/zod-validator'
 import { LogSources } from '@latitude-data/core/browser'
 import { runDocumentAtCommit } from '@latitude-data/core/services/commits/runDocumentAtCommit'
 import { pipeToStream } from '$/common/pipeToStream'
-import { queues } from '$/jobs'
 import { Factory } from 'hono/factory'
 import { streamSSE } from 'hono/streaming'
 import { z } from 'zod'
@@ -21,8 +20,6 @@ export const runHandler = factory.createHandlers(
   zValidator('json', runSchema),
   async (c) => {
     return streamSSE(c, async (stream) => {
-      const startTime = Date.now()
-
       const { projectId, commitUuid } = c.req.param()
       const { documentPath, parameters, source } = c.req.valid('json')
 
@@ -44,17 +41,6 @@ export const runHandler = factory.createHandlers(
       }).then((r) => r.unwrap())
 
       await pipeToStream(stream, result.stream)
-
-      queues.defaultQueue.jobs.enqueueCreateDocumentLogJob({
-        commit,
-        data: {
-          uuid: result.documentLogUuid,
-          documentUuid: document.documentUuid,
-          resolvedContent: result.resolvedContent,
-          parameters,
-          duration: Date.now() - startTime,
-        },
-      })
     })
   },
 )
diff --git a/apps/web/src/actions/evaluations/runBatch.ts b/apps/web/src/actions/evaluations/runBatch.ts
@@ -1,3 +1,5 @@
+'use server'
+
 import {
   DatasetsRepository,
   DocumentVersionsRepository,
@@ -9,16 +11,16 @@ import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
-export const runBatchAction = authProcedure
+export const runBatchEvaluationAction = authProcedure
   .createServerAction()
   .input(
     z.object({
       datasetId: z.number(),
       projectId: z.number(),
       documentUuid: z.string(),
       commitUuid: z.string(),
-      fromLine: z.number(),
-      toLine: z.number(),
+      fromLine: z.number().optional(),
+      toLine: z.number().optional(),
       parameters: z.record(z.number()).optional(),
       evaluationIds: z.array(z.number()),
     }),

diff --git a/apps/web/src/actions/prompts/run.ts b/apps/web/src/actions/prompts/run.ts
@@ -14,7 +14,7 @@ export const runPromptAction = authProcedure
   .input(
     z.object({
       prompt: z.string(),
-      parameters: z.object({ messages: z.string(), last_message: z.string() }),
+      parameters: z.record(z.any()),
     }),
   )
   .handler(async ({ ctx, input }) => {

diff --git a/...on)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx b/...on)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx
@@ -24,10 +24,18 @@ import {
 } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat'
 import { readStreamableValue } from 'ai/rsc'
 
-export const EVALUATION_PARAMETERS = ['messages', 'last_message']
+export const EVALUATION_PARAMETERS = [
+  'messages',
+  'context',
+  'response',
+  'prompt',
+  'parameters',
+  'cost',
+  'latency',
+  'config',
+]
 
 export type Parameters = (typeof EVALUATION_PARAMETERS)[number]
-export type Inputs = { [key in Parameters]: string }
 
 export default function Chat({
   clearChat,
@@ -36,7 +44,7 @@ export default function Chat({
 }: {
   clearChat: () => void
   evaluation: EvaluationDto
-  parameters: Inputs
+  parameters: Record<string, string>
 }) {
   const [error, setError] = useState<Error | undefined>()
   const [tokens, setTokens] = useState<number>(0)
@@ -79,7 +87,7 @@ export default function Chat({
 
     const [data, error] = await runPromptAction({
       prompt: evaluation.metadata.prompt,
-      parameters: parameters as { messages: string; last_message: string },
+      parameters,
     })
     if (error) {
       setError(error)

diff --git a/...luation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx b/...luation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx
@@ -1,50 +1,24 @@
 'use client'
 
 import { useCallback, useEffect, useMemo, useState } from 'react'
-import { capitalize } from 'lodash-es'
 
-import {
-  ConversationMetadata,
-  Message,
-  MessageContent,
-  TextContent,
-} from '@latitude-data/compiler'
+import { ConversationMetadata } from '@latitude-data/compiler'
 import { EvaluationDto } from '@latitude-data/core/browser'
-import { Badge, Icon, Text, TextArea } from '@latitude-data/web-ui'
+import {
+  formatContext,
+  formatConversation,
+} from '@latitude-data/core/services/providerLogs/formatForEvaluation'
+import { Badge, Icon, Input, Text } from '@latitude-data/web-ui'
+import { convertParams } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground'
 import { ROUTES } from '$/services/routes'
 import useProviderLogs from '$/stores/providerLogs'
 import Link from 'next/link'
 import { useSearchParams } from 'next/navigation'
 
 import { Header } from '../Header'
-import Chat, { EVALUATION_PARAMETERS, Inputs } from './Chat'
+import Chat, { EVALUATION_PARAMETERS } from './Chat'
 import Preview from './Preview'
 
-function convertMessage(message: Message) {
-  if (typeof message.content === 'string') {
-    return `${capitalize(message.role)}: \n ${message.content}`
-  } else {
-    const content = message.content[0] as MessageContent
-    if (content.type === 'text') {
-      return `${capitalize(message.role)}: \n ${(content as TextContent).text}`
-    } else {
-      return `${capitalize(message.role)}: <${content.type} message>`
-    }
-  }
-}
-
-function convertMessages(messages: Message[]) {
-  return messages.map((message) => convertMessage(message)).join('\n')
-}
-
-function convertParams(inputs: Inputs) {
-  return Object.fromEntries(
-    Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
-    }),
-  )
-}
-
 export default function Playground({
   evaluation,
   metadata,
@@ -53,7 +27,7 @@ export default function Playground({
   metadata: ConversationMetadata
 }) {
   const [mode, setMode] = useState<'preview' | 'chat'>('preview')
-  const [inputs, setInputs] = useState<Inputs>(
+  const [inputs, setInputs] = useState<Record<string, string>>(
     Object.fromEntries(
       EVALUATION_PARAMETERS.map((param: string) => [param, '']),
     ),
@@ -74,8 +48,14 @@ export default function Playground({
   useEffect(() => {
     if (providerLog) {
       setInputs({
-        messages: convertMessages(providerLog.messages),
-        last_message: `Assistant: ${providerLog.responseText}`,
+        messages: JSON.stringify(formatConversation(providerLog)),
+        context: JSON.stringify(formatContext(providerLog)),
+        response: providerLog.responseText,
+        prompt: '',
+        parameters: '',
+        config: '',
+        duration: '',
+        cost: '',
       })
     }
   }, [setInput, providerLog])
@@ -106,7 +86,7 @@ export default function Playground({
               >
                 <Badge variant='accent'>&#123;&#123;{param}&#125;&#125;</Badge>
                 <div className='flex flex-grow w-full'>
-                  <TextArea
+                  <Input
                     value={value}
                     onChange={(e) => setInput(param, e.currentTarget.value)}
                   />

diff --git a/...mmitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx b/...mmitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx
@@ -10,12 +10,14 @@ import { Header } from '../Header'
 import Chat from './Chat'
 import Preview from './Preview'
 
-function convertParams(
-  inputs: Record<string, string>,
-): Record<string, unknown> {
+export function convertParams(inputs: Record<string, string>) {
   return Object.fromEntries(
     Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
+      try {
+        return [key, JSON.parse(value)]
+      } catch (e) {
+        return [key, value]
+      }
     }),
   )
 }

diff --git a/...sions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx b/...sions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx
@@ -0,0 +1,51 @@
+'use client'
+
+import { Button, TableWithHeader } from '@latitude-data/web-ui'
+import { runBatchEvaluationAction } from '$/actions/evaluations/runBatch'
+import useLatitudeAction from '$/hooks/useLatitudeAction'
+import { ROUTES } from '$/services/routes'
+import Link from 'next/link'
+
+export function Actions({
+  projectId,
+  commitUuid,
+  documentUuid,
+}: {
+  projectId: string
+  commitUuid: string
+  documentUuid: string
+}) {
+  const href = ROUTES.projects
+    .detail({ id: Number(projectId) })
+    .commits.detail({ uuid: commitUuid })
+    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
+  const { execute: executeBatchEvaluation } = useLatitudeAction(
+    runBatchEvaluationAction,
+  )
+
+  return (
+    <>
+      <Link href={href}>
+        <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
+      </Link>
+      <Button
+        fancy
+        onClick={() =>
+          executeBatchEvaluation({
+            projectId: Number(projectId),
+            commitUuid,
+            documentUuid,
+            evaluationIds: [1],
+            datasetId: 1,
+            parameters: {
+              nombre_usuario: 0,
+              nombre: 2,
+            },
+          })
+        }
+      >
+        Execute batch evaluation
+      </Button>
+    </>
+  )
+}
diff --git a/...rojectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx b/...rojectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx
@@ -2,9 +2,8 @@ import { ReactNode } from 'react'
 
 import { TableWithHeader } from '@latitude-data/web-ui'
 import { getEvaluationsByDocumentUuidCached } from '$/app/(private)/_data-access'
-import { ROUTES } from '$/services/routes'
-import Link from 'next/link'
 
+import { Actions } from './_components/Actions'
 import EvaluationsLayoutClient from './_components/Layout'
 
 export default async function EvaluationsLayout({
@@ -15,20 +14,17 @@ export default async function EvaluationsLayout({
   params: { projectId: string; commitUuid: string; documentUuid: string }
 }) {
   const evaluations = await getEvaluationsByDocumentUuidCached(documentUuid)
-  const href = ROUTES.projects
-    .detail({ id: Number(projectId) })
-    .commits.detail({ uuid: commitUuid })
-    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
-
   return (
     <div className='w-full p-6'>
       {children}
       <TableWithHeader
         title='Evaluations'
         actions={
-          <Link href={href}>
-            <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
-          </Link>
+          <Actions
+            projectId={projectId}
+            commitUuid={commitUuid}
+            documentUuid={documentUuid}
+          />
         }
         table={<EvaluationsLayoutClient evaluations={evaluations} />}
       />

diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx
@@ -1,3 +1,5 @@
+'use server'
+
 import { ReactNode } from 'react'
 
 import {

diff --git a/packages/core/public/uploads/workspaces/1/datasets/batchtest.csv b/packages/core/public/uploads/workspaces/1/datasets/batchtest.csv
@@ -0,0 +1,7 @@
+Username, Identifier,First name,Last name
+booker12,9012,Rachel,Booker
+grey07,2070,Laura,Grey
+johnson81,4081,Craig,Johnson
+jenkins46,9346,Mary,Jenkins
+smith79,5079,Jamie,Smith
+
diff --git a/packages/core/src/data-access/documentLogs.ts b/packages/core/src/data-access/documentLogs.ts
@@ -0,0 +1,40 @@
+import { eq, getTableColumns } from 'drizzle-orm'
+
+import { DocumentLog } from '../browser'
+import { database } from '../client'
+import {
+  commits,
+  documentLogs,
+  documentVersions,
+  projects,
+  workspaces,
+} from '../schema'
+
+export const findWorkspaceFromDocumentLog = async (
+  documentLog: DocumentLog,
+  db = database,
+) => {
+  const result = await db
+    .select(getTableColumns(workspaces))
+    .from(workspaces)
+    .innerJoin(projects, eq(projects.workspaceId, workspaces.id))
+    .innerJoin(commits, eq(commits.projectId, projects.id))
+    .innerJoin(documentVersions, eq(documentVersions.commitId, commits.id))
+    .where(eq(documentVersions.documentUuid, documentLog.documentUuid))
+    .limit(1)
+
+  return result[0]
+}
+
+export const unsafelyFindDocumentLogByUuid = async (
+  documentLogUuid: string,
+  db = database,
+) => {
+  const result = await db
+    .select()
+    .from(documentLogs)
+    .where(eq(documentLogs.uuid, documentLogUuid))
+    .limit(1)
+
+  return result[0]
+}
diff --git a/packages/core/src/data-access/evaluations.ts b/packages/core/src/data-access/evaluations.ts
@@ -0,0 +1,14 @@
+import { eq } from 'drizzle-orm'
+
+import { Evaluation } from '../browser'
+import { database } from '../client'
+import { evaluations } from '../schema'
+
+export async function unsafelyFindEvaluation(
+  id: number,
+  db = database,
+): Promise<Evaluation | undefined> {
+  return await db.query.evaluations.findFirst({
+    where: eq(evaluations.id, id),
+  })
+}
diff --git a/packages/core/src/data-access/index.ts b/packages/core/src/data-access/index.ts
@@ -5,3 +5,6 @@ export * from './evaluationTemplates'
 export * from './evaluationTemplateCategories'
 export * from './memberships'
 export * from './magicLinkTokens'
+export * from './providerLogs'
+export * from './documentLogs'
+export * from './evaluations'