From 7d3804dc1e025990d957bfa207bae50d017e2953 Mon Sep 17 00:00:00 2001
From: Gerard Clos <clos.gerard@gmail.com>
Date: Wed, 11 Sep 2024 12:19:28 +0200
Subject: [PATCH] feature: batch evaluations backend #2

This commit implements the full cycle of a batch evaluation. Now users
can run batch evaluations on top of their prompts with dataset data.
---
 .../:commitUuid/documents/handlers/run.ts     |  14 ---
 apps/web/src/actions/evaluations/runBatch.ts  |   8 +-
 apps/web/src/actions/prompts/run.ts           |   2 +-
 .../Editor/Playground/Chat/index.tsx          |  16 ++-
 .../Editor/Playground/index.tsx               |  56 +++------
 .../Editor/Playground/index.tsx               |  10 +-
 .../dashboard/_components/Actions.tsx         |  51 ++++++++
 .../evaluations/dashboard/layout.tsx          |  16 +--
 .../versions/[commitUuid]/layout.tsx          |   2 +
 .../workspaces/1/datasets/batchtest.csv       |   7 ++
 packages/core/src/data-access/documentLogs.ts |  40 +++++++
 packages/core/src/data-access/evaluations.ts  |  14 +++
 packages/core/src/data-access/index.ts        |   3 +
 packages/core/src/data-access/providerLogs.ts |  27 +++++
 .../handlers/createEvaluationResultJob.ts     |  33 ++++++
 .../events/handlers/documentLogs/createJob.ts |  34 ++++++
 packages/core/src/events/handlers/index.ts    |  33 ++++++
 .../documentLogsRepository/index.ts           |  78 ++++---------
 .../core/src/schema/models/documentLogs.ts    |  12 +-
 .../core/src/schema/models/providerLogs.ts    |   2 +-
 packages/core/src/services/chains/run.ts      |   8 ++
 .../services/commits/runDocumentAtCommit.ts   |  21 +++-
 .../documentLogs/_createDocumentLogQuery.ts   |  52 +++++++++
 .../computeDocumentLogWithMetadata.test.ts    | 109 ++++++++++++++++++
 .../computeDocumentLogWithMetadata.ts         |  28 +++++
 .../computeDocumentLogsWithMetadata.ts        |  31 +++++
 .../core/src/services/documentLogs/index.ts   |   2 +
 .../src/services/evaluationResults/create.ts  |   3 +-
 .../core/src/services/evaluations/index.ts    |   1 +
 packages/core/src/services/evaluations/run.ts |  80 +++++++++++++
 .../core/src/services/providerLogs/create.ts  |  10 +-
 .../providerLogs/formatForEvaluation.ts       |  45 ++++++++
 .../core/src/services/providerLogs/index.ts   |   1 +
 packages/core/src/tests/factories/index.ts    |   1 +
 .../core/src/tests/factories/providerLogs.ts  |  44 +++++++
 .../batchEvaluations/runBatchEvaluationJob.ts | 109 +++++++++---------
 .../batchEvaluations/runDocumentJob.ts        |  98 +++++++++-------
 .../batchEvaluations/runEvaluationJob.ts      |  65 ++++++-----
 .../job-definitions/documentLogs/createJob.ts |  13 ---
 packages/jobs/src/job-definitions/index.ts    |   5 +-
 packages/jobs/src/queues/index.ts             |   3 +-
 packages/jobs/src/workers/_shared.ts          |  11 +-
 .../web-ui/src/ds/atoms/SplitPane/index.tsx   |   2 +-
 43 files changed, 913 insertions(+), 287 deletions(-)
 create mode 100644 apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx
 create mode 100644 packages/core/public/uploads/workspaces/1/datasets/batchtest.csv
 create mode 100644 packages/core/src/data-access/documentLogs.ts
 create mode 100644 packages/core/src/data-access/evaluations.ts
 create mode 100644 packages/core/src/data-access/providerLogs.ts
 create mode 100644 packages/core/src/events/handlers/createEvaluationResultJob.ts
 create mode 100644 packages/core/src/events/handlers/documentLogs/createJob.ts
 create mode 100644 packages/core/src/services/documentLogs/_createDocumentLogQuery.ts
 create mode 100644 packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts
 create mode 100644 packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.ts
 create mode 100644 packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts
 create mode 100644 packages/core/src/services/evaluations/run.ts
 create mode 100644 packages/core/src/services/providerLogs/formatForEvaluation.ts
 create mode 100644 packages/core/src/tests/factories/providerLogs.ts
 delete mode 100644 packages/jobs/src/job-definitions/documentLogs/createJob.ts

diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts
index 7ef0b349d..d50bb2c35 100644
--- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts
+++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts
@@ -2,7 +2,6 @@ import { zValidator } from '@hono/zod-validator'
 import { LogSources } from '@latitude-data/core/browser'
 import { runDocumentAtCommit } from '@latitude-data/core/services/commits/runDocumentAtCommit'
 import { pipeToStream } from '$/common/pipeToStream'
-import { queues } from '$/jobs'
 import { Factory } from 'hono/factory'
 import { streamSSE } from 'hono/streaming'
 import { z } from 'zod'
@@ -21,8 +20,6 @@ export const runHandler = factory.createHandlers(
   zValidator('json', runSchema),
   async (c) => {
     return streamSSE(c, async (stream) => {
-      const startTime = Date.now()
-
       const { projectId, commitUuid } = c.req.param()
       const { documentPath, parameters, source } = c.req.valid('json')
 
@@ -44,17 +41,6 @@ export const runHandler = factory.createHandlers(
       }).then((r) => r.unwrap())
 
       await pipeToStream(stream, result.stream)
-
-      queues.defaultQueue.jobs.enqueueCreateDocumentLogJob({
-        commit,
-        data: {
-          uuid: result.documentLogUuid,
-          documentUuid: document.documentUuid,
-          resolvedContent: result.resolvedContent,
-          parameters,
-          duration: Date.now() - startTime,
-        },
-      })
     })
   },
 )
diff --git a/apps/web/src/actions/evaluations/runBatch.ts b/apps/web/src/actions/evaluations/runBatch.ts
index 778b108b2..5db530095 100644
--- a/apps/web/src/actions/evaluations/runBatch.ts
+++ b/apps/web/src/actions/evaluations/runBatch.ts
@@ -1,3 +1,5 @@
+'use server'
+
 import {
   DatasetsRepository,
   DocumentVersionsRepository,
@@ -9,7 +11,7 @@ import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
-export const runBatchAction = authProcedure
+export const runBatchEvaluationAction = authProcedure
   .createServerAction()
   .input(
     z.object({
@@ -17,8 +19,8 @@ export const runBatchAction = authProcedure
       projectId: z.number(),
       documentUuid: z.string(),
       commitUuid: z.string(),
-      fromLine: z.number(),
-      toLine: z.number(),
+      fromLine: z.number().optional(),
+      toLine: z.number().optional(),
       parameters: z.record(z.number()).optional(),
       evaluationIds: z.array(z.number()),
     }),
diff --git a/apps/web/src/actions/prompts/run.ts b/apps/web/src/actions/prompts/run.ts
index 0a870ecdf..36f0a0f69 100644
--- a/apps/web/src/actions/prompts/run.ts
+++ b/apps/web/src/actions/prompts/run.ts
@@ -14,7 +14,7 @@ export const runPromptAction = authProcedure
   .input(
     z.object({
       prompt: z.string(),
-      parameters: z.object({ messages: z.string(), last_message: z.string() }),
+      parameters: z.record(z.any()),
     }),
   )
   .handler(async ({ ctx, input }) => {
diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx
index 2cbc9df82..3879b2392 100644
--- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx
+++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Chat/index.tsx
@@ -24,10 +24,18 @@ import {
 } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat'
 import { readStreamableValue } from 'ai/rsc'
 
-export const EVALUATION_PARAMETERS = ['messages', 'last_message']
+export const EVALUATION_PARAMETERS = [
+  'messages',
+  'context',
+  'response',
+  'prompt',
+  'parameters',
+  'cost',
+  'latency',
+  'config',
+]
 
 export type Parameters = (typeof EVALUATION_PARAMETERS)[number]
-export type Inputs = { [key in Parameters]: string }
 
 export default function Chat({
   clearChat,
@@ -36,7 +44,7 @@ export default function Chat({
 }: {
   clearChat: () => void
   evaluation: EvaluationDto
-  parameters: Inputs
+  parameters: Record<string, string>
 }) {
   const [error, setError] = useState<Error | undefined>()
   const [tokens, setTokens] = useState<number>(0)
@@ -79,7 +87,7 @@ export default function Chat({
 
     const [data, error] = await runPromptAction({
       prompt: evaluation.metadata.prompt,
-      parameters: parameters as { messages: string; last_message: string },
+      parameters,
     })
     if (error) {
       setError(error)
diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx
index 9a63b6cc7..fb3a1bf15 100644
--- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx
+++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx
@@ -1,50 +1,24 @@
 'use client'
 
 import { useCallback, useEffect, useMemo, useState } from 'react'
-import { capitalize } from 'lodash-es'
 
-import {
-  ConversationMetadata,
-  Message,
-  MessageContent,
-  TextContent,
-} from '@latitude-data/compiler'
+import { ConversationMetadata } from '@latitude-data/compiler'
 import { EvaluationDto } from '@latitude-data/core/browser'
-import { Badge, Icon, Text, TextArea } from '@latitude-data/web-ui'
+import {
+  formatContext,
+  formatConversation,
+} from '@latitude-data/core/services/providerLogs/formatForEvaluation'
+import { Badge, Icon, Input, Text } from '@latitude-data/web-ui'
+import { convertParams } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground'
 import { ROUTES } from '$/services/routes'
 import useProviderLogs from '$/stores/providerLogs'
 import Link from 'next/link'
 import { useSearchParams } from 'next/navigation'
 
 import { Header } from '../Header'
-import Chat, { EVALUATION_PARAMETERS, Inputs } from './Chat'
+import Chat, { EVALUATION_PARAMETERS } from './Chat'
 import Preview from './Preview'
 
-function convertMessage(message: Message) {
-  if (typeof message.content === 'string') {
-    return `${capitalize(message.role)}: \n ${message.content}`
-  } else {
-    const content = message.content[0] as MessageContent
-    if (content.type === 'text') {
-      return `${capitalize(message.role)}: \n ${(content as TextContent).text}`
-    } else {
-      return `${capitalize(message.role)}: <${content.type} message>`
-    }
-  }
-}
-
-function convertMessages(messages: Message[]) {
-  return messages.map((message) => convertMessage(message)).join('\n')
-}
-
-function convertParams(inputs: Inputs) {
-  return Object.fromEntries(
-    Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
-    }),
-  )
-}
-
 export default function Playground({
   evaluation,
   metadata,
@@ -53,7 +27,7 @@ export default function Playground({
   metadata: ConversationMetadata
 }) {
   const [mode, setMode] = useState<'preview' | 'chat'>('preview')
-  const [inputs, setInputs] = useState<Inputs>(
+  const [inputs, setInputs] = useState<Record<string, string>>(
     Object.fromEntries(
       EVALUATION_PARAMETERS.map((param: string) => [param, '']),
     ),
@@ -74,8 +48,14 @@ export default function Playground({
   useEffect(() => {
     if (providerLog) {
       setInputs({
-        messages: convertMessages(providerLog.messages),
-        last_message: `Assistant: ${providerLog.responseText}`,
+        messages: JSON.stringify(formatConversation(providerLog)),
+        context: JSON.stringify(formatContext(providerLog)),
+        response: providerLog.responseText,
+        prompt: '',
+        parameters: '',
+        config: '',
+        duration: '',
+        cost: '',
       })
     }
   }, [setInput, providerLog])
@@ -106,7 +86,7 @@ export default function Playground({
               >
                 <Badge variant='accent'>&#123;&#123;{param}&#125;&#125;</Badge>
                 <div className='flex flex-grow w-full'>
-                  <TextArea
+                  <Input
                     value={value}
                     onChange={(e) => setInput(param, e.currentTarget.value)}
                   />
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx
index 4f2882415..cebd7b158 100644
--- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx
+++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/index.tsx
@@ -10,12 +10,14 @@ import { Header } from '../Header'
 import Chat from './Chat'
 import Preview from './Preview'
 
-function convertParams(
-  inputs: Record<string, string>,
-): Record<string, unknown> {
+export function convertParams(inputs: Record<string, string>) {
   return Object.fromEntries(
     Object.entries(inputs).map(([key, value]) => {
-      return [key, value]
+      try {
+        return [key, JSON.parse(value)]
+      } catch (e) {
+        return [key, value]
+      }
     }),
   )
 }
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx
new file mode 100644
index 000000000..9d67c8d63
--- /dev/null
+++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/_components/Actions.tsx
@@ -0,0 +1,51 @@
+'use client'
+
+import { Button, TableWithHeader } from '@latitude-data/web-ui'
+import { runBatchEvaluationAction } from '$/actions/evaluations/runBatch'
+import useLatitudeAction from '$/hooks/useLatitudeAction'
+import { ROUTES } from '$/services/routes'
+import Link from 'next/link'
+
+export function Actions({
+  projectId,
+  commitUuid,
+  documentUuid,
+}: {
+  projectId: string
+  commitUuid: string
+  documentUuid: string
+}) {
+  const href = ROUTES.projects
+    .detail({ id: Number(projectId) })
+    .commits.detail({ uuid: commitUuid })
+    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
+  const { execute: executeBatchEvaluation } = useLatitudeAction(
+    runBatchEvaluationAction,
+  )
+
+  return (
+    <>
+      <Link href={href}>
+        <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
+      </Link>
+      <Button
+        fancy
+        onClick={() =>
+          executeBatchEvaluation({
+            projectId: Number(projectId),
+            commitUuid,
+            documentUuid,
+            evaluationIds: [1],
+            datasetId: 1,
+            parameters: {
+              nombre_usuario: 0,
+              nombre: 2,
+            },
+          })
+        }
+      >
+        Execute batch evaluation
+      </Button>
+    </>
+  )
+}
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx
index 36cb43917..fd2238093 100644
--- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx
+++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/dashboard/layout.tsx
@@ -2,9 +2,8 @@ import { ReactNode } from 'react'
 
 import { TableWithHeader } from '@latitude-data/web-ui'
 import { getEvaluationsByDocumentUuidCached } from '$/app/(private)/_data-access'
-import { ROUTES } from '$/services/routes'
-import Link from 'next/link'
 
+import { Actions } from './_components/Actions'
 import EvaluationsLayoutClient from './_components/Layout'
 
 export default async function EvaluationsLayout({
@@ -15,20 +14,17 @@ export default async function EvaluationsLayout({
   params: { projectId: string; commitUuid: string; documentUuid: string }
 }) {
   const evaluations = await getEvaluationsByDocumentUuidCached(documentUuid)
-  const href = ROUTES.projects
-    .detail({ id: Number(projectId) })
-    .commits.detail({ uuid: commitUuid })
-    .documents.detail({ uuid: documentUuid }).evaluations.dashboard.connect.root
-
   return (
     <div className='w-full p-6'>
       {children}
       <TableWithHeader
         title='Evaluations'
         actions={
-          <Link href={href}>
-            <TableWithHeader.Button>Connect evaluation</TableWithHeader.Button>
-          </Link>
+          <Actions
+            projectId={projectId}
+            commitUuid={commitUuid}
+            documentUuid={documentUuid}
+          />
         }
         table={<EvaluationsLayoutClient evaluations={evaluations} />}
       />
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx
index 31ae8ec9f..79ca23a30 100644
--- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx
+++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/layout.tsx
@@ -1,3 +1,5 @@
+'use server'
+
 import { ReactNode } from 'react'
 
 import {
diff --git a/packages/core/public/uploads/workspaces/1/datasets/batchtest.csv b/packages/core/public/uploads/workspaces/1/datasets/batchtest.csv
new file mode 100644
index 000000000..402a34807
--- /dev/null
+++ b/packages/core/public/uploads/workspaces/1/datasets/batchtest.csv
@@ -0,0 +1,7 @@
+Username, Identifier,First name,Last name
+booker12,9012,Rachel,Booker
+grey07,2070,Laura,Grey
+johnson81,4081,Craig,Johnson
+jenkins46,9346,Mary,Jenkins
+smith79,5079,Jamie,Smith
+
diff --git a/packages/core/src/data-access/documentLogs.ts b/packages/core/src/data-access/documentLogs.ts
new file mode 100644
index 000000000..d0e58813f
--- /dev/null
+++ b/packages/core/src/data-access/documentLogs.ts
@@ -0,0 +1,40 @@
+import { eq, getTableColumns } from 'drizzle-orm'
+
+import { DocumentLog } from '../browser'
+import { database } from '../client'
+import {
+  commits,
+  documentLogs,
+  documentVersions,
+  projects,
+  workspaces,
+} from '../schema'
+
+export const findWorkspaceFromDocumentLog = async (
+  documentLog: DocumentLog,
+  db = database,
+) => {
+  const result = await db
+    .select(getTableColumns(workspaces))
+    .from(workspaces)
+    .innerJoin(projects, eq(projects.workspaceId, workspaces.id))
+    .innerJoin(commits, eq(commits.projectId, projects.id))
+    .innerJoin(documentVersions, eq(documentVersions.commitId, commits.id))
+    .where(eq(documentVersions.documentUuid, documentLog.documentUuid))
+    .limit(1)
+
+  return result[0]
+}
+
+export const unsafelyFindDocumentLogByUuid = async (
+  documentLogUuid: string,
+  db = database,
+) => {
+  const result = await db
+    .select()
+    .from(documentLogs)
+    .where(eq(documentLogs.uuid, documentLogUuid))
+    .limit(1)
+
+  return result[0]
+}
diff --git a/packages/core/src/data-access/evaluations.ts b/packages/core/src/data-access/evaluations.ts
new file mode 100644
index 000000000..bb8528cb5
--- /dev/null
+++ b/packages/core/src/data-access/evaluations.ts
@@ -0,0 +1,14 @@
+import { eq } from 'drizzle-orm'
+
+import { Evaluation } from '../browser'
+import { database } from '../client'
+import { evaluations } from '../schema'
+
+export async function unsafelyFindEvaluation(
+  id: number,
+  db = database,
+): Promise<Evaluation | undefined> {
+  return await db.query.evaluations.findFirst({
+    where: eq(evaluations.id, id),
+  })
+}
diff --git a/packages/core/src/data-access/index.ts b/packages/core/src/data-access/index.ts
index 0c21a8339..c0e63eb03 100644
--- a/packages/core/src/data-access/index.ts
+++ b/packages/core/src/data-access/index.ts
@@ -5,3 +5,6 @@ export * from './evaluationTemplates'
 export * from './evaluationTemplateCategories'
 export * from './memberships'
 export * from './magicLinkTokens'
+export * from './providerLogs'
+export * from './documentLogs'
+export * from './evaluations'
diff --git a/packages/core/src/data-access/providerLogs.ts b/packages/core/src/data-access/providerLogs.ts
new file mode 100644
index 000000000..141611e17
--- /dev/null
+++ b/packages/core/src/data-access/providerLogs.ts
@@ -0,0 +1,27 @@
+import { desc, eq } from 'drizzle-orm'
+
+import { database } from '../client'
+import { providerLogs } from '../schema'
+
+export const findLastProviderLogFromDocumentLogUuid = async (
+  documentLogUuid: string,
+  db = database,
+) => {
+  return await db.query.providerLogs.findFirst({
+    where: eq(providerLogs.documentLogUuid, documentLogUuid),
+    orderBy: desc(providerLogs.generatedAt),
+  })
+}
+
+export const unsafelyFindProviderLogByUuid = async (
+  providerLogUuid: string,
+  db = database,
+) => {
+  const result = await db
+    .select()
+    .from(providerLogs)
+    .where(eq(providerLogs.uuid, providerLogUuid))
+    .limit(1)
+
+  return result[0]
+}
diff --git a/packages/core/src/events/handlers/createEvaluationResultJob.ts b/packages/core/src/events/handlers/createEvaluationResultJob.ts
new file mode 100644
index 000000000..09ffa92cc
--- /dev/null
+++ b/packages/core/src/events/handlers/createEvaluationResultJob.ts
@@ -0,0 +1,33 @@
+import { EvaluationRunEvent } from '.'
+import {
+  unsafelyFindDocumentLogByUuid,
+  unsafelyFindEvaluation,
+  unsafelyFindProviderLogByUuid,
+} from '../../data-access'
+import { NotFoundError } from '../../lib'
+import { createEvaluationResult } from '../../services/evaluationResults'
+
+export const createEvaluationResultJob = async ({
+  data: event,
+}: {
+  data: EvaluationRunEvent
+}) => {
+  const { evaluationId, documentLogUuid, providerLogUuid, response } =
+    event.data
+
+  const evaluation = await unsafelyFindEvaluation(evaluationId)
+  if (!evaluation) throw new NotFoundError('Evaluation not found')
+
+  const documentLog = await unsafelyFindDocumentLogByUuid(documentLogUuid)
+  if (!documentLog) throw new NotFoundError('Document log not found')
+
+  const providerLog = await unsafelyFindProviderLogByUuid(providerLogUuid)
+  if (!providerLog) throw new NotFoundError('Provider log not found')
+
+  await createEvaluationResult({
+    evaluation,
+    documentLog,
+    providerLog,
+    result: response.text,
+  }).then((r) => r.unwrap())
+}
diff --git a/packages/core/src/events/handlers/documentLogs/createJob.ts b/packages/core/src/events/handlers/documentLogs/createJob.ts
new file mode 100644
index 000000000..afd29ed9a
--- /dev/null
+++ b/packages/core/src/events/handlers/documentLogs/createJob.ts
@@ -0,0 +1,34 @@
+import { DocumentRunEvent } from '..'
+import { CommitsRepository } from '../../../repositories'
+import {
+  createDocumentLog,
+  CreateDocumentLogProps,
+} from '../../../services/documentLogs'
+
+export type CreateDocumentLogJobData = CreateDocumentLogProps
+
+export const createDocumentLogJob = async ({
+  data: event,
+}: {
+  data: DocumentRunEvent
+}) => {
+  const scope = new CommitsRepository(event.data.workspaceId)
+  const commit = await scope
+    .getCommitByUuid({
+      projectId: event.data.projectId,
+      uuid: event.data.commitUuid,
+    })
+    .then((r) => r.unwrap())
+
+  await createDocumentLog({
+    commit,
+    data: {
+      customIdentifier: event.data.customIdentifier,
+      documentUuid: event.data.documentUuid,
+      duration: event.data.duration,
+      parameters: event.data.parameters,
+      resolvedContent: event.data.resolvedContent,
+      uuid: event.data.documentLogUuid,
+    },
+  }).then((r) => r.unwrap())
+}
diff --git a/packages/core/src/events/handlers/index.ts b/packages/core/src/events/handlers/index.ts
index d10dbc336..ca7d86c70 100644
--- a/packages/core/src/events/handlers/index.ts
+++ b/packages/core/src/events/handlers/index.ts
@@ -2,6 +2,7 @@ import { ToolCall } from '@latitude-data/compiler'
 import { CompletionTokenUsage } from 'ai'
 
 import {
+  ChainCallResponse,
   LogSources,
   MagicLinkToken,
   Membership,
@@ -10,7 +11,9 @@ import {
   User,
 } from '../../browser'
 import { PartialConfig } from '../../services/ai'
+import { createEvaluationResultJob } from './createEvaluationResultJob'
 import { createProviderLogJob } from './createProviderLogJob'
+import { createDocumentLogJob } from './documentLogs/createJob'
 import { sendInvitationToUserJob } from './sendInvitationToUser'
 import { sendMagicLinkJob } from './sendMagicLinkHandler'
 
@@ -55,18 +58,46 @@ export type MembershipCreatedEvent = LatitudeEventGeneric<
   'membershipCreated',
   Membership & { authorId?: string }
 >
+export type EvaluationRunEvent = LatitudeEventGeneric<
+  'evaluationRun',
+  {
+    evaluationId: number
+    documentLogUuid: string
+    providerLogUuid: string
+    response: ChainCallResponse
+  }
+>
+export type DocumentRunEvent = LatitudeEventGeneric<
+  'documentRun',
+  {
+    workspaceId: number
+    documentUuid: string
+    commitUuid: string
+    projectId: number
+    customIdentifier?: string
+    duration: number
+    documentLogUuid: string
+    response: ChainCallResponse
+    resolvedContent: string
+    parameters: Record<string, unknown>
+  }
+>
 
 export type LatitudeEvent =
   | MembershipCreatedEvent
   | UserCreatedEvent
   | MagicLinkTokenCreated
   | AIProviderCallCompleted
+  | EvaluationRunEvent
+  | DocumentRunEvent
 
 export interface IEventsHandlers {
   aiProviderCallCompleted: EventHandler<AIProviderCallCompleted>[]
   magicLinkTokenCreated: EventHandler<MagicLinkTokenCreated>[]
   membershipCreated: EventHandler<MembershipCreatedEvent>[]
   userCreated: EventHandler<UserCreatedEvent>[]
+  evaluationRun: EventHandler<EvaluationRunEvent>[]
+  documentRun: EventHandler<DocumentRunEvent>[]
 }
 
 export const EventHandlers: IEventsHandlers = {
@@ -74,4 +105,6 @@ export const EventHandlers: IEventsHandlers = {
   membershipCreated: [sendInvitationToUserJob],
   userCreated: [],
   aiProviderCallCompleted: [createProviderLogJob],
+  evaluationRun: [createEvaluationResultJob],
+  documentRun: [createDocumentLogJob],
 } as const
diff --git a/packages/core/src/repositories/documentLogsRepository/index.ts b/packages/core/src/repositories/documentLogsRepository/index.ts
index 76efeb6af..f5a9ba443 100644
--- a/packages/core/src/repositories/documentLogsRepository/index.ts
+++ b/packages/core/src/repositories/documentLogsRepository/index.ts
@@ -1,20 +1,16 @@
-import { and, desc, eq, getTableColumns, isNotNull, or, sum } from 'drizzle-orm'
+import { eq, getTableColumns } from 'drizzle-orm'
 
 import { Commit, DocumentLog } from '../../browser'
 import { NotFoundError, Result, TypedResult } from '../../lib'
-import {
-  commits,
-  documentLogs,
-  projects,
-  providerLogs,
-  workspaces,
-} from '../../schema'
+import { commits, documentLogs, projects, workspaces } from '../../schema'
+import { computeDocumentLogsWithMetadata } from '../../services/documentLogs'
 import Repository from '../repository'
 
 export type DocumentLogWithMetadata = DocumentLog & {
   commit: Commit
   tokens: number | null
-  cost_in_millicents: number | null
+  durationInMs: number | null
+  costInMillicents: number | null
 }
 
 const tt = getTableColumns(documentLogs)
@@ -30,59 +26,35 @@ export class DocumentLogsRepository extends Repository<typeof tt, DocumentLog> {
       .as('documentLogsScope')
   }
 
-  async findByUuid(uuid: string | undefined) {
+  async findByUuid(uuid: string) {
     const result = await this.db
       .select()
       .from(this.scope)
-      .where(eq(this.scope.uuid, uuid ?? ''))
+      .where(eq(this.scope.uuid, uuid))
 
     if (!result.length) {
-      return Result.error(new NotFoundError('ProviderLog not found'))
+      return Result.error(
+        new NotFoundError(`DocumentLog not found with uuid ${uuid}`),
+      )
     }
 
     return Result.ok(result[0]!)
   }
 
-  async getDocumentLogsWithMetadata(
-    { documentUuid, draft }: { documentUuid: string; draft?: Commit },
-    tx = this.db,
-  ): Promise<TypedResult<DocumentLogWithMetadata[], NotFoundError>> {
-    const aggregatedFieldsSubQuery = tx
-      .select({
-        id: this.scope.id,
-        tokens: sum(providerLogs.tokens).mapWith(Number).as('tokens'),
-        cost_in_millicents: sum(providerLogs.cost_in_millicents)
-          .mapWith(Number)
-          .as('cost_in_millicents'),
-      })
-      .from(this.scope)
-      .innerJoin(
-        providerLogs,
-        eq(providerLogs.documentLogUuid, this.scope.uuid),
-      )
-      .groupBy(this.scope.id)
-      .as('aggregatedFieldsSubQuery')
-
-    const commitFilter = draft
-      ? or(isNotNull(commits.mergedAt), eq(commits.id, draft.id))
-      : isNotNull(commits.mergedAt)
-
-    const result = await tx
-      .select({
-        ...this.scope._.selectedFields,
-        commit: commits,
-        tokens: aggregatedFieldsSubQuery.tokens,
-        cost_in_millicents: aggregatedFieldsSubQuery.cost_in_millicents,
-      })
-      .from(this.scope)
-      .innerJoin(commits, eq(commits.id, this.scope.commitId))
-      .innerJoin(
-        aggregatedFieldsSubQuery,
-        eq(aggregatedFieldsSubQuery.id, this.scope.id),
-      )
-      .where(and(eq(this.scope.documentUuid, documentUuid), commitFilter))
-      .orderBy(desc(this.scope.createdAt))
-
-    return Result.ok(result)
+  async getDocumentLogsWithMetadata({
+    documentUuid,
+    draft,
+  }: {
+    documentUuid: string
+    draft?: Commit
+  }): Promise<TypedResult<DocumentLogWithMetadata[], Error>> {
+    return computeDocumentLogsWithMetadata(
+      {
+        workspaceId: this.workspaceId,
+        documentUuid,
+        draft,
+      },
+      this.db,
+    )
   }
 }
diff --git a/packages/core/src/schema/models/documentLogs.ts b/packages/core/src/schema/models/documentLogs.ts
index 9b642de32..2787f9b01 100644
--- a/packages/core/src/schema/models/documentLogs.ts
+++ b/packages/core/src/schema/models/documentLogs.ts
@@ -1,4 +1,11 @@
-import { bigint, bigserial, index, json, text, uuid } from 'drizzle-orm/pg-core'
+import {
+  bigint,
+  bigserial,
+  index,
+  jsonb,
+  text,
+  uuid,
+} from 'drizzle-orm/pg-core'
 
 import { latitudeSchema } from '../db-schema'
 import { timestamps } from '../schemaHelpers'
@@ -17,12 +24,13 @@ export const documentLogs = latitudeSchema.table(
         onUpdate: 'cascade',
       }),
     resolvedContent: text('resolved_content').notNull(),
-    parameters: json('parameters').notNull(),
+    parameters: jsonb('parameters').$type<Record<string, unknown>>().notNull(),
     customIdentifier: text('custom_identifier'),
     duration: bigint('duration', { mode: 'number' }).notNull(),
     ...timestamps(),
   },
   (table) => ({
     documentLogUuidIdx: index('document_log_uuid_idx').on(table.documentUuid),
+    commitIdIdx: index('commit_id_idx').on(table.commitId),
   }),
 )
diff --git a/packages/core/src/schema/models/providerLogs.ts b/packages/core/src/schema/models/providerLogs.ts
index ac92b89ac..fd7f3ca87 100644
--- a/packages/core/src/schema/models/providerLogs.ts
+++ b/packages/core/src/schema/models/providerLogs.ts
@@ -40,7 +40,7 @@ export const providerLogs = latitudeSchema.table('provider_logs', {
   toolCalls: json('tool_calls').$type<ToolCall[]>().notNull().default([]),
   tokens: bigint('tokens', { mode: 'number' }).notNull(),
   cost_in_millicents: integer('cost_in_millicents').notNull().default(0),
-  duration: bigint('duration', { mode: 'number' }).notNull(),
+  duration: bigint('duration', { mode: 'number' }).notNull(), // in milliseconds!
   source: logSourcesEnum('source').notNull(),
   apiKeyId: bigint('apiKeyId', { mode: 'number' }).references(
     () => apiKeys.id,
diff --git a/packages/core/src/services/chains/run.ts b/packages/core/src/services/chains/run.ts
index e8dd99158..3bcd893ee 100644
--- a/packages/core/src/services/chains/run.ts
+++ b/packages/core/src/services/chains/run.ts
@@ -37,6 +37,7 @@ export async function runChain({
     responseReject = reject
   })
 
+  const startTime = Date.now()
   const stream = new ReadableStream<ChainEvent>({
     start(controller) {
       iterate({
@@ -51,11 +52,18 @@ export async function runChain({
     },
   })
 
+  const duration = async () => {
+    await response
+
+    return Date.now() - startTime
+  }
+
   return Result.ok({
     stream,
     response,
     resolvedContent: chain.rawText,
     documentLogUuid,
+    duration: duration(),
   })
 }
 
diff --git a/packages/core/src/services/commits/runDocumentAtCommit.ts b/packages/core/src/services/commits/runDocumentAtCommit.ts
index b6a4f440a..e47cb83e9 100644
--- a/packages/core/src/services/commits/runDocumentAtCommit.ts
+++ b/packages/core/src/services/commits/runDocumentAtCommit.ts
@@ -6,6 +6,7 @@ import {
   type DocumentVersion,
   type Workspace,
 } from '../../browser'
+import { publisher } from '../../events/publisher'
 import { Result } from '../../lib'
 import { runChain } from '../chains/run'
 import { getResolvedContent } from '../documents'
@@ -38,11 +39,27 @@ export async function runDocumentAtCommit({
     source,
   })
 
-  const { stream, response, documentLogUuid } = rezult.value
+  const { stream, response, duration, resolvedContent, documentLogUuid } =
+    rezult.value
 
   return Result.ok({
     stream,
-    response,
+    response: response.then(async (response) =>
+      publisher.publish({
+        type: 'documentRun',
+        data: {
+          workspaceId,
+          projectId: commit.projectId,
+          documentUuid: document.documentUuid,
+          commitUuid: commit.uuid,
+          documentLogUuid,
+          response,
+          resolvedContent,
+          parameters,
+          duration: await duration,
+        },
+      }),
+    ),
     resolvedContent: result.value,
     documentLogUuid,
   })
diff --git a/packages/core/src/services/documentLogs/_createDocumentLogQuery.ts b/packages/core/src/services/documentLogs/_createDocumentLogQuery.ts
new file mode 100644
index 000000000..68132a666
--- /dev/null
+++ b/packages/core/src/services/documentLogs/_createDocumentLogQuery.ts
@@ -0,0 +1,52 @@
+import { eq, isNotNull, or, sum } from 'drizzle-orm'
+
+import { Commit } from '../../browser'
+import { database } from '../../client'
+import { DocumentLogsRepository } from '../../repositories/documentLogsRepository'
+import { commits, providerLogs } from '../../schema'
+
+export function createDocumentLogQuery(workspaceId: number, db = database) {
+  const documentLogsScope = new DocumentLogsRepository(workspaceId, db)
+  const scope = documentLogsScope.scope
+
+  const aggregatedFieldsSubQuery = db
+    .select({
+      id: scope.id,
+      tokens: sum(providerLogs.tokens).mapWith(Number).as('tokens'),
+      durationInMs: sum(providerLogs.duration)
+        .mapWith(Number)
+        .as('duration_in_ms'),
+      costInMillicents: sum(providerLogs.cost_in_millicents)
+        .mapWith(Number)
+        .as('cost_in_millicents'),
+    })
+    .from(scope)
+    .innerJoin(providerLogs, eq(providerLogs.documentLogUuid, scope.uuid))
+    .groupBy(scope.id)
+    .as('aggregatedFieldsSubQuery')
+
+  return {
+    scope,
+    aggregatedFieldsSubQuery,
+    baseQuery: db
+      .select({
+        ...scope._.selectedFields,
+        commit: commits,
+        tokens: aggregatedFieldsSubQuery.tokens,
+        durationInMs: aggregatedFieldsSubQuery.durationInMs,
+        costInMillicents: aggregatedFieldsSubQuery.costInMillicents,
+      })
+      .from(scope)
+      .innerJoin(commits, eq(commits.id, scope.commitId))
+      .innerJoin(
+        aggregatedFieldsSubQuery,
+        eq(aggregatedFieldsSubQuery.id, scope.id),
+      ),
+  }
+}
+
+export function getCommitFilter(draft?: Commit) {
+  return draft
+    ? or(isNotNull(commits.mergedAt), eq(commits.id, draft.id))
+    : isNotNull(commits.mergedAt)
+}
diff --git a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts
new file mode 100644
index 000000000..33406e734
--- /dev/null
+++ b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts
@@ -0,0 +1,109 @@
+import { beforeEach, describe, expect, it } from 'vitest'
+
+import {
+  DocumentLog,
+  LogSources,
+  ProviderApiKey,
+  ProviderLog,
+  Providers,
+  Workspace,
+} from '../../browser'
+import { NotFoundError } from '../../lib'
+import * as factories from '../../tests/factories'
+import { computeDocumentLogWithMetadata } from './computeDocumentLogWithMetadata'
+
+describe('computeDocumentLogWithMetadata', () => {
+  let documentLog: DocumentLog
+  let workspace: Workspace
+  let provider: ProviderApiKey
+  let providerLogs: ProviderLog[]
+
+  beforeEach(async () => {
+    const setup = await factories.createProject()
+    workspace = setup.workspace
+    provider = await factories.createProviderApiKey({
+      workspace,
+      type: Providers.OpenAI,
+      name: 'foo',
+      user: setup.user,
+    })
+    const { commit } = await factories.createDraft({
+      project: setup.project,
+      user: setup.user,
+    })
+    const { documentVersion } = await factories.createDocumentVersion({
+      commit,
+      content: `
+      ---
+      provider: ${provider.name}
+      model: 'gpt-4o-mini'
+      ---
+      `,
+    })
+    const { documentLog: dl, providerLogs: _providerLogs } =
+      await factories.createDocumentLog({
+        document: documentVersion,
+        commit,
+      })
+
+    documentLog = dl
+    providerLogs = _providerLogs
+  })
+
+  it('returns DocumentLogWithMetadata when successful', async () => {
+    // Create provider logs for the document log
+    const providerLog1 = await factories.createProviderLog({
+      documentLogUuid: documentLog.uuid,
+      providerId: provider.id,
+      providerType: provider.provider,
+      tokens: 100,
+      duration: 1000,
+      costInMillicents: 1250,
+    })
+    const providerLog2 = await factories.createProviderLog({
+      documentLogUuid: documentLog.uuid,
+      providerId: provider.id,
+      providerType: provider.provider,
+      tokens: 150,
+      duration: 1500,
+      costInMillicents: 1250,
+    })
+
+    const totalProviderLogs = [providerLog1, providerLog2, ...providerLogs]
+
+    const result = await computeDocumentLogWithMetadata(documentLog)
+
+    expect(result.ok).toBe(true)
+    if (result.ok) {
+      expect(result.value).toMatchObject({
+        id: documentLog.id,
+        uuid: documentLog.uuid,
+        documentUuid: documentLog.documentUuid,
+        tokens: totalProviderLogs.reduce((acc, log) => acc + log.tokens, 0),
+        durationInMs: totalProviderLogs.reduce(
+          (acc, log) => acc + log.duration,
+          0,
+        ),
+        costInMillicents: totalProviderLogs.reduce(
+          (acc, log) => acc + log.cost_in_millicents,
+          0,
+        ),
+      })
+    }
+  })
+
+  it('returns NotFoundError when document log is not found', async () => {
+    // Create a non-existent document log
+    const nonExistentDocumentLog: DocumentLog = {
+      ...documentLog,
+      id: 999999,
+      uuid: 'non-existent-uuid',
+    }
+
+    const result = await computeDocumentLogWithMetadata(nonExistentDocumentLog)
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toBeInstanceOf(NotFoundError)
+    expect(result.error?.message).toBe('Document log not found')
+  })
+})
diff --git a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.ts b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.ts
new file mode 100644
index 000000000..f147501e1
--- /dev/null
+++ b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.ts
@@ -0,0 +1,28 @@
+import { and, eq } from 'drizzle-orm'
+
+import { DocumentLog } from '../../browser'
+import { database } from '../../client'
+import { findWorkspaceFromDocumentLog } from '../../data-access'
+import { NotFoundError, Result, TypedResult } from '../../lib'
+import { DocumentLogWithMetadata } from '../../repositories/documentLogsRepository'
+import { createDocumentLogQuery } from './_createDocumentLogQuery'
+
+export async function computeDocumentLogWithMetadata(
+  documentLog: DocumentLog,
+  db = database,
+): Promise<TypedResult<DocumentLogWithMetadata, Error>> {
+  const workspace = await findWorkspaceFromDocumentLog(documentLog, db)
+  if (!workspace) return Result.error(new NotFoundError('Workspace not found'))
+
+  const { scope, baseQuery } = createDocumentLogQuery(workspace.id, db)
+
+  const result = await baseQuery
+    .where(and(eq(scope.id, documentLog.id)))
+    .limit(1)
+
+  if (result.length === 0) {
+    return Result.error(new NotFoundError('Document log not found'))
+  }
+
+  return Result.ok(result[0]!)
+}
diff --git a/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts
new file mode 100644
index 000000000..ec6c3de29
--- /dev/null
+++ b/packages/core/src/services/documentLogs/computeDocumentLogsWithMetadata.ts
@@ -0,0 +1,31 @@
+import { and, desc, eq } from 'drizzle-orm'
+
+import { Commit } from '../../browser'
+import { database } from '../../client'
+import { Result, TypedResult } from '../../lib'
+import { DocumentLogWithMetadata } from '../../repositories/documentLogsRepository'
+import {
+  createDocumentLogQuery,
+  getCommitFilter,
+} from './_createDocumentLogQuery'
+
+export async function computeDocumentLogsWithMetadata(
+  {
+    workspaceId,
+    documentUuid,
+    draft,
+  }: {
+    workspaceId: number
+    documentUuid: string
+    draft?: Commit
+  },
+  db = database,
+): Promise<TypedResult<DocumentLogWithMetadata[], Error>> {
+  const { scope, baseQuery } = createDocumentLogQuery(workspaceId, db)
+
+  const result = await baseQuery
+    .where(and(eq(scope.documentUuid, documentUuid), getCommitFilter(draft)))
+    .orderBy(desc(scope.createdAt))
+
+  return Result.ok(result)
+}
diff --git a/packages/core/src/services/documentLogs/index.ts b/packages/core/src/services/documentLogs/index.ts
index 19af944f4..7f8a6f4b1 100644
--- a/packages/core/src/services/documentLogs/index.ts
+++ b/packages/core/src/services/documentLogs/index.ts
@@ -1,2 +1,4 @@
 export * from './create'
 export * from './addMessages'
+export * from './computeDocumentLogWithMetadata'
+export * from './computeDocumentLogsWithMetadata'
diff --git a/packages/core/src/services/evaluationResults/create.ts b/packages/core/src/services/evaluationResults/create.ts
index f916bd666..7100af4d8 100644
--- a/packages/core/src/services/evaluationResults/create.ts
+++ b/packages/core/src/services/evaluationResults/create.ts
@@ -1,6 +1,7 @@
 import {
   DocumentLog,
   Evaluation,
+  EvaluationDto,
   EvaluationResult,
   ProviderLog,
 } from '../../browser'
@@ -9,7 +10,7 @@ import { Result, Transaction } from '../../lib'
 import { evaluationResults } from '../../schema'
 
 export type CreateEvaluationResultProps = {
-  evaluation: Evaluation
+  evaluation: Evaluation | EvaluationDto
   documentLog: DocumentLog
   providerLog: ProviderLog
   result: string
diff --git a/packages/core/src/services/evaluations/index.ts b/packages/core/src/services/evaluations/index.ts
index 6e84af5d4..308e81283 100644
--- a/packages/core/src/services/evaluations/index.ts
+++ b/packages/core/src/services/evaluations/index.ts
@@ -3,3 +3,4 @@ export * from './update'
 export * from './destroy'
 export * from './compiler'
 export * from './connect'
+export * from './run'
diff --git a/packages/core/src/services/evaluations/run.ts b/packages/core/src/services/evaluations/run.ts
new file mode 100644
index 000000000..139eecc89
--- /dev/null
+++ b/packages/core/src/services/evaluations/run.ts
@@ -0,0 +1,80 @@
+import { createChain, readMetadata } from '@latitude-data/compiler'
+
+import { DocumentLog, EvaluationDto, LogSources } from '../../browser'
+import { database } from '../../client'
+import { findLastProviderLogFromDocumentLogUuid } from '../../data-access'
+import { publisher } from '../../events/publisher'
+import { NotFoundError, Result } from '../../lib'
+import { runChain } from '../chains/run'
+import { computeDocumentLogWithMetadata } from '../documentLogs'
+import { buildProviderApikeysMap } from '../providerApiKeys/buildMap'
+import { formatContext, formatConversation } from '../providerLogs'
+
+export const runEvaluation = async (
+  {
+    documentLog,
+    evaluation,
+  }: {
+    documentLog: DocumentLog
+    evaluation: EvaluationDto
+  },
+  db = database,
+) => {
+  const lastProviderLog = await findLastProviderLogFromDocumentLogUuid(
+    documentLog.uuid,
+    db,
+  )
+  if (!lastProviderLog) {
+    return Result.error(
+      new NotFoundError(
+        `ProviderLog not found with documentLogUuid ${documentLog.uuid}`,
+      ),
+    )
+  }
+
+  const rezult = await computeDocumentLogWithMetadata(documentLog)
+  if (rezult.error) return rezult
+  const documentLogWithMetadata = rezult.value
+
+  // TODO: This will need to become polymorphic in the future given different
+  // types of evaluations
+  const metadata = await readMetadata({ prompt: documentLog.resolvedContent })
+  const chain = createChain({
+    prompt: evaluation.metadata.prompt,
+    parameters: {
+      messages: formatConversation(lastProviderLog),
+      context: formatContext(lastProviderLog),
+      response: lastProviderLog.responseText,
+      prompt: documentLog.resolvedContent,
+      parameters: documentLog.parameters,
+      config: metadata.config,
+      duration: documentLogWithMetadata.durationInMs,
+      cost: documentLogWithMetadata.costInMillicents
+        ? documentLogWithMetadata.costInMillicents * 1000
+        : 0,
+    },
+  })
+
+  const result = await runChain({
+    chain,
+    source: LogSources.Evaluation,
+    apikeys: await buildProviderApikeysMap({
+      workspaceId: evaluation.workspaceId,
+    }),
+  })
+  if (result.error) return result
+
+  result.value.response.then((response) => {
+    publisher.publishLater({
+      type: 'evaluationRun',
+      data: {
+        evaluationId: evaluation.id,
+        documentLogUuid: documentLog.uuid,
+        providerLogUuid: lastProviderLog.uuid,
+        response,
+      },
+    })
+  })
+
+  return result
+}
diff --git a/packages/core/src/services/providerLogs/create.ts b/packages/core/src/services/providerLogs/create.ts
index c70041728..06b888a65 100644
--- a/packages/core/src/services/providerLogs/create.ts
+++ b/packages/core/src/services/providerLogs/create.ts
@@ -24,6 +24,7 @@ export type CreateProviderLogProps = {
   source: LogSources
   apiKeyId?: number
   documentLogUuid?: string
+  costInMillicents?: number
 }
 
 export async function createProviderLog(
@@ -42,11 +43,16 @@ export async function createProviderLog(
     apiKeyId,
     documentLogUuid,
     generatedAt,
+    costInMillicents,
   }: CreateProviderLogProps,
   db = database,
 ) {
   return await Transaction.call<ProviderLog>(async (trx) => {
-    const cost = estimateCost({ provider: providerType, model, usage })
+    const cost =
+      costInMillicents ??
+      Math.floor(
+        estimateCost({ provider: providerType, model, usage }) * 100_000,
+      )
     const inserts = await trx
       .insert(providerLogs)
       .values({
@@ -60,7 +66,7 @@ export async function createProviderLog(
         responseText,
         toolCalls,
         tokens: isNaN(usage.totalTokens) ? 0 : (usage.totalTokens ?? 0),
-        cost_in_millicents: Math.floor(cost * 100_000),
+        cost_in_millicents: cost,
         duration,
         source,
         apiKeyId,
diff --git a/packages/core/src/services/providerLogs/formatForEvaluation.ts b/packages/core/src/services/providerLogs/formatForEvaluation.ts
new file mode 100644
index 000000000..7eb76b0a9
--- /dev/null
+++ b/packages/core/src/services/providerLogs/formatForEvaluation.ts
@@ -0,0 +1,45 @@
+import { MessageRole } from '@latitude-data/compiler'
+
+import { Message, ProviderLog } from '../../browser'
+
+export function formatConversation(providerLog: ProviderLog) {
+  const messages: Message[] = [...(providerLog.messages || [])]
+
+  // Add the response as an assistant message if it exists
+  if (providerLog.responseText) {
+    messages.push({
+      role: MessageRole.assistant,
+      content: providerLog.responseText,
+      toolCalls: providerLog.toolCalls,
+    })
+  }
+
+  return formatMessages(messages)
+}
+
+export function formatContext(providerLog: ProviderLog) {
+  return formatMessages(providerLog.messages)
+}
+
+function formatMessages(messages: Message[]) {
+  const filterMessages = (role: MessageRole) =>
+    messages.filter((m) => m.role === role)
+
+  const formatRoleMessages = (role: MessageRole) => {
+    const roleMessages = filterMessages(role)
+    return {
+      all: roleMessages,
+      first: roleMessages[0] || null,
+      last: roleMessages[roleMessages.length - 1] || null,
+    }
+  }
+
+  return {
+    all: messages,
+    first: messages[0] || null,
+    last: messages[messages.length - 1] || null,
+    user: formatRoleMessages(MessageRole.user),
+    system: formatRoleMessages(MessageRole.system),
+    assistant: formatRoleMessages(MessageRole.assistant),
+  }
+}
diff --git a/packages/core/src/services/providerLogs/index.ts b/packages/core/src/services/providerLogs/index.ts
index f756a8bb3..aa5ce14c7 100644
--- a/packages/core/src/services/providerLogs/index.ts
+++ b/packages/core/src/services/providerLogs/index.ts
@@ -1 +1,2 @@
 export * from './create'
+export * from './formatForEvaluation'
diff --git a/packages/core/src/tests/factories/index.ts b/packages/core/src/tests/factories/index.ts
index 80edcff3f..f82cd4d4d 100644
--- a/packages/core/src/tests/factories/index.ts
+++ b/packages/core/src/tests/factories/index.ts
@@ -10,3 +10,4 @@ export * from './evaluationResults'
 export * from './helpers'
 export * from './evaluationTemplates'
 export * from './datasets'
+export * from './providerLogs'
diff --git a/packages/core/src/tests/factories/providerLogs.ts b/packages/core/src/tests/factories/providerLogs.ts
new file mode 100644
index 000000000..f72be748e
--- /dev/null
+++ b/packages/core/src/tests/factories/providerLogs.ts
@@ -0,0 +1,44 @@
+import { faker } from '@faker-js/faker'
+import { v4 as uuid } from 'uuid'
+
+import { LogSources, ProviderLog, Providers } from '../../browser'
+import { createProviderLog as createProviderLogService } from '../../services/providerLogs'
+
+export type IProviderLogData = {
+  documentLogUuid: string
+  providerId: number
+  providerType: Providers
+  model?: string
+  tokens?: number
+  duration?: number
+  source?: LogSources
+  costInMillicents?: number
+}
+
+export async function createProviderLog(
+  data: IProviderLogData,
+): Promise<ProviderLog> {
+  const providerLog = await createProviderLogService({
+    uuid: uuid(),
+    generatedAt: new Date(),
+    documentLogUuid: data.documentLogUuid,
+    providerId: data.providerId,
+    providerType: data.providerType,
+    model: data.model ?? faker.lorem.word(),
+    config: {},
+    messages: [],
+    responseText: faker.lorem.sentence(),
+    toolCalls: [],
+    usage: {
+      promptTokens: data.tokens ?? faker.number.int({ min: 10, max: 100 }),
+      completionTokens: data.tokens ?? faker.number.int({ min: 10, max: 100 }),
+      totalTokens: data.tokens ?? faker.number.int({ min: 20, max: 200 }),
+    },
+    duration: data.duration ?? faker.number.int({ min: 100, max: 5000 }),
+    source: data.source ?? LogSources.Playground,
+    costInMillicents:
+      data.costInMillicents ?? faker.number.int({ min: 100, max: 5000 }),
+  }).then((r) => r.unwrap())
+
+  return providerLog
+}
diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts
index 8947a4303..84c4545d8 100644
--- a/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts
+++ b/packages/jobs/src/job-definitions/batchEvaluations/runBatchEvaluationJob.ts
@@ -12,7 +12,6 @@ import { previewDataset } from '@latitude-data/core/services/datasets/preview'
 import { Job } from 'bullmq'
 
 import { setupJobs } from '../..'
-import { Queues } from '../../constants'
 import { connection } from '../../utils/connection'
 import { ProgressTracker } from '../../utils/progressTracker'
 
@@ -26,69 +25,65 @@ type RunBatchEvaluationJobParams = {
   batchId?: string
 }
 
-export const runBatchEvaluationJob = {
-  name: 'runBatchEvaluationJob',
-  queue: Queues.defaultQueue,
-  handler: async (job: Job<RunBatchEvaluationJobParams>) => {
-    const {
-      evaluation,
-      dataset,
-      document,
-      fromLine = 0,
-      toLine,
-      parametersMap,
-      batchId = randomUUID(),
-    } = job.data
-    const workspace = await findWorkspaceFromDocument(document)
-    if (!workspace) throw new NotFoundError('Workspace not found')
+export const runBatchEvaluationJob = async (
+  job: Job<RunBatchEvaluationJobParams>,
+) => {
+  const {
+    evaluation,
+    dataset,
+    document,
+    fromLine = 0,
+    toLine,
+    parametersMap,
+    batchId = randomUUID(),
+  } = job.data
+  const workspace = await findWorkspaceFromDocument(document)
+  if (!workspace) throw new NotFoundError('Workspace not found')
 
-    const commit = await new CommitsRepository(workspace.id)
-      .find(document.commitId)
-      .then((r) => r.unwrap())
-    const fileMetadata = dataset.fileMetadata
-    // TODO: use streaming instead of this service in order to avoid loading the
-    // whole dataset in memory
-    const result = await previewDataset({
-      dataset,
-      fromLine,
-      toLine: toLine || fileMetadata.rowCount,
-    }).then((r) => r.unwrap())
+  const commit = await new CommitsRepository(workspace.id)
+    .find(document.commitId)
+    .then((r) => r.unwrap())
+  const fileMetadata = dataset.fileMetadata
+  // TODO: use streaming instead of this service in order to avoid loading the
+  // whole dataset in memory
+  const result = await previewDataset({
+    dataset,
+    fromLine,
+    toLine: toLine || fileMetadata.rowCount,
+  }).then((r) => r.unwrap())
 
-    const { rows } = result
+  const { rows } = result
 
-    const parameters = rows.map((row) => {
-      return Object.fromEntries(
-        Object.entries(parametersMap!).map(([key, index]) => [
-          key,
-          row[index]!,
-        ]),
-      )
-    })
+  const parameters = rows.map((row) => {
+    return Object.fromEntries(
+      Object.entries(parametersMap!).map(([key, index]) => [key, row[index]!]),
+    )
+  })
 
-    const progressTracker = new ProgressTracker(connection, batchId)
+  const progressTracker = new ProgressTracker(connection, batchId)
 
-    if (job.attemptsMade === 0) {
-      await progressTracker.initializeProgress(parameters.length)
-    }
+  if (job.attemptsMade === 0) {
+    await progressTracker.initializeProgress(parameters.length)
+  }
 
-    const { enqueued } = await progressTracker.getProgress()
-    const queues = setupJobs()
+  const { enqueued } = await progressTracker.getProgress()
+  const queues = setupJobs()
 
-    // Enqueue runDocumentJob for each set of parameters, starting from the last
-    // enqueued job. This allows us to resume the batch if the job fails.
-    for (let i = enqueued; i < parameters.length; i++) {
-      await queues.defaultQueue.jobs.enqueueRunDocumentJob({
-        workspaceId: workspace.id,
-        document,
-        commit,
-        parameters: parameters[i]!,
-        evaluationId: evaluation.id,
-        batchId,
-      })
+  // Enqueue runDocumentJob for each set of parameters, starting from the last
+  // enqueued job. This allows us to resume the batch if the job fails.
+  for (let i = enqueued; i < parameters.length; i++) {
+    await queues.defaultQueue.jobs.enqueueRunDocumentJob({
+      workspaceId: workspace.id,
+      documentUuid: document.documentUuid,
+      commitUuid: commit.uuid,
+      projectId: commit.projectId,
+      parameters: parameters[i]!,
+      evaluationId: evaluation.id,
+      batchId,
+    })
 
-      await progressTracker.incrementEnqueued()
-    }
+    await progressTracker.incrementEnqueued()
+  }
 
-    return { batchId }
-  },
+  return { batchId }
 }
diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.ts
index 6b3c12674..cae0011de 100644
--- a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.ts
+++ b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.ts
@@ -1,61 +1,73 @@
+import { LogSources } from '@latitude-data/core/browser'
 import {
-  Commit,
-  DocumentVersion,
-  LogSources,
-} from '@latitude-data/core/browser'
+  CommitsRepository,
+  DocumentVersionsRepository,
+} from '@latitude-data/core/repositories'
 import { runDocumentAtCommit } from '@latitude-data/core/services/commits/runDocumentAtCommit'
 import { env } from '@latitude-data/env'
 import { Job } from 'bullmq'
 
 import { setupJobs } from '../../'
-import { Queues } from '../../constants'
 import { connection } from '../../utils/connection'
 import { ProgressTracker } from '../../utils/progressTracker'
 
 type RunDocumentJobData = {
   workspaceId: number
-  document: DocumentVersion
-  commit: Commit
+  documentUuid: string
+  commitUuid: string
+  projectId: number
   parameters: Record<string, unknown>
   evaluationId: number
   batchId: string
 }
 
-export const runDocumentJob = {
-  name: 'runDocumentJob',
-  queue: Queues.defaultQueue,
-  handler: async (job: Job<RunDocumentJobData>) => {
-    const { workspaceId, document, commit, parameters, evaluationId, batchId } =
-      job.data
-
-    const progressTracker = new ProgressTracker(connection, batchId)
-
-    try {
-      const result = await runDocumentAtCommit({
-        workspaceId,
-        document,
-        commit,
-        parameters,
-        source: LogSources.Evaluation,
-      }).then((r) => r.unwrap())
-
-      await result.response
-
-      const queues = setupJobs()
-
-      // Enqueue the evaluation job
-      await queues.defaultQueue.jobs.enqueueRunEvaluationJob({
-        documentLogUuid: result.documentLogUuid,
-        evaluationId,
-        batchId,
-      })
-    } catch (error) {
-      if (env.NODE_ENV !== 'production') {
-        console.error(error)
-      }
-
-      await progressTracker.incrementErrors()
-      await progressTracker.decrementTotal()
+export const runDocumentJob = async (job: Job<RunDocumentJobData>) => {
+  const {
+    workspaceId,
+    documentUuid,
+    commitUuid,
+    projectId,
+    parameters,
+    evaluationId,
+    batchId,
+  } = job.data
+
+  const progressTracker = new ProgressTracker(connection, batchId)
+
+  try {
+    const documentsScope = new DocumentVersionsRepository(workspaceId)
+    const commitsScope = new CommitsRepository(workspaceId)
+    const document = await documentsScope
+      .getDocumentAtCommit({ projectId, documentUuid, commitUuid })
+      .then((r) => r.unwrap())
+    const commit = await commitsScope
+      .getCommitByUuid({ projectId, uuid: commitUuid })
+      .then((r) => r.unwrap())
+    const result = await runDocumentAtCommit({
+      workspaceId,
+      document,
+      commit,
+      parameters,
+      source: LogSources.Evaluation,
+    }).then((r) => r.unwrap())
+
+    await result.response
+
+    const queues = setupJobs()
+
+    // Enqueue the evaluation job
+    await queues.defaultQueue.jobs.enqueueRunEvaluationJob({
+      workspaceId,
+      documentLogUuid: result.documentLogUuid,
+      evaluationId,
+      batchId,
+    })
+  } catch (error) {
+    if (env.NODE_ENV !== 'production') {
+      console.error(error)
     }
-  },
+
+    await progressTracker.incrementErrors()
+    await progressTracker.decrementTotal()
+  }
 }
diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts
index 1d1467fab..f023d3007 100644
--- a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts
+++ b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts
@@ -1,45 +1,48 @@
+import {
+  DocumentLogsRepository,
+  EvaluationsRepository,
+} from '@latitude-data/core/repositories'
+import { runEvaluation } from '@latitude-data/core/services/evaluations/run'
 import { env } from '@latitude-data/env'
 import { Job } from 'bullmq'
 
-import { Queues } from '../../constants'
 import { connection } from '../../utils/connection'
 import { ProgressTracker } from '../../utils/progressTracker'
 
 type RunEvaluationJobData = {
+  workspaceId: number
   documentLogUuid: string
   evaluationId: number
   batchId: string
 }
 
-export const runEvaluationJob = {
-  name: 'runEvaluationJob',
-  queue: Queues.defaultQueue,
-  handler: async (job: Job<RunEvaluationJobData>) => {
-    const { batchId } = job.data
-
-    const progressTracker = new ProgressTracker(connection, batchId)
-
-    try {
-      // Mock implementation of evaluation logic
-      const mockEvaluationLogic = async () => {
-        // Simulate some processing time
-        await new Promise((resolve) => setTimeout(resolve, 2000))
-
-        // Randomly throw an error 5% of the time
-        if (Math.random() < 0.05) {
-          throw new Error('Random evaluation error')
-        }
-      }
-
-      await mockEvaluationLogic()
-      await progressTracker.incrementCompleted()
-    } catch (error) {
-      if (env.NODE_ENV !== 'production') {
-        console.error('Error in runEvaluationJob:', error)
-      }
-
-      await progressTracker.incrementErrors()
-      await progressTracker.decrementTotal()
+export const runEvaluationJob = async (job: Job<RunEvaluationJobData>) => {
+  const { workspaceId, batchId, documentLogUuid, evaluationId } = job.data
+
+  const progressTracker = new ProgressTracker(connection, batchId)
+  const documentLogsScope = new DocumentLogsRepository(workspaceId)
+  const evaluationsScope = new EvaluationsRepository(workspaceId)
+
+  try {
+    const documentLog = await documentLogsScope
+      .findByUuid(documentLogUuid)
+      .then((r) => r.unwrap())
+    const evaluation = await evaluationsScope
+      .find(evaluationId)
+      .then((r) => r.unwrap())
+
+    await runEvaluation({
+      documentLog,
+      evaluation,
+    })
+
+    await progressTracker.incrementCompleted()
+  } catch (error) {
+    if (env.NODE_ENV !== 'production') {
+      console.error('Error in runEvaluationJob:', error)
     }
-  },
+
+    await progressTracker.incrementErrors()
+    await progressTracker.decrementTotal()
+  }
 }
diff --git a/packages/jobs/src/job-definitions/documentLogs/createJob.ts b/packages/jobs/src/job-definitions/documentLogs/createJob.ts
deleted file mode 100644
index fa09166cc..000000000
--- a/packages/jobs/src/job-definitions/documentLogs/createJob.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import {
-  createDocumentLog,
-  CreateDocumentLogProps,
-} from '@latitude-data/core/services/documentLogs/create'
-import { Job } from 'bullmq'
-
-export type CreateDocumentLogJobData = CreateDocumentLogProps
-
-export const createDocumentLogJob = async (
-  job: Job<CreateDocumentLogJobData>,
-) => {
-  await createDocumentLog(job.data).then((r) => r.unwrap())
-}
diff --git a/packages/jobs/src/job-definitions/index.ts b/packages/jobs/src/job-definitions/index.ts
index b19cc900a..ac3a96b07 100644
--- a/packages/jobs/src/job-definitions/index.ts
+++ b/packages/jobs/src/job-definitions/index.ts
@@ -1,10 +1,9 @@
 import { LatitudeEvent } from '@latitude-data/core/events/handlers/index'
 
 import { Jobs, Queues } from '../constants'
-import { CreateDocumentLogJobData } from './documentLogs/createJob'
 
+// TODO: fix these types
 export type JobDataMap = {
-  [Jobs.createDocumentLogJob]: CreateDocumentLogJobData
   [Jobs.publishEventJob]: LatitudeEvent
 }
 
@@ -22,5 +21,3 @@ export type JobDefinition = {
     [K in Jobs]: JobSpec<K>
   }
 }
-
-export * from './documentLogs/createJob'
diff --git a/packages/jobs/src/queues/index.ts b/packages/jobs/src/queues/index.ts
index 892759556..07dcc2fe6 100644
--- a/packages/jobs/src/queues/index.ts
+++ b/packages/jobs/src/queues/index.ts
@@ -2,7 +2,7 @@ import { EventHandlers } from '@latitude-data/core/events/handlers/index'
 import { Job, JobsOptions, Queue, QueueEvents } from 'bullmq'
 
 import { Jobs, Queues } from '../constants'
-import { createDocumentLogJob, JobDefinition } from '../job-definitions'
+import { JobDefinition } from '../job-definitions'
 import { runBatchEvaluationJob } from '../job-definitions/batchEvaluations/runBatchEvaluationJob'
 import { runDocumentJob } from '../job-definitions/batchEvaluations/runDocumentJob'
 import { runEvaluationJob } from '../job-definitions/batchEvaluations/runEvaluationJob'
@@ -67,7 +67,6 @@ export const QUEUES = {
   [Queues.defaultQueue]: {
     name: Queues.defaultQueue,
     jobs: [
-      createDocumentLogJob,
       runBatchEvaluationJob,
       runDocumentJob,
       runEvaluationJob,
diff --git a/packages/jobs/src/workers/_shared.ts b/packages/jobs/src/workers/_shared.ts
index 7811bcf4a..450e210bf 100644
--- a/packages/jobs/src/workers/_shared.ts
+++ b/packages/jobs/src/workers/_shared.ts
@@ -1,3 +1,4 @@
+import { env } from '@latitude-data/env'
 import { Processor } from 'bullmq'
 
 import { Queues } from '../constants'
@@ -11,7 +12,15 @@ export const buildProcessor =
         await Promise.all(
           QUEUES[q].jobs.map(async (j) => {
             if (j.name === job.name) {
-              await j(job)
+              try {
+                await j(job)
+              } catch (err) {
+                if (env.NODE_ENV !== 'production') {
+                  console.error(err)
+                }
+
+                throw err
+              }
             }
           }),
         )
diff --git a/packages/web-ui/src/ds/atoms/SplitPane/index.tsx b/packages/web-ui/src/ds/atoms/SplitPane/index.tsx
index 8e121cf54..6825e92c3 100644
--- a/packages/web-ui/src/ds/atoms/SplitPane/index.tsx
+++ b/packages/web-ui/src/ds/atoms/SplitPane/index.tsx
@@ -24,7 +24,7 @@ const PaneWrapper = ({
   children,
   isResizable = false,
 }: {
-  width?: number | 'auto'
+  width: number | 'auto'
   isResizable?: boolean
   children: ReactNode
 }) => {