feature: live evaluations (#279)

* feature: live evaluations

* chore: fix broken test

apps/web/src/actions/connectedEvaluations/update.test.ts

@@ -0,0 +1,121 @@
+import { Providers } from '@latitude-data/core/browser'
+import * as factories from '@latitude-data/core/factories'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { updateConnectedEvaluationAction } from './update'
+
+const mocks = vi.hoisted(() => {
+  return {
+    getSession: vi.fn(),
+  }
+})
+
+vi.mock('$/services/auth/getSession', () => ({
+  getSession: mocks.getSession,
+}))
+
+describe('updateConnectedEvaluationAction', () => {
+  let workspace: any
+  let project: any
+  let user: any
+  let connectedEvaluation: any
+
+  beforeEach(async () => {
+    const prompt = factories.helpers.createPrompt({
+      provider: 'Latitude',
+      model: 'gpt-4o',
+    })
+    const setup = await factories.createProject({
+      providers: [{ type: Providers.OpenAI, name: 'Latitude' }],
+      name: 'Default Project',
+      documents: {
+        foo: {
+          content: prompt,
+        },
+      },
+    })
+    workspace = setup.workspace
+    project = setup.project
+    user = setup.user
+
+    // Create a connected evaluation using a factory
+    const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace })
+    const { commit } = await factories.createDraft({ project, user })
+    const { documentLog } = await factories.createDocumentLog({
+      document: setup.documents[0]!,
+      commit,
+    })
+
+    connectedEvaluation = await factories.createConnectedEvaluation({
+      workspace,
+      evaluationUuid: evaluation.uuid,
+      documentUuid: documentLog.documentUuid,
+    })
+  })
+
+  describe('unauthorized', () => {
+    it('errors when the user is not authenticated', async () => {
+      mocks.getSession.mockReturnValue(null)
+
+      const [_, error] = await updateConnectedEvaluationAction({
+        id: connectedEvaluation.id,
+        data: { live: true },
+      })
+
+      expect(error!.name).toEqual('UnauthorizedError')
+    })
+  })
+
+  describe('authorized', () => {
+    beforeEach(() => {
+      mocks.getSession.mockReturnValue({
+        user,
+        workspace: { id: workspace.id, name: workspace.name },
+      })
+    })
+
+    it('successfully updates a connected evaluation', async () => {
+      const [data, error] = await updateConnectedEvaluationAction({
+        id: connectedEvaluation.id,
+        data: { live: true },
+      })
+
+      expect(error).toBeNull()
+      expect(data).toBeDefined()
+      expect(data!.id).toEqual(connectedEvaluation.id)
+      expect(data!.live).toEqual(true)
+    })
+
+    it('returns an error when the connected evaluation is not found', async () => {
+      const [_, error] = await updateConnectedEvaluationAction({
+        id: 9999, // Non-existent ID
+        data: { live: true },
+      })
+
+      expect(error).toBeDefined()
+      expect(error!.name).toEqual('NotFoundError')
+    })
+
+    it('does not update fields that are not provided', async () => {
+      const [data, _] = await updateConnectedEvaluationAction({
+        id: connectedEvaluation.id,
+        data: { live: connectedEvaluation.live }, // Provide the required 'live' field
+      })
+
+      expect(data).toBeDefined()
+      expect(data!.id).toEqual(connectedEvaluation.id)
+      expect(data!.live).toEqual(connectedEvaluation.live) // Should remain unchanged
+    })
+
+    it('handles invalid input data', async () => {
+      const [_, error] = await updateConnectedEvaluationAction({
+        id: connectedEvaluation.id,
+        // @ts-expect-error - Testing invalid input
+        data: { live: 'not a boolean' },
+      })
+
+      expect(error).toBeDefined()
+      expect(error!.name).toEqual('ZodError')
+    })
+  })
+})

apps/web/src/actions/connectedEvaluations/update.ts

@@ -0,0 +1,33 @@
+'use server'
+
+import { ConnectedEvaluationsRepository } from '@latitude-data/core/repositories'
+import { updateConnectedEvaluation } from '@latitude-data/core/services/connectedEvaluations/update'
+import { z } from 'zod'
+
+import { authProcedure } from '../procedures'
+
+export const updateConnectedEvaluationAction = authProcedure
+  .createServerAction()
+  .input(
+    z.object({
+      id: z.number(),
+      data: z.object({
+        live: z.boolean(),
+      }),
+    }),
+  )
+  .handler(async ({ input, ctx }) => {
+    const connectedEvaluationsScope = new ConnectedEvaluationsRepository(
+      ctx.workspace.id,
+    )
+    const connectedEvaluation = await connectedEvaluationsScope
+      .find(input.id)
+      .then((r) => r.unwrap())
+
+    const result = await updateConnectedEvaluation({
+      connectedEvaluation,
+      data: input.data,
+    })
+
+    return result.unwrap()
+  })

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx

@@ -7,7 +7,7 @@ import { EvaluationDto } from '@latitude-data/core/browser'
 import { ConnectedDocumentWithMetadata } from '@latitude-data/core/repositories'
 import { Skeleton, Text } from '@latitude-data/web-ui'
 import { formatCostInMillicents } from '$/app/_lib/formatUtils'
-import useConnectedDocuments from '$/stores/connectedEvaluations'
+import useConnectedDocuments from '$/stores/connectedDocuments'
 
 export function Stat({ label, value }: { label: string; value?: string }) {
   return (

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/LiveEvaluationToggle/index.tsx

@@ -0,0 +1,58 @@
+import { useCallback } from 'react'
+
+import { EvaluationDto } from '@latitude-data/core/browser'
+import {
+  Label,
+  SwitchToogle,
+  useCurrentCommit,
+  useCurrentProject,
+  useToast,
+} from '@latitude-data/web-ui'
+import useConnectedEvaluations from '$/stores/connectedEvaluations'
+
+export default function LiveEvaluationToggle({
+  documentUuid,
+  evaluation,
+}: {
+  documentUuid: string
+  evaluation: EvaluationDto
+}) {
+  const { toast } = useToast()
+  const { commit } = useCurrentCommit()
+  const { project } = useCurrentProject()
+  const { data, update, isUpdating } = useConnectedEvaluations({
+    documentUuid,
+    projectId: project.id,
+    commitUuid: commit.uuid,
+  })
+  const connectedEvaluation = data.find(
+    (ev) => ev.evaluationId === evaluation.id,
+  )
+  const toggleLive = useCallback(async () => {
+    if (!connectedEvaluation) return
+
+    const live = !connectedEvaluation.live
+    const [_, error] = await update({
+      id: connectedEvaluation.id,
+      data: { live },
+    })
+    if (error) return
+
+    toast({
+      title: 'Successfully updated evaluation',
+      description: live ? 'Evaluation is now live' : 'Evaluation is now paused',
+    })
+  }, [connectedEvaluation, update])
+  if (!connectedEvaluation) return null
+
+  return (
+    <div className='flex flex-row gap-2 items-center'>
+      <Label>Evaluate live logs</Label>
+      <SwitchToogle
+        disabled={isUpdating}
+        checked={connectedEvaluation.live}
+        onCheckedChange={toggleLive}
+      />
+    </div>
+  )
+}

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/index.tsx

@@ -5,6 +5,8 @@ import { Button, Icon, TableWithHeader } from '@latitude-data/web-ui'
 import { ROUTES } from '$/services/routes'
 import Link from 'next/link'
 
+import LiveEvaluationToggle from './LiveEvaluationToggle'
+
 export function Actions({
   evaluation,
   projectId,
@@ -24,6 +26,10 @@ export function Actions({
 
   return (
     <div className='flex flex-row items-center gap-4'>
+      <LiveEvaluationToggle
+        documentUuid={documentUuid}
+        evaluation={evaluation}
+      />
       <Link href={ROUTES.evaluations.detail({ uuid: evaluation.uuid }).root}>
         <Button variant='ghost'>
           Go to evaluation <Icon name='externalLink' />

apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.test.ts

@@ -0,0 +1,123 @@
+import { Providers } from '@latitude-data/core/browser'
+import * as factories from '@latitude-data/core/factories'
+import { Result } from '@latitude-data/core/lib/Result'
+import { ConnectedEvaluationsRepository } from '@latitude-data/core/repositories'
+import { NextRequest, NextResponse } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { GET } from './route'
+
+vi.mock('@latitude-data/core/repositories', async (importOriginal) => {
+  const original = await importOriginal()
+  return {
+    // @ts-expect-error
+    ...original,
+    ConnectedEvaluationsRepository: vi.fn(),
+  }
+})
+vi.mock('$/middlewares/authHandler', () => ({
+  authHandler: (fn: Function) => fn,
+}))
+
+describe('GET /api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations', async () => {
+  let workspace: any
+  let documents: any
+  let mockEvaluations: any
+
+  beforeEach(async () => {
+    vi.resetAllMocks()
+
+    const setup = await factories.createProject({
+      providers: [{ type: Providers.OpenAI, name: 'openai' }],
+      documents: {
+        doc1: factories.helpers.createPrompt({
+          provider: 'openai',
+          content: 'foo',
+        }),
+        doc2: factories.helpers.createPrompt({
+          provider: 'openai',
+          content: 'bar',
+        }),
+      },
+    })
+
+    workspace = setup.workspace
+    documents = setup.documents
+
+    mockEvaluations = [
+      await factories.createConnectedEvaluation({
+        workspace,
+        documentUuid: documents[0]!.documentUuid,
+      }),
+      await factories.createConnectedEvaluation({
+        workspace,
+        documentUuid: documents[1]!.documentUuid,
+      }),
+    ]
+  })
+
+  it('should return evaluations for the given document UUID', async () => {
+    const mockFilterByDocumentUuid = vi
+      .fn()
+      .mockResolvedValue(Result.ok(mockEvaluations))
+    vi.mocked(ConnectedEvaluationsRepository).mockImplementation(
+      () =>
+        ({
+          filterByDocumentUuid: mockFilterByDocumentUuid,
+        }) as any,
+    )
+
+    const request = new NextRequest(
+      'http://localhost/api/documents/test-project/test-commit/test-document-uuid/evaluations',
+    )
+    const response = await GET(request, {
+      // @ts-expect-error
+      params: { documentUuid: documents[0]!.documentUuid },
+      workspace,
+    })
+
+    expect(ConnectedEvaluationsRepository).toHaveBeenCalledWith(workspace.id)
+    expect(mockFilterByDocumentUuid).toHaveBeenCalledWith(
+      documents[0]!.documentUuid,
+    )
+    expect(response).toBeInstanceOf(NextResponse)
+    // @ts-expect-error
+    expect((await response.json()).map((ev) => ev.id)).toEqual(
+      // @ts-expect-error
+      mockEvaluations.map((ev) => ev.id),
+    )
+    expect(response.status).toBe(200)
+  })
+
+  it('should handle errors when fetching evaluations fails', async () => {
+    const mockError = new Error('Failed to fetch evaluations')
+    const mockFilterByDocumentUuid = vi
+      .fn()
+      .mockResolvedValue(Result.error(mockError))
+    vi.mocked(ConnectedEvaluationsRepository).mockImplementation(
+      () =>
+        ({
+          filterByDocumentUuid: mockFilterByDocumentUuid,
+        }) as any,
+    )
+
+    const request = new NextRequest(
+      'http://localhost/api/documents/test-project/test-commit/test-document-uuid/evaluations',
+    )
+    const response = await GET(request, {
+      // @ts-expect-error
+      params: { documentUuid: documents[0]!.documentUuid },
+      workspace,
+    })
+
+    expect(ConnectedEvaluationsRepository).toHaveBeenCalledWith(workspace.id)
+    expect(mockFilterByDocumentUuid).toHaveBeenCalledWith(
+      documents[0]!.documentUuid,
+    )
+    expect(response).toBeInstanceOf(NextResponse)
+    expect(response.status).toBe(500)
+    expect(await response.json()).toEqual({
+      message: 'Internal Server Error',
+    })
+  })
+})

apps/web/src/app/api/documents/[projectId]/[commitUuid]/[documentUuid]/evaluations/route.ts

@@ -0,0 +1,30 @@
+import { Workspace } from '@latitude-data/core/browser'
+import { ConnectedEvaluationsRepository } from '@latitude-data/core/repositories'
+import { authHandler } from '$/middlewares/authHandler'
+import { errorHandler } from '$/middlewares/errorHandler'
+import { NextRequest, NextResponse } from 'next/server'
+
+export const GET = errorHandler(
+  authHandler(
+    async (
+      _: NextRequest,
+      {
+        params,
+        workspace,
+      }: {
+        params: {
+          documentUuid: string
+        }
+        workspace: Workspace
+      },
+    ) => {
+      const { documentUuid } = params
+      const scope = new ConnectedEvaluationsRepository(workspace.id)
+      const evaluations = await scope
+        .filterByDocumentUuid(documentUuid)
+        .then((r) => r.unwrap())
+
+      return NextResponse.json(evaluations, { status: 200 })
+    },
+  ),
+)