feature: connect & run evaluations

apps/web/src/actions/documents/create.ts

@@ -18,7 +18,7 @@ export const createDocumentVersionAction = withProject
   .handler(async ({ input, ctx }) => {
     const commitsScope = new CommitsRepository(ctx.project.workspaceId)
     const commit = await commitsScope
-      .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project })
+      .getCommitByUuid({ uuid: input.commitUuid, projectId: ctx.project.id })
       .then((r) => r.unwrap())
 
     const result = await createNewDocument({

apps/web/src/actions/documents/destroyDocumentAction/index.ts

@@ -16,12 +16,13 @@ export const destroyDocumentAction = withProject
   .handler(async ({ input, ctx }) => {
     const commitsScope = new CommitsRepository(ctx.project.workspaceId)
     const commit = await commitsScope
-      .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project })
+      .getCommitByUuid({ uuid: input.commitUuid, projectId: ctx.project.id })
       .then((r) => r.unwrap())
     const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId)
     const document = await docsScope
       .getDocumentAtCommit({
-        commit,
+        commitUuid: input.commitUuid,
+        projectId: ctx.project.id,
         documentUuid: input.documentUuid,
       })
       .then((r) => r.unwrap())

apps/web/src/actions/documents/destroyFolderAction.ts

@@ -14,7 +14,7 @@ export const destroyFolderAction = withProject
   .handler(async ({ input, ctx }) => {
     const commitsScope = new CommitsRepository(ctx.project.workspaceId)
     const commit = await commitsScope
-      .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project })
+      .getCommitByUuid({ uuid: input.commitUuid, projectId: ctx.project.id })
       .then((r) => r.unwrap())
     const result = await destroyFolder({
       path: input.path,

apps/web/src/actions/documents/getDocumentsAtCommitAction.ts

@@ -13,7 +13,7 @@ export const getDocumentsAtCommitAction = withProject
   .input(z.object({ commitUuid: z.string() }))
   .handler(async ({ input, ctx }) => {
     const commit = await new CommitsRepository(ctx.project.workspaceId)
-      .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project })
+      .getCommitByUuid({ uuid: input.commitUuid, projectId: ctx.project.id })
       .then((r) => r.unwrap())
     const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId)
     const result = await docsScope.getDocumentsAtCommit(commit)

apps/web/src/actions/documents/updateContent.ts

@@ -22,12 +22,13 @@ export const updateDocumentContentAction = withProject
   .handler(async ({ input, ctx }) => {
     const commitsScope = new CommitsRepository(ctx.project.workspaceId)
     const commit = await commitsScope
-      .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project })
+      .getCommitByUuid({ uuid: input.commitUuid, projectId: ctx.project.id })
       .then((r) => r.unwrap())
     const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId)
     const document = await docsScope
       .getDocumentAtCommit({
-        commit,
+        commitUuid: input.commitUuid,
+        projectId: ctx.project.id,
         documentUuid: input.documentUuid,
       })
       .then((r) => r.unwrap())

apps/web/src/actions/evaluationTemplates/fetch.ts

@@ -0,0 +1,12 @@
+'use server'
+
+import { findAllEvaluationTemplates } from '@latitude-data/core/data-access'
+
+import { authProcedure } from '../procedures'
+
+export const fetchEvaluationTemplatesAction = authProcedure
+  .createServerAction()
+  .handler(async () => {
+    const result = await findAllEvaluationTemplates()
+    return result.unwrap()
+  })

apps/web/src/actions/evaluations/connect.test.ts

@@ -0,0 +1,146 @@
+import { randomUUID } from 'crypto'
+
+import {
+  Commit,
+  DocumentVersion,
+  Project,
+  ProviderApiKey,
+  Providers,
+  User,
+  Workspace,
+} from '@latitude-data/core/browser'
+import * as factories from '@latitude-data/core/factories'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { connectEvaluationsAction } from './connect'
+
+const mocks = vi.hoisted(() => {
+  return {
+    getSession: vi.fn(),
+  }
+})
+
+vi.mock('$/services/auth/getSession', () => ({
+  getSession: mocks.getSession,
+}))
+
+describe('connectEvaluationsAction', () => {
+  describe('unauthorized', () => {
+    it('errors when the user is not authenticated', async () => {
+      const [_, error] = await connectEvaluationsAction({
+        projectId: 1,
+        documentUuid: 'fake-document-uuid',
+        commitUuid: 'fake-commit-uuid',
+        templateIds: [1],
+        evaluationUuids: ['fake-evaluation-uuid'],
+      })
+
+      expect(error!.name).toEqual('UnauthorizedError')
+    })
+  })
+
+  describe('authorized', () => {
+    let workspace: Workspace,
+      user: User,
+      document: DocumentVersion,
+      commit: Commit,
+      provider: ProviderApiKey,
+      project: Project
+
+    beforeEach(async () => {
+      const setup = await factories.createProject({
+        documents: { 'test-doc': 'Test content' },
+      })
+      workspace = setup.workspace
+      user = setup.user
+      document = setup.documents[0]!
+      commit = setup.commit
+      project = setup.project
+
+      provider = await factories.createProviderApiKey({
+        workspace,
+        type: Providers.OpenAI,
+        name: 'Test Provider',
+        user,
+      })
+
+      mocks.getSession.mockReturnValue({
+        user,
+        workspace: { id: workspace.id, name: workspace.name },
+      })
+    })
+
+    it('connects evaluations and templates to a document', async () => {
+      const evaluation = await factories.createEvaluation({
+        provider,
+        name: 'Test Evaluation',
+      })
+
+      const template = await factories.createEvaluationTemplate({
+        name: 'Test Template',
+        description: 'Test description',
+        prompt: 'Test prompt',
+      })
+
+      const [result, error] = await connectEvaluationsAction({
+        projectId: project.id,
+        documentUuid: document.documentUuid,
+        commitUuid: commit.uuid,
+        templateIds: [template.id],
+        evaluationUuids: [evaluation.uuid],
+      })
+
+      expect(error).toBeNull()
+      expect(result).toHaveLength(2)
+      expect(result).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            documentUuid: document.documentUuid,
+            evaluationId: evaluation.id,
+          }),
+          expect.objectContaining({
+            documentUuid: document.documentUuid,
+            evaluationId: expect.any(Number),
+          }),
+        ]),
+      )
+    })
+
+    it('returns an empty array when no evaluations or templates are provided', async () => {
+      const [result, error] = await connectEvaluationsAction({
+        projectId: project.id,
+        documentUuid: document.documentUuid,
+        commitUuid: commit.uuid,
+        templateIds: [],
+        evaluationUuids: [],
+      })
+
+      expect(error).toBeNull()
+      expect(result).toHaveLength(0)
+    })
+
+    it('fails when the document does not exist', async () => {
+      const [_, error] = await connectEvaluationsAction({
+        projectId: project.id,
+        documentUuid: 'non-existent-uuid',
+        commitUuid: randomUUID(),
+        templateIds: [],
+        evaluationUuids: [],
+      })
+
+      expect(error!.name).toEqual('NotFoundError')
+    })
+
+    it('fails when the commit does not exist', async () => {
+      const [_, error] = await connectEvaluationsAction({
+        projectId: project.id,
+        documentUuid: document.documentUuid,
+        commitUuid: randomUUID(),
+        templateIds: [],
+        evaluationUuids: [],
+      })
+
+      expect(error!.name).toEqual('NotFoundError')
+    })
+  })
+})

apps/web/src/actions/evaluations/connect.ts

@@ -0,0 +1,53 @@
+'use server'
+
+import { findAllEvaluationTemplates } from '@latitude-data/core/data-access'
+import {
+  DocumentVersionsRepository,
+  EvaluationsRepository,
+} from '@latitude-data/core/repositories'
+import { connectEvaluations } from '@latitude-data/core/services/evaluations/connect'
+import { z } from 'zod'
+
+import { withProject } from '../procedures'
+
+export const connectEvaluationsAction = withProject
+  .createServerAction()
+  .input(
+    z.object({
+      documentUuid: z.string(),
+      commitUuid: z.string(),
+      templateIds: z.array(z.number()),
+      evaluationUuids: z.array(z.string()),
+    }),
+  )
+  .handler(async ({ ctx, input }) => {
+    const evaluationTemplates = await findAllEvaluationTemplates().then((r) =>
+      r.unwrap(),
+    )
+    const scope = new EvaluationsRepository(ctx.workspace.id)
+    const evaluations = await scope.findAll().then((r) => r.unwrap())
+
+    const selectedTemplates = evaluationTemplates.filter((template) =>
+      input.templateIds.includes(template.id),
+    )
+    const selectedEvaluations = evaluations.filter((evaluation) =>
+      input.evaluationUuids.includes(evaluation.uuid),
+    )
+
+    const documentsScope = new DocumentVersionsRepository(ctx.workspace.id)
+    const document = await documentsScope
+      .getDocumentAtCommit({
+        projectId: ctx.project.id,
+        commitUuid: input.commitUuid,
+        documentUuid: input.documentUuid,
+      })
+      .then((r) => r.unwrap())
+
+    const connectedEvaluations = await connectEvaluations({
+      document,
+      templates: selectedTemplates,
+      evaluations: selectedEvaluations,
+    }).then((r) => r.unwrap())
+
+    return connectedEvaluations
+  })

apps/web/src/actions/evaluations/fetch.ts

@@ -1,14 +1,21 @@
 'use server'
 
 import { EvaluationsRepository } from '@latitude-data/core/repositories'
+import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
 
 export const fetchEvaluationsAction = authProcedure
   .createServerAction()
-  .handler(async ({ ctx }) => {
-    const evaluationsScope = new EvaluationsRepository(ctx.workspace.id)
-    const evaluations = await evaluationsScope.findAll().then((r) => r.unwrap())
+  .input(() => z.object({ documentUuid: z.string().optional() }).optional())
+  .handler(async ({ ctx, input }) => {
+    const scope = new EvaluationsRepository(ctx.workspace.id)
+    let result
+    if (input?.documentUuid) {
+      result = await scope.findByDocumentUuid(input.documentUuid)
+    } else {
+      result = await scope.findAll()
+    }
 
-    return evaluations
+    return result.unwrap()
   })

apps/web/src/app/(private)/_data-access/index.ts

@@ -1,6 +1,6 @@
 import { cache } from 'react'
 
-import { type Commit, type Project } from '@latitude-data/core/browser'
+import { type Commit } from '@latitude-data/core/browser'
 import { findAllEvaluationTemplates } from '@latitude-data/core/data-access'
 import { NotFoundError } from '@latitude-data/core/lib/errors'
 import {
@@ -51,9 +51,10 @@ export const findProjectCached = cache(
 )
 
 export const findCommitCached = cache(
-  async ({ uuid, project }: { uuid: string; project: Project }) => {
-    const commitsScope = new CommitsRepository(project.workspaceId)
-    const result = await commitsScope.getCommitByUuid({ project, uuid })
+  async ({ uuid, projectId }: { uuid: string; projectId: number }) => {
+    const { workspace } = await getCurrentUser()
+    const commitsScope = new CommitsRepository(workspace.id)
+    const result = await commitsScope.getCommitByUuid({ projectId, uuid })
     const commit = result.unwrap()
 
     return commit
@@ -62,15 +63,21 @@ export const findCommitCached = cache(
 
 export const getDocumentByUuidCached = cache(
   async ({
+    projectId,
     documentUuid,
-    commit,
+    commitUuid,
   }: {
+    projectId: number
     documentUuid: string
-    commit: Commit
+    commitUuid: string
   }) => {
     const { workspace } = await getCurrentUser()
     const scope = new DocumentVersionsRepository(workspace.id)
-    const result = await scope.getDocumentAtCommit({ documentUuid, commit })
+    const result = await scope.getDocumentAtCommit({
+      documentUuid,
+      commitUuid,
+      projectId,
+    })
     if (result.error) {
       const error = result.error
       if (error instanceof NotFoundError) {
@@ -170,3 +177,12 @@ export const getProviderLogCached = cache(async (uuid: string) => {
   const scope = new ProviderLogsRepository(workspace.id)
   return await scope.findByUuid(uuid).then((r) => r.unwrap())
 })
+
+export const getEvaluationsByDocumentUuidCached = cache(
+  async (documentUuid: string) => {
+    const { workspace } = await getCurrentUser()
+    const scope = new EvaluationsRepository(workspace.id)
+    const result = await scope.findByDocumentUuid(documentUuid)
+    return result.unwrap()
+  },
+)

apps/web/src/app/(private)/evaluations/_components/ActiveEvaluations/Table/index.tsx

@@ -1,3 +1,5 @@
+'use client'
+
 import { Evaluation } from '@latitude-data/core/browser'
 import {
   Icon,

apps/web/src/app/(private)/projects/[projectId]/page.tsx

@@ -1,9 +1,6 @@
-import { HEAD_COMMIT, type Project } from '@latitude-data/core/browser'
+import { type Project } from '@latitude-data/core/browser'
 import { NotFoundError } from '@latitude-data/core/lib/errors'
-import {
-  findCommitCached,
-  findProjectCached,
-} from '$/app/(private)/_data-access'
+import { findProjectCached } from '$/app/(private)/_data-access'
 import { getCurrentUser, SessionData } from '$/services/auth/getCurrentUser'
 import { ROUTES } from '$/services/routes'
 import { notFound, redirect } from 'next/navigation'
@@ -25,7 +22,7 @@ export default async function ProjectPage({ params }: ProjectPageParams) {
       projectId: Number(params.projectId),
       workspaceId: session.workspace.id,
     })
-    await findCommitCached({ uuid: HEAD_COMMIT, project })
+
     url = PROJECT_ROUTE({ id: +project.id }).commits.latest
   } catch (error) {
     if (error instanceof NotFoundError) {

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/_components/DocumentsLayout/index.tsx

@@ -32,7 +32,7 @@ export default async function DocumentsLayout({
     workspaceId: session.workspace.id,
   })
   const commit = await findCommitCached({
-    project,
+    projectId,
     uuid: commitUuid,
   })
   const resizableId = ResizableGroups.DocumentSidebar