feat: migrate evaluation results documentlog pointer #2 (#541)

Adds the new columns to the creation service and adapts all the other services to use them.

packages/core/src/jobs/job-definitions/batchEvaluations/runEvaluationJob.test.ts

@@ -2,7 +2,12 @@ import { RunErrorCodes } from '@latitude-data/constants/errors'
 import { Job } from 'bullmq'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-import { DocumentLog, EvaluationDto, Workspace } from '../../../browser'
+import {
+  DocumentLog,
+  EvaluationDto,
+  ProviderLog,
+  Workspace,
+} from '../../../browser'
 import { Providers } from '../../../constants'
 import { Result } from '../../../lib'
 import * as queues from '../../../queues'
@@ -50,14 +55,15 @@ function buildJobData(
   return {
     workspaceId: data.workspaceId || 1,
     documentUuid: data.documentUuid || 'doc-uuid',
-    documentLogUuid: data.documentLogUuid || 'log-uuid',
+    providerLogUuid: data.providerLogUuid || 'log-uuid',
     evaluationId: data.evaluationId || 2,
     batchId: 'batch-123',
   }
 }
 let workspace: Workspace
 let documentUuid: string
 let documentLog: DocumentLog
+let providerLog: ProviderLog
 let evaluation: EvaluationDto
 let runChainResponse: ChainResponse<'object'>
 
@@ -79,6 +85,12 @@ describe('runEvaluationJob', () => {
       document: documentVersion,
       commit: setup.commit,
     })
+    const pl = await factories.createProviderLog({
+      workspace: setup.workspace,
+      providerId: setup.providers[0]!.id,
+      providerType: setup.providers[0]!.provider,
+      documentLogUuid: docLog.uuid,
+    })
     const llmEval = await factories.createLlmAsJudgeEvaluation({
       user: setup.user,
       workspace: setup.workspace,
@@ -89,6 +101,7 @@ describe('runEvaluationJob', () => {
     evaluation = await evaluationsScope.find(llmEval.id).then((r) => r.unwrap())
     documentUuid = documentVersion.documentUuid
     documentLog = docLog
+    providerLog = pl
   })
 
   describe('with valid data', () => {
@@ -98,7 +111,7 @@ describe('runEvaluationJob', () => {
         data: buildJobData({
           workspaceId: workspace.id,
           documentUuid,
-          documentLogUuid: documentLog.uuid,
+          providerLogUuid: providerLog.uuid,
           evaluationId: evaluation.id,
         }),
       } as Job<RunEvaluationJobData>
@@ -107,14 +120,7 @@ describe('runEvaluationJob', () => {
     it('calls runEvaluation', async () => {
       await runEvaluationJob(jobData)
       expect(runEvaluationSpy).toHaveBeenCalledWith({
-        documentLog: {
-          ...documentLog,
-          error: {
-            code: null,
-            message: null,
-            details: null,
-          },
-        },
+        providerLog,
         evaluation,
         documentUuid: documentUuid,
       })
@@ -248,13 +254,13 @@ describe('runEvaluationJob', () => {
           data: buildJobData({
             workspaceId: workspace.id,
             documentUuid,
-            documentLogUuid: '12345678-1234-1234-1234-123456789034',
+            providerLogUuid: '12345678-1234-1234-1234-123456789034',
             evaluationId: evaluation.id,
           }),
         } as Job<RunEvaluationJobData>),
       ).rejects.toThrowError(
         new Error(
-          'DocumentLog not found with uuid 12345678-1234-1234-1234-123456789034',
+          'ProviderLog with uuid 12345678-1234-1234-1234-123456789034 not found',
         ),
       )
     })

packages/core/src/jobs/job-definitions/batchEvaluations/runEvaluationJob.ts

@@ -3,8 +3,8 @@ import { Job } from 'bullmq'
 import { getUnknownError, Result } from '../../../lib'
 import { queues } from '../../../queues'
 import {
-  DocumentLogsWithErrorsRepository,
   EvaluationsRepository,
+  ProviderLogsRepository,
 } from '../../../repositories'
 import { runEvaluation } from '../../../services/evaluations/run'
 import { WebsocketClient } from '../../../websockets/workers'
@@ -13,31 +13,31 @@ import { ProgressTracker } from '../../utils/progressTracker'
 async function fetchData({
   workspaceId,
   evaluationId,
-  documentLogUuid,
+  providerLogUuid,
 }: {
   workspaceId: number
   evaluationId: number
-  documentLogUuid: string
+  providerLogUuid: string
 }) {
-  const documentLogsScope = new DocumentLogsWithErrorsRepository(workspaceId)
-  const evaluationsScope = new EvaluationsRepository(workspaceId)
-  const docLogResult = await documentLogsScope.findByUuid(documentLogUuid)
+  const providerLogsRepository = new ProviderLogsRepository(workspaceId)
+  const evaluationsRepository = new EvaluationsRepository(workspaceId)
+  const providerLogResult =
+    await providerLogsRepository.findByUuid(providerLogUuid)
+  if (providerLogResult.error) return providerLogResult
 
-  if (docLogResult.error) return docLogResult
-
-  const result = await evaluationsScope.find(evaluationId)
+  const result = await evaluationsRepository.find(evaluationId)
   if (result.error) return result
 
   return Result.ok({
-    documentLog: docLogResult.value,
+    providerLog: providerLogResult.value,
     evaluation: result.value,
   })
 }
 
 export type RunEvaluationJobData = {
   workspaceId: number
   documentUuid: string
-  documentLogUuid: string
+  providerLogUuid: string
   evaluationId: number
   batchId?: string
 }
@@ -52,21 +52,21 @@ async function isSuccessful(run: Awaited<ReturnType<typeof runEvaluation>>) {
 }
 
 export async function runEvaluationJob(job: Job<RunEvaluationJobData>) {
-  const { workspaceId, batchId, documentUuid, documentLogUuid, evaluationId } =
+  const { workspaceId, batchId, documentUuid, providerLogUuid, evaluationId } =
     job.data
   const websockets = await WebsocketClient.getSocket()
   let progressTracker: ProgressTracker | undefined
   if (batchId) {
     progressTracker = new ProgressTracker(await queues(), batchId)
   }
-  const { documentLog, evaluation } = await fetchData({
+  const { providerLog, evaluation } = await fetchData({
     workspaceId,
     evaluationId,
-    documentLogUuid,
+    providerLogUuid,
   }).then((r) => r.unwrap())
 
   const run = await runEvaluation({
-    documentLog,
+    providerLog,
     evaluation,
     documentUuid,
   })

packages/core/src/jobs/job-definitions/liveEvaluations/runLiveEvaluationJob.ts

@@ -1,6 +1,8 @@
 import { Job } from 'bullmq'
 
 import { DocumentLog, EvaluationDto } from '../../../browser'
+import { findLastProviderLogFromDocumentLogUuid } from '../../../data-access'
+import { NotFoundError } from '../../../lib'
 import { runEvaluation } from '../../../services/evaluations'
 
 export type RunLiveEvaluationJobData = {
@@ -13,9 +15,16 @@ export const runLiveEvaluationJob = async (
   job: Job<RunLiveEvaluationJobData>,
 ) => {
   const { evaluation, documentLog, documentUuid } = job.data
+  const providerLog = await findLastProviderLogFromDocumentLogUuid(
+    documentLog.uuid,
+  )
+  if (!providerLog)
+    throw new NotFoundError(
+      `Provider log not found for document log ${documentLog.uuid}`,
+    )
 
   const { response } = await runEvaluation({
-    documentLog,
+    providerLog,
     evaluation,
     documentUuid,
   }).then((r) => r.unwrap())

packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts

@@ -39,13 +39,11 @@ async function generateDocumentLogs({
 }) {
   return await Promise.all(
     Array.from({ length: quantity }).map(async () => {
-      return factories
-        .createDocumentLog({
-          document,
-          commit,
-          parameters,
-        })
-        .then((r) => r.documentLog)
+      return factories.createDocumentLog({
+        document,
+        commit,
+        parameters,
+      })
     }),
   )
 }
@@ -222,10 +220,11 @@ describe('getConnectedDocumentsWithMetadata', () => {
       quantity: 5,
     })
     const results = await Promise.all(
-      logs.map((documentLog) => {
+      logs.map(({ documentLog, providerLogs }) => {
         return factories.createEvaluationResult({
           evaluation,
           documentLog,
+          evaluatedProviderLog: providerLogs[0]!,
         })
       }),
     )
@@ -266,10 +265,11 @@ describe('getConnectedDocumentsWithMetadata', () => {
       quantity: 10,
     })
     await Promise.all(
-      logs.map((documentLog, index) => {
+      logs.map(({ documentLog, providerLogs }, index) => {
         return factories.createEvaluationResult({
           evaluation,
           documentLog,
+          evaluatedProviderLog: providerLogs[0]!,
           result: index < 6 ? 'yes' : 'no', // yes should appear 6 times, while no should appear 4 times
         })
       }),

packages/core/src/repositories/evaluationResultsRepository/index.test.ts

@@ -5,6 +5,7 @@ import {
   DocumentLog,
   ErrorableEntity,
   EvaluationDto,
+  ProviderLog,
   Providers,
   Workspace,
 } from '../../browser'
@@ -21,6 +22,7 @@ import { EvaluationResultsRepository } from './index'
 
 let workspace: Workspace
 let documentLog: DocumentLog
+let providerLogs: ProviderLog[]
 let evaluation: EvaluationDto
 
 describe('EvaluationResultsRepository', () => {
@@ -41,18 +43,21 @@ describe('EvaluationResultsRepository', () => {
       })
       workspace = wps
       const document = doc!
-      const { documentLog: docLog } = await createDocumentLog({
-        document,
-        commit: commit1,
-      })
+      const { documentLog: docLog, providerLogs: pls } =
+        await createDocumentLog({
+          document,
+          commit: commit1,
+        })
       documentLog = docLog
+      providerLogs = pls
       evaluation = await createLlmAsJudgeEvaluation({
         user: user,
         workspace: workspace,
       })
 
       await createEvaluationResult({
         documentLog,
+        evaluatedProviderLog: providerLogs[0]!,
         evaluation,
         result: 'Result 1',
       })
@@ -79,20 +84,22 @@ describe('EvaluationResultsRepository', () => {
         workspace: workspace2,
       })
 
-      const documentLog2 = await createDocumentLog({
-        document: document2!,
-        commit: commit2,
-      })
+      const { documentLog: documentLog2, providerLogs: providerLogs2 } =
+        await createDocumentLog({
+          document: document2!,
+          commit: commit2,
+        })
 
       await createProviderLog({
         workspace,
-        documentLogUuid: documentLog2.documentLog.uuid,
+        documentLogUuid: documentLog2.uuid,
         providerId: provider2!.id,
         providerType: Providers.OpenAI,
       })
 
       await createEvaluationResult({
-        documentLog: documentLog2.documentLog,
+        documentLog: documentLog2,
+        evaluatedProviderLog: providerLogs2[0]!,
         evaluation: evaluation2,
         result: 'Result 2',
       })
@@ -109,6 +116,7 @@ describe('EvaluationResultsRepository', () => {
     it('does not return evaluation results with errors', async () => {
       const { evaluationResult } = await createEvaluationResult({
         documentLog,
+        evaluatedProviderLog: providerLogs[0]!,
         evaluation,
         result: 'Result 2',
       })
@@ -131,6 +139,7 @@ describe('EvaluationResultsRepository', () => {
     it('filter evaluation results without provider log', async () => {
       await createEvaluationResult({
         documentLog,
+        evaluatedProviderLog: providerLogs[0]!,
         evaluation,
         result: 'Result 2',
         skipProviderLogCreation: true,
@@ -148,6 +157,7 @@ describe('EvaluationResultsRepository', () => {
     it('filter evaluation results without result', async () => {
       await createEvaluationResult({
         documentLog,
+        evaluatedProviderLog: providerLogs[0]!,
         evaluation,
         skipEvaluationResultCreation: true,
       })

packages/core/src/repositories/providerLogsRepository.ts

@@ -21,7 +21,9 @@ export class ProviderLogsRepository extends Repository<ProviderLog> {
     const result = await this.scope.where(eq(providerLogs.uuid, uuid)).limit(1)
 
     if (!result.length) {
-      return Result.error(new NotFoundError('ProviderLog not found'))
+      return Result.error(
+        new NotFoundError(`ProviderLog with uuid ${uuid} not found`),
+      )
     }
 
     return Result.ok(result[0]!)