Connect evaluation with dataset (#162)

.gitignore

@@ -43,7 +43,8 @@ docker/pgdata/
 # Next.js
 next-env.d.ts
 
-apps/web/public/uploads
+// File uploads in development
+tmp
 
 # Misc
 TODO.md

apps/web/src/actions/datasets/preview.ts

@@ -16,5 +16,7 @@ export const previewDatasetAction = authProcedure
   .handler(async ({ ctx, input }) => {
     const repo = new DatasetsRepository(ctx.workspace.id)
     const dataset = await repo.find(input.id).then((r) => r.unwrap())
-    return await previewDataset({ dataset }).then((r) => r.unwrap())
+    return await previewDataset({ dataset, prependIndex: true }).then((r) =>
+      r.unwrap(),
+    )
   })

apps/web/src/actions/evaluations/runBatch.ts

@@ -1,59 +1,107 @@
 'use server'
 
+import { readMetadata } from '@latitude-data/compiler'
 import {
   DatasetsRepository,
-  DocumentVersionsRepository,
   EvaluationsRepository,
 } from '@latitude-data/core/repositories'
 import { setupJobs } from '@latitude-data/jobs'
 import { nanoid } from 'nanoid'
 import { z } from 'zod'
+import { createServerActionProcedure } from 'zsa'
 
-import { authProcedure } from '../procedures'
+import { widthDocument } from '../procedures'
 
-export const runBatchEvaluationAction = authProcedure
+const USER_DECIDED_TO_IGNORE_THIS_PARAMETER = -1
+
+const withDataset = createServerActionProcedure(widthDocument)
+  .input(z.object({ datasetId: z.number() }))
+  .handler(async ({ input, ctx }) => {
+    const datasetsRepo = new DatasetsRepository(ctx.workspace.id)
+    const dataset = await datasetsRepo
+      .find(input.datasetId)
+      .then((r) => r.unwrap())
+
+    return { ...ctx, dataset }
+  })
+
+function isValidParameter(valueIndex: number | undefined, headers: string[]) {
+  if (valueIndex === undefined) return false
+  if (valueIndex === USER_DECIDED_TO_IGNORE_THIS_PARAMETER) return true
+  const hasIndex = headers[valueIndex]
+  return hasIndex !== undefined
+}
+function parameterErrorMessage({
+  param,
+  message,
+}: {
+  param: string
+  message: string
+}) {
+  return `${param}: ${message}`
+}
+
+export const runBatchEvaluationAction = withDataset
   .createServerAction()
-  .input(
-    z.object({
-      datasetId: z.number(),
-      projectId: z.number(),
-      documentUuid: z.string(),
-      commitUuid: z.string(),
+  .input(async ({ ctx }) => {
+    return z.object({
+      evaluationIds: z.array(z.number()),
       fromLine: z.number().optional(),
       toLine: z.number().optional(),
-      parameters: z.record(z.number()).optional(),
-      evaluationIds: z.array(z.number()),
-    }),
-  )
+      parameters: z
+        .record(z.number())
+        .optional()
+        .superRefine(async (parameters = {}, refineCtx) => {
+          const metadata = await readMetadata({
+            prompt: ctx.document.content ?? '',
+            fullPath: ctx.document.path,
+          })
+          const docParams = metadata.parameters
+          const headers = ctx.dataset.fileMetadata.headers
+          const paramKeys = Object.keys(parameters)
+          Array.from(docParams).forEach((key) => {
+            const existsInDocument = paramKeys.includes(key)
+
+            if (!existsInDocument) {
+              refineCtx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['parameters', key],
+                message: parameterErrorMessage({
+                  param: key,
+                  message: 'Is not a valid parameter in this document',
+                }),
+              })
+            }
+
+            const valueIndex = isValidParameter(parameters[key], headers)
+
+            if (!valueIndex) {
+              refineCtx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['parameters', key],
+                message: parameterErrorMessage({
+                  param: key,
+                  message:
+                    'Has not a valid header assigned in this dataset. If you want to keep empty this parameter choose "Leave empty in that parameter"',
+                }),
+              })
+            }
+          })
+        }),
+    })
+  })
   .handler(async ({ input, ctx }) => {
     const evaluationsRepo = new EvaluationsRepository(ctx.workspace.id)
     const evaluations = await evaluationsRepo
       .filterById(input.evaluationIds)
       .then((r) => r.unwrap())
-
-    const datasetsRepo = new DatasetsRepository(ctx.workspace.id)
-    const dataset = await datasetsRepo
-      .find(input.datasetId)
-      .then((r) => r.unwrap())
-
-    const docsRepo = new DocumentVersionsRepository(ctx.workspace.id)
-    const document = await docsRepo
-      .getDocumentAtCommit({
-        projectId: input.projectId,
-        commitUuid: input.commitUuid,
-        documentUuid: input.documentUuid,
-      })
-      .then((r) => r.unwrap())
-
     const queues = setupJobs()
-
     evaluations.forEach((evaluation) => {
       const batchId = `evaluation:${evaluation.id}:${nanoid(5)}`
-
       queues.defaultQueue.jobs.enqueueRunBatchEvaluationJob({
         evaluation,
-        dataset,
-        document,
+        dataset: ctx.dataset,
+        document: ctx.document,
         fromLine: input.fromLine,
         toLine: input.toLine,
         parametersMap: input.parameters,

apps/web/src/actions/procedures/index.ts

@@ -1,5 +1,8 @@
 import { UnauthorizedError } from '@latitude-data/core/lib/errors'
-import { ProjectsRepository } from '@latitude-data/core/repositories'
+import {
+  DocumentVersionsRepository,
+  ProjectsRepository,
+} from '@latitude-data/core/repositories'
 import { getCurrentUser } from '$/services/auth/getCurrentUser'
 import { z } from 'zod'
 import { createServerActionProcedure } from 'zsa'
@@ -29,3 +32,18 @@ export const withProject = createServerActionProcedure(authProcedure)
 
     return { ...ctx, project }
   })
+
+export const widthDocument = createServerActionProcedure(withProject)
+  .input(z.object({ commitUuid: z.string(), documentUuid: z.string() }))
+  .handler(async ({ input, ctx }) => {
+    const repo = new DocumentVersionsRepository(ctx.workspace.id)
+    const document = await repo
+      .getDocumentAtCommit({
+        projectId: ctx.project.id,
+        commitUuid: input.commitUuid,
+        documentUuid: input.documentUuid,
+      })
+      .then((r) => r.unwrap())
+
+    return { ...ctx, document }
+  })

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/ConnectedDocumentsTable/index.tsx

@@ -136,7 +136,7 @@ export default function ConnectedDocumentsTable({
                   .detail({ id: document.projectId })
                   .commits.detail({ uuid: HEAD_COMMIT })
                   .documents.detail({ uuid: document.documentUuid })
-                  .evaluations.detail(document.evaluationUuid).root,
+                  .evaluations.detail(document.evaluationId).root,
               )
             }
           />

apps/web/src/app/(private)/evaluations/_components/ActiveEvaluations/Table/index.tsx

@@ -15,7 +15,7 @@ import { useNavigate } from '$/hooks/useNavigate'
 import { ROUTES } from '$/services/routes'
 import Link from 'next/link'
 
-export const ActiveEvaluationsTableRow = ({
+const ActiveEvaluationsTableRow = ({
   evaluation,
   onSelect,
 }: {

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/index.tsx

@@ -0,0 +1,30 @@
+'use client'
+
+import { EvaluationDto } from '@latitude-data/core/browser'
+import { TableWithHeader } from '@latitude-data/web-ui'
+import { ROUTES } from '$/services/routes'
+import Link from 'next/link'
+
+export function Actions({
+  evaluation,
+  projectId,
+  commitUuid,
+  documentUuid,
+}: {
+  evaluation: EvaluationDto
+  projectId: string
+  commitUuid: string
+  documentUuid: string
+}) {
+  const href = ROUTES.projects
+    .detail({ id: Number(projectId) })
+    .commits.detail({ uuid: commitUuid })
+    .documents.detail({ uuid: documentUuid })
+    .evaluations.detail(evaluation.id).createBatch
+
+  return (
+    <Link href={href}>
+      <TableWithHeader.Button>Run batch evaluation</TableWithHeader.Button>
+    </Link>
+  )
+}