feat: wip

apps/web/src/actions/evaluations/runBatch.ts

@@ -1,58 +1,105 @@
 'use server'
 
+import { readMetadata } from '@latitude-data/compiler'
 import {
   DatasetsRepository,
-  DocumentVersionsRepository,
   EvaluationsRepository,
 } from '@latitude-data/core/repositories'
 import { setupJobs } from '@latitude-data/jobs'
 import { nanoid } from 'nanoid'
 import { z } from 'zod'
+import { createServerActionProcedure } from 'zsa'
 
-import { authProcedure } from '../procedures'
+import { widthDocument } from '../procedures'
 
-export const runBatchEvaluationAction = authProcedure
+const withDataset = createServerActionProcedure(widthDocument)
+  .input(z.object({ datasetId: z.number() }))
+  .handler(async ({ input, ctx }) => {
+    const datasetsRepo = new DatasetsRepository(ctx.workspace.id)
+    const dataset = await datasetsRepo
+      .find(input.datasetId)
+      .then((r) => r.unwrap())
+
+    return { ...ctx, dataset }
+  })
+
+function isValidParameter(valueIndex: number | undefined, headers: string[]) {
+  if (valueIndex === undefined) return false
+  if (valueIndex === -1) return true
+  const hasIndex = headers[valueIndex]
+  return hasIndex !== undefined
+}
+function parameterErrorMessage({
+  param,
+  message,
+}: {
+  param: string
+  message: string
+}) {
+  return `${param}: ${message}`
+}
+
+export const runBatchEvaluationAction = withDataset
   .createServerAction()
-  .input(
-    z.object({
-      projectId: z.number(),
-      documentUuid: z.string(),
-      commitUuid: z.string(),
-      datasetId: z.number(),
+  .input(async ({ ctx }) => {
+    return z.object({
       evaluationIds: z.array(z.number()),
       fromLine: z.number().optional(),
       toLine: z.number().optional(),
-      parameters: z.record(z.number()).optional(),
-    }),
-  )
+      parameters: z
+        .record(z.number())
+        .superRefine(async (parameters, refineCtx) => {
+          const metadata = await readMetadata({
+            prompt: ctx.document.content ?? '',
+            fullPath: ctx.document.path,
+          })
+          const docParams = metadata.parameters
+          const headers = ctx.dataset.fileMetadata.headers
+          const paramKeys = Object.keys(parameters)
+          Array.from(docParams).forEach((key) => {
+            const existsInDocument = paramKeys.includes(key)
+
+            if (!existsInDocument) {
+              refineCtx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['parameters', key],
+                message: parameterErrorMessage({
+                  param: key,
+                  message: 'Is not a valid parameter in this document',
+                }),
+              })
+            }
+
+            const valueIndex = isValidParameter(parameters[key], headers)
+
+            if (!valueIndex) {
+              refineCtx.addIssue({
+                code: z.ZodIssueCode.custom,
+                path: ['parameters', key],
+                message: parameterErrorMessage({
+                  param: key,
+                  message:
+                    'Has not a valid header assigned in this dataset. If you want to keep empty this parameter choose "Leave empty in that parameter"',
+                }),
+              })
+            }
+          })
+        }),
+    })
+  })
   .handler(async ({ input, ctx }) => {
+    debugger
     const evaluationsRepo = new EvaluationsRepository(ctx.workspace.id)
     const evaluations = await evaluationsRepo
       .filterById(input.evaluationIds)
       .then((r) => r.unwrap())
-    const datasetsRepo = new DatasetsRepository(ctx.workspace.id)
-    const dataset = await datasetsRepo
-      .find(input.datasetId)
-      .then((r) => r.unwrap())
-
-    const docsRepo = new DocumentVersionsRepository(ctx.workspace.id)
-    const document = await docsRepo
-      .getDocumentAtCommit({
-        projectId: input.projectId,
-        commitUuid: input.commitUuid,
-        documentUuid: input.documentUuid,
-      })
-      .then((r) => r.unwrap())
-
     const queues = setupJobs()
-
     evaluations.forEach((evaluation) => {
       const batchId = `evaluation:${evaluation.id}:${nanoid(5)}`
-
       queues.defaultQueue.jobs.enqueueRunBatchEvaluationJob({
         evaluation,
-        dataset,
-        document,
+        dataset: ctx.dataset,
+        document: ctx.document,
         fromLine: input.fromLine,
         toLine: input.toLine,
         parametersMap: input.parameters,

apps/web/src/actions/procedures/index.ts

@@ -1,5 +1,8 @@
 import { UnauthorizedError } from '@latitude-data/core/lib/errors'
-import { ProjectsRepository } from '@latitude-data/core/repositories'
+import {
+  DocumentVersionsRepository,
+  ProjectsRepository,
+} from '@latitude-data/core/repositories'
 import { getCurrentUser } from '$/services/auth/getCurrentUser'
 import { z } from 'zod'
 import { createServerActionProcedure } from 'zsa'
@@ -29,3 +32,18 @@ export const withProject = createServerActionProcedure(authProcedure)
 
     return { ...ctx, project }
   })
+
+export const widthDocument = createServerActionProcedure(withProject)
+  .input(z.object({ commitUuid: z.string(), documentUuid: z.string() }))
+  .handler(async ({ input, ctx }) => {
+    const repo = new DocumentVersionsRepository(ctx.workspace.id)
+    const document = await repo
+      .getDocumentAtCommit({
+        projectId: ctx.project.id,
+        commitUuid: input.commitUuid,
+        documentUuid: input.documentUuid,
+      })
+      .then((r) => r.unwrap())
+
+    return { ...ctx, document }
+  })

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/create-batch/_components/CreateBatchEvaluationModal/DatasetForm/index.tsx

@@ -61,6 +61,7 @@ export default function DatasetForm({
   parametersList,
   onToggleAllLines,
   onSelectDataset,
+  errors,
 }: {
   onParametersChange: (param: string) => (header: string) => void
   parametersList: string[]
@@ -73,7 +74,27 @@ export default function DatasetForm({
   isLoadingDatasets: boolean
   onSelectDataset: (value: string) => void
   onToggleAllLines: (checked: boolean) => void
+  errors: Record<string, string[] | undefined> | undefined
 }) {
+  const paramaterErrors = useMemo(() => {
+    if (!errors) return {}
+    if (!errors.parameters) return {}
+
+    const paramErrors = errors.parameters
+    if (!Array.isArray(paramErrors)) return {}
+
+    return paramErrors.reduce((acc, error) => {
+      const parts = error.split(': ')
+      const param = parts[0]
+      const message = parts[1]
+      if (!param || !message) return acc
+
+      const prevMessage = acc[param] || []
+      prevMessage.push(message)
+      acc[param] = prevMessage
+      return acc
+    }, {} as Record<string, string[]>)
+  }, [errors])
   const datasetOptions = useMemo(
     () => datasets.map((ds) => ({ value: ds.id, label: ds.name })),
     [datasets],
@@ -83,7 +104,7 @@ export default function DatasetForm({
       <NumeredList>
         <NumeredList.Item title='Pick dataset' width='w-1/2'>
           <Select
-            name='dataset'
+            name='datasetId'
             placeholder='Select dataset'
             disabled={isLoadingDatasets}
             options={datasetOptions}
@@ -122,6 +143,7 @@ export default function DatasetForm({
                   key={param}
                   name={`parameter[${param}]`}
                   disabled={headers.length === 0}
+                  errors={paramaterErrors[param]}
                   badgeLabel
                   label={param}
                   options={headers}

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/create-batch/_components/CreateBatchEvaluationModal/index.tsx

@@ -27,13 +27,12 @@ export default function CreateBatchEvaluationModal({
 }) {
   const navigate = useNavigate()
   const documentUuid = document.documentUuid
-  const { runBatch, isRunningBatch } = useRunBatch({
+  const { runBatch, errors, isRunningBatch } = useRunBatch({
     document,
     projectId,
     commitUuid,
   })
   const form = useRunBatchForm({ documentMetadata })
-
   const onRunBatch = useCallback(() => {
     runBatch({
       datasetId: form.selectedDataset?.id,
@@ -43,9 +42,8 @@ export default function CreateBatchEvaluationModal({
       wantAllLines: form.wantAllLines,
       parameters: form.parameters,
     })
-    return 'lol'
   }, [
-    evaluation.uuid,
+    evaluation.id,
     runBatch,
     form.fromLine,
     form.toLine,
@@ -83,6 +81,7 @@ export default function CreateBatchEvaluationModal({
       }
     >
       <DatasetForm
+        errors={errors}
         datasets={form.datasets}
         isLoadingDatasets={form.isLoadingDatasets}
         selectedDataset={form.selectedDataset}

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/create-batch/_components/CreateBatchEvaluationModal/useRunBatch.ts

@@ -14,9 +14,11 @@ export function useRunBatch({
   document: DocumentVersion
   commitUuid: string
 }) {
-  const { execute: run, isPending: isRunning } = useLatitudeAction(
+  const { error, execute: run, isPending: isRunning } = useLatitudeAction(
     runBatchEvaluationAction,
   )
+  console.log("ERROR_ACTION", error?.fieldErrors)
+  const errors = error?.fieldErrors
   const runBatch = useCallback(
     async ({
       evaluationIds,
@@ -33,8 +35,7 @@ export function useRunBatch({
       evaluationIds: number[]
       parameters: RunBatchParameters
     }) => {
-      console.log('RUNNING BATCH')
-      const [_data, _error] = await run({
+      const [_, error] = await run({
         projectId: Number(projectId),
         documentUuid: document.documentUuid,
         commitUuid,
@@ -45,13 +46,16 @@ export function useRunBatch({
         // @ts-ignore
         parameters,
       })
-      /* console.log(data, error) */
+
+      // Errors
+      console.log("ERROR", error)
     },
     [run, projectId, document.documentUuid, commitUuid],
   )
 
   return {
     runBatch,
     isRunningBatch: isRunning,
+    errors,
   }
 }

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/create-batch/_components/CreateBatchEvaluationModal/useRunBatchForm.ts

@@ -52,12 +52,13 @@ export function useRunBatchForm({
       setParameters(buildEmptyParameters(parametersList))
       setFromLine(0)
       setToLine(ds.fileMetadata.rowCount)
-      setHeaders(
-        ds.fileMetadata.headers.map((header) => ({
+      setHeaders([
+        { value: '-1', label: '-- Leave this parameter empty' },
+        ...ds.fileMetadata.headers.map((header) => ({
           value: header,
           label: header,
         })),
-      )
+      ])
     },
     [parametersList, datasets],
   )