feature: evaluation result types and configuration

Evaluations/templates now have a configuration to configure the type of result we are expecting, stored in a jsonb column. EvaluationResults now have a polymorphic table to store the result depending on its type – This is so that we can aggregate results in SQL later on.
latitude-dev · Sep 12, 2024 · d980ffb · d980ffb
1 parent 2a9b942
commit d980ffb
Show file tree

Hide file tree

Showing 40 changed files with 3,027 additions and 93 deletions.
diff --git a/apps/web/src/actions/evaluations/create.ts b/apps/web/src/actions/evaluations/create.ts
@@ -1,6 +1,9 @@
 'use server'
 
-import { EvaluationMetadataType } from '@latitude-data/core/browser'
+import {
+  EvaluationMetadataType,
+  EvaluationResultableType,
+} from '@latitude-data/core/browser'
 import { createEvaluation } from '@latitude-data/core/services/evaluations/create'
 import { z } from 'zod'
 
@@ -16,6 +19,12 @@ export const createEvaluationAction = authProcedure
         .nativeEnum(EvaluationMetadataType)
         .optional()
         .default(EvaluationMetadataType.LlmAsJudge),
+      configuration: z.object({
+        type: z.nativeEnum(EvaluationResultableType),
+        detail: z
+          .object({ range: z.object({ from: z.number(), to: z.number() }) })
+          .optional(),
+      }),
       metadata: z
         .object({
           prompt: z.string(),
@@ -25,11 +34,14 @@ export const createEvaluationAction = authProcedure
     { type: 'json' },
   )
   .handler(async ({ input, ctx }) => {
+    console.log(input)
+
     const result = await createEvaluation({
       workspace: ctx.workspace,
       name: input.name,
       description: input.description,
       metadata: input.metadata,
+      configuration: input.configuration,
       type: input.type,
     })
 

diff --git a/apps/web/src/app/(private)/error.tsx b/apps/web/src/app/(private)/error.tsx
@@ -4,7 +4,9 @@ import { useEffect } from 'react'
 
 import { ErrorComponent, useSession } from '@latitude-data/web-ui/browser'
 import { NAV_LINKS } from '$/app/(private)/_lib/constants'
+import BreadcrumpLink from '$/components/BreadcrumpLink'
 import { AppLayout } from '$/components/layouts'
+import { ROUTES } from '$/services/routes'
 
 export default function Error({
   error,
@@ -19,7 +21,14 @@ export default function Error({
   return (
     <AppLayout
       currentUser={session.currentUser}
-      breadcrumbs={[{ name: session.workspace.name }, { name: 'Error' }]}
+      breadcrumbs={[
+        {
+          name: (
+            <BreadcrumpLink name={session.workspace.name} href={ROUTES.root} />
+          ),
+        },
+        { name: 'Error' },
+      ]}
       navigationLinks={NAV_LINKS}
     >
       <ErrorComponent

diff --git a/apps/web/src/app/(private)/evaluations/_components/CreateEvaluationModal/index.tsx b/apps/web/src/app/(private)/evaluations/_components/CreateEvaluationModal/index.tsx
@@ -1,10 +1,15 @@
 import { useCallback, useEffect, useMemo, useState } from 'react'
 
+import {
+  EvaluationResultableType,
+  EvaluationResultConfiguration,
+} from '@latitude-data/core/browser'
 import {
   ConfirmModal,
+  FormField,
   Input,
   ReactStateDispatch,
-  Text,
+  TabSelector,
   TextArea,
 } from '@latitude-data/web-ui'
 import { ROUTES } from '$/services/routes'
@@ -27,6 +32,10 @@ export default function CreateEvaluationModal({
   const [title, setTitle] = useState(initialData?.title ?? '')
   const [description, setDescription] = useState(initialData?.description ?? '')
   const [prompt, setPrompt] = useState(initialData?.prompt ?? '')
+  const [configuration, setConfiguration] =
+    useState<EvaluationResultConfiguration>({
+      type: EvaluationResultableType.Text,
+    })
 
   const router = useRouter()
 
@@ -54,9 +63,10 @@ export default function CreateEvaluationModal({
       name: title,
       description,
       metadata: { prompt },
+      configuration,
     })
     onClose(null)
-  }, [create, onClose, title, description, prompt])
+  }, [create, onClose, title, description, prompt, configuration])
 
   const titleError = useMemo<string | undefined>(() => {
     if (!title) return 'Please enter a name for your evaluation.'
@@ -82,18 +92,16 @@ export default function CreateEvaluationModal({
       }}
     >
       <div className='w-full flex flex-col gap-4'>
-        <div className='w-full flex flex-col gap-4'>
-          <Text.H5M>Name</Text.H5M>
+        <FormField label='Title'>
           <Input
             value={title}
             errors={titleError ? [titleError] : undefined}
             onChange={(e) => setTitle(e.target.value)}
             placeholder='Enter title'
             className='w-full'
           />
-        </div>
-        <div className='w-full flex flex-col gap-4'>
-          <Text.H5M>Description</Text.H5M>
+        </FormField>
+        <FormField label='Description'>
           <TextArea
             value={description}
             minRows={4}
@@ -102,7 +110,103 @@ export default function CreateEvaluationModal({
             placeholder='Describe what is the purpose of this evaluation'
             className='w-full'
           />
-        </div>
+        </FormField>
+        <FormField label='Type'>
+          <TabSelector
+            options={[
+              { label: 'Text', value: EvaluationResultableType.Text },
+              { label: 'Number', value: EvaluationResultableType.Number },
+              { label: 'Boolean', value: EvaluationResultableType.Boolean },
+            ]}
+            onSelect={(value) => {
+              if (value === EvaluationResultableType.Number) {
+                setConfiguration({
+                  type: value,
+                  detail: { range: { from: 0, to: 1 } },
+                })
+              } else {
+                setConfiguration({ type: value })
+              }
+            }}
+            selected={configuration.type}
+          />
+        </FormField>
+        {configuration.type === EvaluationResultableType.Number && (
+          <FormField label='Range'>
+            <div className='flex flex-row items-center flex-1 gap-4'>
+              <Input
+                type='number'
+                min={0}
+                value={configuration.detail?.range.from.toString() || ''}
+                placeholder='From'
+                onChange={(e) => {
+                  setConfiguration((prev) => {
+                    const next = { ...prev }
+
+                    if (e.target.value === '') {
+                      next.detail = {
+                        range: { from: 0, to: next.detail?.range.to || 0 },
+                      }
+
+                      return next
+                    }
+
+                    if (next.detail?.range.from) {
+                      next.detail.range.from = parseInt(e.target.value)
+                      if (next.detail.range.from > next.detail.range.to) {
+                        next.detail.range.to = next.detail.range.from + 1
+                      }
+                    } else {
+                      next.detail = {
+                        range: {
+                          from: parseInt(e.target.value),
+                          to: parseInt(e.target.value) + 1,
+                        },
+                      }
+                    }
+
+                    return next
+                  })
+                }}
+              />
+              <Input
+                type='number'
+                min={0}
+                value={configuration.detail?.range.to.toString() || ''}
+                placeholder='To'
+                onChange={(e) =>
+                  setConfiguration((prev) => {
+                    const next = { ...prev }
+
+                    if (e.target.value === '') {
+                      next.detail = {
+                        range: { from: 0, to: 0 },
+                      }
+
+                      return next
+                    }
+
+                    if (next.detail?.range.to) {
+                      next.detail.range.to = parseInt(e.target.value)
+                      if (next.detail.range.to < next.detail.range.from) {
+                        next.detail.range.from = next.detail.range.to - 1
+                      }
+                    } else {
+                      next.detail = {
+                        range: {
+                          from: parseInt(e.target.value) - 1,
+                          to: parseInt(e.target.value),
+                        },
+                      }
+                    }
+
+                    return next
+                  })
+                }
+              />
+            </div>
+          </FormField>
+        )}
       </div>
     </ConfirmModal>
   )

diff --git a/...Uuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx b/...Uuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx
@@ -75,7 +75,7 @@ export function DocumentLogMetadata({
       providerLogs?.reduce(
         (acc, log) => {
           const key = String(log.providerId)
-          acc[key] = (acc[key] ?? 0) + log.cost_in_millicents
+          acc[key] = (acc[key] ?? 0) + log.costInMillicents
           return acc
         },
         {} as Record<string, number>,

diff --git a/apps/web/src/stores/evaluationTemplates.ts b/apps/web/src/stores/evaluationTemplates.ts
@@ -2,7 +2,7 @@
 
 import { compact, flatten } from 'lodash-es'
 
-import type { EvaluationTemplateWithCategory } from '@latitude-data/core/browser'
+import { EvaluationTemplateWithCategory } from '@latitude-data/core/browser'
 import { useToast } from '@latitude-data/web-ui'
 import { fetchEvaluationTemplatesAction } from '$/actions/evaluationTemplates/fetch'
 import useSWR, { SWRConfiguration } from 'swr'

diff --git a/packages/core/drizzle/0051_colorful_lightspeed.sql b/packages/core/drizzle/0051_colorful_lightspeed.sql
@@ -0,0 +1,33 @@
+DO $$ BEGIN
+ CREATE TYPE "public"."evaluation_result_types" AS ENUM('evaluation_resultable_booleans', 'evaluation_resultable_texts', 'evaluation_resultable_numbers');
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "latitude"."evaluation_resultable_numbers" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"result" bigint NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "latitude"."evaluation_resultable_texts" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"result" text NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "latitude"."evaluation_resultable_booleans" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"result" boolean NOT NULL,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	"updated_at" timestamp DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "latitude"."evaluations" ADD COLUMN "configuration" jsonb NOT NULL;--> statement-breakpoint
+ALTER TABLE "latitude"."evaluation_results" ADD COLUMN "resultable_type" "evaluation_result_types" NOT NULL;--> statement-breakpoint
+ALTER TABLE "latitude"."evaluation_results" ADD COLUMN "resultable_id" bigint NOT NULL;--> statement-breakpoint
+ALTER TABLE "latitude"."evaluations_templates" ADD COLUMN "configuration" jsonb NOT NULL;--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "resultable_idx" ON "latitude"."evaluation_results" USING btree ("resultable_id","resultable_type");--> statement-breakpoint
+ALTER TABLE "latitude"."evaluation_results" DROP COLUMN IF EXISTS "result";