Simpler evaluations - part 1

latitude-dev · Oct 28, 2024 · 6b125f0 · 6b125f0
1 parent fb7edca
commit 6b125f0
Show file tree

Hide file tree

Showing 42 changed files with 3,277 additions and 373 deletions.
diff --git a/apps/web/src/actions/evaluationTemplates/create.ts b/apps/web/src/actions/evaluationTemplates/create.ts
@@ -2,7 +2,7 @@
 
 import { EvaluationResultableType } from '@latitude-data/core/browser'
 import { UnauthorizedError } from '@latitude-data/core/lib/errors'
-import { createEvaluationTemplate } from '@latitude-data/core/services/evaluationTemplates/create'
+import { createEvaluationTemplate } from '@latitude-data/core/services/evaluationLegacyTemplates/create'
 import { z } from 'zod'
 
 import { authProcedure } from '../procedures'

diff --git a/apps/web/src/actions/evaluationTemplates/destroy.ts b/apps/web/src/actions/evaluationTemplates/destroy.ts
@@ -1,7 +1,7 @@
 'use server'
 
 import { UnauthorizedError } from '@latitude-data/core/lib/errors'
-import { destroyEvaluationTemplate } from '@latitude-data/core/services/evaluationTemplates/destroy'
+import { destroyEvaluationTemplate } from '@latitude-data/core/services/evaluationLegacyTemplates/destroy'
 import { z } from 'zod'
 
 import { authProcedure } from '../procedures'

diff --git a/apps/web/src/actions/evaluations/create.ts b/apps/web/src/actions/evaluations/create.ts
@@ -18,7 +18,7 @@ export const createEvaluationAction = authProcedure
       type: z
         .nativeEnum(EvaluationMetadataType)
         .optional()
-        .default(EvaluationMetadataType.LlmAsJudge),
+        .default(EvaluationMetadataType.LlmAsJudgeLegacy),
       configuration: z.object({
         type: z.nativeEnum(EvaluationResultableType),
         detail: z

diff --git a/apps/web/src/actions/evaluations/createFromPrompt.ts b/apps/web/src/actions/evaluations/createFromPrompt.ts
@@ -32,7 +32,7 @@ export const createEvaluationFromPromptAction = withDocument
       workspace: ctx.workspace,
       name: input.name,
       description: 'AI-generated evaluation',
-      type: EvaluationMetadataType.LlmAsJudge,
+      type: EvaluationMetadataType.LlmAsJudgeLegacy,
       configuration: {
         type:
           input.type === 'number'

diff --git a/...r/Editor/RefineModal/steps/2_SelectEvaluationResults/SelectableEvaluationResultsTable.tsx b/...r/Editor/RefineModal/steps/2_SelectEvaluationResults/SelectableEvaluationResultsTable.tsx
@@ -29,17 +29,27 @@ export const ResultCellContent = ({
   evaluation: EvaluationDto
   value: unknown
 }) => {
-  if (evaluation.configuration.type === EvaluationResultableType.Boolean) {
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Boolean
+  ) {
     return (
       <Text.H4 color={(value as boolean) ? 'success' : 'destructive'}>
         {String(value)}
       </Text.H4>
     )
   }
 
-  if (evaluation.configuration.type === EvaluationResultableType.Number) {
-    const minValue = evaluation.configuration.detail?.range.from ?? 0
-    const maxValue = evaluation.configuration.detail?.range.to ?? 10
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Number
+  ) {
+    const minValue =
+      (evaluation.configuration ?? evaluation.metadata.configuration).detail
+        ?.range.from ?? 0
+    const maxValue =
+      (evaluation.configuration ?? evaluation.metadata.configuration).detail
+        ?.range.to ?? 10
 
     return (
       <RangeBadge

diff --git a/...Uuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx b/...Uuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx
@@ -39,17 +39,27 @@ export const ResultCellContent = ({
   evaluation: EvaluationDto
   value: unknown
 }) => {
-  if (evaluation.configuration.type === EvaluationResultableType.Boolean) {
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Boolean
+  ) {
     return (
       <Badge variant={value === 'true' ? 'success' : 'destructive'}>
         {String(value)}
       </Badge>
     )
   }
 
-  if (evaluation.configuration.type === EvaluationResultableType.Number) {
-    const minValue = evaluation.configuration.detail?.range.from ?? 0
-    const maxValue = evaluation.configuration.detail?.range.to ?? 10
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Number
+  ) {
+    const minValue =
+      (evaluation.configuration ?? evaluation.metadata.configuration).detail
+        ?.range.from ?? 0
+    const maxValue =
+      (evaluation.configuration ?? evaluation.metadata.configuration).detail
+        ?.range.to ?? 10
 
     return (
       <RangeBadge

diff --git a/...ations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx b/...ations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx
@@ -41,7 +41,8 @@ export default function MeanValuePanel({
     documentUuid,
     onStatusChange,
   })
-  const config = evaluation.configuration.detail!
+  const config = (evaluation.configuration ?? evaluation.metadata.configuration)
+    .detail!
   const defaultMinValue = config.range.from
   const defaultMaxValue = config.range.to
   return (

diff --git a/...valuations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/CostOverResults.tsx b/...valuations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/CostOverResults.tsx
@@ -2,7 +2,7 @@
 
 import { useCallback, useMemo } from 'react'
 
-import { Evaluation } from '@latitude-data/core/browser'
+import { EvaluationDto } from '@latitude-data/core/browser'
 import {
   Badge,
   ScatterChart,
@@ -20,7 +20,7 @@ export function CostOverResultsChart({
   evaluation,
   documentUuid,
 }: {
-  evaluation: Evaluation
+  evaluation: EvaluationDto
   documentUuid: string
 }) {
   const { project } = useCurrentProject()
@@ -75,8 +75,12 @@ export function CostOverResultsChart({
             xAxis: {
               label: 'Average result',
               type: 'number',
-              min: evaluation.configuration.detail!.range.from,
-              max: evaluation.configuration.detail!.range.to,
+              min: (
+                evaluation.configuration ?? evaluation.metadata.configuration
+              ).detail!.range.from,
+              max: (
+                evaluation.configuration ?? evaluation.metadata.configuration
+              ).detail!.range.to,
             },
             yAxis: {
               label: 'Average cost',

diff --git a/...valuations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/ResultsOverTime.tsx b/...valuations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/ResultsOverTime.tsx
@@ -2,7 +2,7 @@
 
 import { useCallback, useMemo } from 'react'
 
-import { Evaluation } from '@latitude-data/core/browser'
+import { EvaluationDto } from '@latitude-data/core/browser'
 import {
   AreaChart,
   Text,
@@ -26,7 +26,7 @@ export function ResultOverTimeChart({
   evaluation,
   documentUuid,
 }: {
-  evaluation: Evaluation
+  evaluation: EvaluationDto
   documentUuid: string
 }) {
   const { project } = useCurrentProject()
@@ -97,8 +97,12 @@ export function ResultOverTimeChart({
             yAxis: {
               label: 'Average result',
               type: 'number',
-              min: evaluation.configuration.detail!.range.from,
-              max: evaluation.configuration.detail!.range.to,
+              min: (
+                evaluation.configuration ?? evaluation.metadata.configuration
+              ).detail!.range.from,
+              max: (
+                evaluation.configuration ?? evaluation.metadata.configuration
+              ).detail!.range.to,
             },
             data: parsedData,
             tooltipLabel: (item) => {

diff --git a/...entUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx b/...entUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx
@@ -1,4 +1,4 @@
-import { Evaluation } from '@latitude-data/core/browser'
+import { EvaluationDto } from '@latitude-data/core/browser'
 
 import { CostOverResultsChart } from './CostOverResults'
 import { ResultOverTimeChart } from './ResultsOverTime'
@@ -7,7 +7,7 @@ export function NumericalCharts({
   evaluation,
   documentUuid,
 }: {
-  evaluation: Evaluation
+  evaluation: EvaluationDto
   documentUuid: string
 }) {
   return (

diff --git a/...nts/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx b/...nts/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx
@@ -13,7 +13,8 @@ export function EvaluationResultsCharts({
   documentUuid: string
 }) {
   const isNumerical =
-    evaluation.configuration.type === EvaluationResultableType.Number
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Number
 
   if (!isNumerical) return null
 

diff --git a/...tId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx b/...tId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx
@@ -48,7 +48,8 @@ export default async function ConnectedEvaluationLayout({
     uuid: params.commitUuid,
   })
   const isNumeric =
-    evaluation.configuration.type == EvaluationResultableType.Number
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ==
+    EvaluationResultableType.Number
 
   let provider
   if (evaluation.metadata.prompt) {
@@ -92,7 +93,14 @@ export default async function ConnectedEvaluationLayout({
                 {evaluation.name}
               </Text.H4M>
               <Text.H4M color='foregroundMuted'>
-                {TYPE_TEXT[evaluation.configuration.type]}
+                {
+                  TYPE_TEXT[
+                    (
+                      evaluation.configuration ??
+                      evaluation.metadata.configuration
+                    ).type
+                  ]
+                }
               </Text.H4M>
               <Tooltip
                 asChild

diff --git a/apps/web/src/components/EvaluationAggregatedResult/index.tsx b/apps/web/src/components/EvaluationAggregatedResult/index.tsx
@@ -136,7 +136,10 @@ export default function EvaluationAggregatedResult({
   documentUuid: string
   commitUuid: string
 }) {
-  if (evaluation.configuration.type === EvaluationResultableType.Number) {
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Number
+  ) {
     return (
       <EvaluationMeanValue
         evaluation={evaluation}
@@ -146,7 +149,10 @@ export default function EvaluationAggregatedResult({
     )
   }
 
-  if (evaluation.configuration.type === EvaluationResultableType.Boolean) {
+  if (
+    (evaluation.configuration ?? evaluation.metadata.configuration).type ===
+    EvaluationResultableType.Boolean
+  ) {
     return (
       <EvaluationBooleanValue
         evaluation={evaluation}

diff --git a/packages/core/drizzle/0082_legacy_evaluations.sql b/packages/core/drizzle/0082_legacy_evaluations.sql
@@ -0,0 +1,17 @@
+--- Move "configuration" column from "evaluations" to "llm_as_judge_evaluation_metadatas"
+ALTER TABLE "latitude"."llm_as_judge_evaluation_metadatas" ADD COLUMN "configuration" jsonb;--> statement-breakpoint
+
+--- Migrate configuration column from "evaluations" to "llm_as_judge_evaluation_metadatas"
+UPDATE "latitude"."llm_as_judge_evaluation_metadatas"
+SET "configuration" = "latitude"."evaluations"."configuration"
+FROM "latitude"."evaluations"
+WHERE "latitude"."evaluations"."metadata_id" = "latitude"."llm_as_judge_evaluation_metadatas"."id";--> statement-breakpoint
+
+--- Remove evaluations with NULL configuratios (these are llm_as_judge_evaluation_metadatas rows that are left when removing the original evaluation)
+DELETE FROM "latitude"."llm_as_judge_evaluation_metadatas" WHERE "latitude"."llm_as_judge_evaluation_metadatas"."configuration" IS NULL--> statement-breakpoint
+
+ALTER TABLE "latitude"."llm_as_judge_evaluation_metadatas" ALTER COLUMN "configuration" SET NOT NULL;--> statement-breakpoint
+
+--- Remove the now unnecessary columns
+ALTER TABLE "latitude"."evaluations" DROP COLUMN IF EXISTS "configuration";--> statement-breakpoint
+ALTER TABLE "latitude"."llm_as_judge_evaluation_metadatas" DROP COLUMN IF EXISTS "metadata_type";--> statement-breakpoint