From 71d6c401be0c94580558eb7ede01e0eb937824e0 Mon Sep 17 00:00:00 2001 From: Gerard Clos Date: Thu, 12 Sep 2024 10:30:34 +0200 Subject: [PATCH] feature: evaluation result types and configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evaluations/templates now have a configuration to configure the type of result we are expecting, stored in a jsonb column. EvaluationResults now have a polymorphic table to store the result depending on its type – This is so that we can aggregate results in SQL later on. --- apps/web/src/actions/evaluations/create.ts | 14 +- apps/web/src/app/(private)/error.tsx | 11 +- .../CreateEvaluationModal/index.tsx | 120 +- .../DocumentLogs/DocumentLogInfo/Metadata.tsx | 2 +- apps/web/src/stores/evaluationTemplates.ts | 2 +- .../core/drizzle/0051_colorful_lightspeed.sql | 33 + packages/core/drizzle/meta/0051_snapshot.json | 2189 +++++++++++++++++ packages/core/drizzle/meta/_journal.json | 7 + packages/core/src/constants.ts | 6 + .../src/data-access/evaluationTemplates.ts | 6 +- .../getConnectedDocumentsWithMetadata.test.ts | 5 +- .../connectedEvaluationsRepository/index.ts | 20 +- .../documentLogsRepository/index.ts | 2 +- .../findByDocumentUuid.test.ts | 8 +- .../evaluationResultsRepository/index.ts | 88 +- packages/core/src/schema/index.ts | 4 + .../models/evaluationResultableBooleans.ts | 13 + .../models/evaluationResultableNumbers.ts | 13 + .../models/evaluationResultableTexts.ts | 13 + .../src/schema/models/evaluationResults.ts | 16 +- .../src/schema/models/evaluationTemplates.ts | 6 +- .../core/src/schema/models/evaluations.ts | 9 +- .../core/src/schema/models/providerLogs.ts | 2 +- packages/core/src/schema/types.ts | 16 +- .../documentLogs/_createDocumentLogQuery.ts | 8 +- .../computeDocumentLogWithMetadata.test.ts | 7 +- .../services/evaluationResults/create.test.ts | 179 ++ .../src/services/evaluationResults/create.ts | 40 +- .../evaluationTemplates/create.test.ts | 33 + .../services/evaluationTemplates/create.ts | 11 +- .../src/services/evaluations/create.test.ts | 141 ++ .../core/src/services/evaluations/create.ts | 25 +- packages/core/src/services/evaluations/run.ts | 2 +- .../core/src/services/providerLogs/create.ts | 2 +- .../tests/factories/evaluationTemplates.ts | 4 + .../core/src/tests/factories/evaluations.ts | 12 +- packages/jobs/package.json | 1 + packages/jobs/tests/setup.ts | 4 + packages/jobs/vitest.config.mjs | 12 + pnpm-lock.yaml | 34 +- 40 files changed, 3027 insertions(+), 93 deletions(-) create mode 100644 packages/core/drizzle/0051_colorful_lightspeed.sql create mode 100644 packages/core/drizzle/meta/0051_snapshot.json create mode 100644 packages/core/src/schema/models/evaluationResultableBooleans.ts create mode 100644 packages/core/src/schema/models/evaluationResultableNumbers.ts create mode 100644 packages/core/src/schema/models/evaluationResultableTexts.ts create mode 100644 packages/core/src/services/evaluationResults/create.test.ts create mode 100644 packages/core/src/services/evaluations/create.test.ts create mode 100644 packages/jobs/tests/setup.ts create mode 100644 packages/jobs/vitest.config.mjs diff --git a/apps/web/src/actions/evaluations/create.ts b/apps/web/src/actions/evaluations/create.ts index a494806c3..20b64747f 100644 --- a/apps/web/src/actions/evaluations/create.ts +++ b/apps/web/src/actions/evaluations/create.ts @@ -1,6 +1,9 @@ 'use server' -import { EvaluationMetadataType } from '@latitude-data/core/browser' +import { + EvaluationMetadataType, + EvaluationResultableType, +} from '@latitude-data/core/browser' import { createEvaluation } from '@latitude-data/core/services/evaluations/create' import { z } from 'zod' @@ -16,6 +19,12 @@ export const createEvaluationAction = authProcedure .nativeEnum(EvaluationMetadataType) .optional() .default(EvaluationMetadataType.LlmAsJudge), + configuration: z.object({ + type: z.nativeEnum(EvaluationResultableType), + detail: z + .object({ range: z.object({ from: z.number(), to: z.number() }) }) + .optional(), + }), metadata: z .object({ prompt: z.string(), @@ -25,11 +34,14 @@ export const createEvaluationAction = authProcedure { type: 'json' }, ) .handler(async ({ input, ctx }) => { + console.log(input) + const result = await createEvaluation({ workspace: ctx.workspace, name: input.name, description: input.description, metadata: input.metadata, + configuration: input.configuration, type: input.type, }) diff --git a/apps/web/src/app/(private)/error.tsx b/apps/web/src/app/(private)/error.tsx index 986ea6946..84dec548a 100644 --- a/apps/web/src/app/(private)/error.tsx +++ b/apps/web/src/app/(private)/error.tsx @@ -4,7 +4,9 @@ import { useEffect } from 'react' import { ErrorComponent, useSession } from '@latitude-data/web-ui/browser' import { NAV_LINKS } from '$/app/(private)/_lib/constants' +import BreadcrumpLink from '$/components/BreadcrumpLink' import { AppLayout } from '$/components/layouts' +import { ROUTES } from '$/services/routes' export default function Error({ error, @@ -19,7 +21,14 @@ export default function Error({ return ( + ), + }, + { name: 'Error' }, + ]} navigationLinks={NAV_LINKS} > ({ + type: EvaluationResultableType.Text, + }) const router = useRouter() @@ -54,9 +63,10 @@ export default function CreateEvaluationModal({ name: title, description, metadata: { prompt }, + configuration, }) onClose(null) - }, [create, onClose, title, description, prompt]) + }, [create, onClose, title, description, prompt, configuration]) const titleError = useMemo(() => { if (!title) return 'Please enter a name for your evaluation.' @@ -82,8 +92,7 @@ export default function CreateEvaluationModal({ }} >
-
- Name + -
-
- Description + +