From 6dd0730568870af2d5783e7935cd6c9722157611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Sans=C3=B3n?= Date: Mon, 9 Sep 2024 10:17:09 +0200 Subject: [PATCH] Evaluation dashboard --- .../fetchConnectedDocuments.ts | 25 +++ .../src/app/(private)/_data-access/index.ts | 15 ++ .../_components/EvaluationTabs/index.tsx | 11 +- .../_components/EvaluationTitle/index.tsx | 19 +-- .../ConnectedDocumentsTable/index.tsx | 84 ++++++++++ .../dashboard/_components/EvaluationStats.tsx | 71 +++++++++ .../[evaluationUuid]/dashboard/layout.tsx | 14 -- .../[evaluationUuid]/dashboard/page.tsx | 31 +++- .../(evaluation)/[evaluationUuid]/layout.tsx | 8 +- .../DocumentLogs/DocumentLogInfo/Metadata.tsx | 3 +- .../DocumentLogs/DocumentLogsTable.tsx | 3 +- .../logs/_components/DocumentLogs/utils.ts | 6 +- .../src/app/_lib/formatCostInMillicents.ts | 3 + apps/web/src/stores/connectedEvaluations.ts | 51 ++++++ .../connectedEvaluationsRepository.ts | 147 ++++++++++++++++++ .../evaluationResultsRepository/index.ts | 8 +- packages/core/src/repositories/index.ts | 1 + 17 files changed, 447 insertions(+), 53 deletions(-) create mode 100644 apps/web/src/actions/connectedEvaluations/fetchConnectedDocuments.ts create mode 100644 apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/ConnectedDocumentsTable/index.tsx create mode 100644 apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx delete mode 100644 apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/layout.tsx create mode 100644 apps/web/src/app/_lib/formatCostInMillicents.ts create mode 100644 apps/web/src/stores/connectedEvaluations.ts create mode 100644 packages/core/src/repositories/connectedEvaluationsRepository.ts diff --git a/apps/web/src/actions/connectedEvaluations/fetchConnectedDocuments.ts b/apps/web/src/actions/connectedEvaluations/fetchConnectedDocuments.ts new file mode 100644 index 000000000..7da58f5d1 --- /dev/null +++ b/apps/web/src/actions/connectedEvaluations/fetchConnectedDocuments.ts @@ -0,0 +1,25 @@ +'use server' + +import { ConnectedEvaluationsRepository } from '@latitude-data/core/repositories' +import { z } from 'zod' + +import { authProcedure } from '../procedures' + +export const fetchConnectedDocumentsAction = authProcedure + .createServerAction() + .input( + z.object({ + evaluationId: z.number(), + }), + ) + .handler(async ({ input, ctx }) => { + const connectedEvaluationsScope = new ConnectedEvaluationsRepository( + ctx.workspace.id, + ) + const connectedDocuments = + await connectedEvaluationsScope.getConnectedDocumentsWithMetadata( + input.evaluationId, + ) + + return connectedDocuments.unwrap() + }) diff --git a/apps/web/src/app/(private)/_data-access/index.ts b/apps/web/src/app/(private)/_data-access/index.ts index b33ab0c9f..bb7e234c4 100644 --- a/apps/web/src/app/(private)/_data-access/index.ts +++ b/apps/web/src/app/(private)/_data-access/index.ts @@ -5,6 +5,7 @@ import { findAllEvaluationTemplates } from '@latitude-data/core/data-access' import { NotFoundError } from '@latitude-data/core/lib/errors' import { CommitsRepository, + ConnectedEvaluationsRepository, DocumentLogsRepository, DocumentVersionsRepository, EvaluationsRepository, @@ -170,3 +171,17 @@ export const getProviderLogCached = cache(async (uuid: string) => { const scope = new ProviderLogsRepository(workspace.id) return await scope.findByUuid(uuid).then((r) => r.unwrap()) }) + +export const getConnectedDocumentsWithMetadataCached = cache( + async (evaluationId: number) => { + const { workspace } = await getCurrentUser() + const connectedEvaluationsScope = new ConnectedEvaluationsRepository( + workspace.id, + ) + const result = + await connectedEvaluationsScope.getConnectedDocumentsWithMetadata( + evaluationId, + ) + return result.unwrap() + }, +) diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/_components/EvaluationTabs/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/_components/EvaluationTabs/index.tsx index 473945e0c..a05f99f08 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/_components/EvaluationTabs/index.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/_components/EvaluationTabs/index.tsx @@ -1,20 +1,23 @@ 'use client' +import { Evaluation } from '@latitude-data/core/browser' import { TabSelector } from '@latitude-data/web-ui' import { useNavigate } from '$/hooks/useNavigate' import { EvaluationRoutes, ROUTES } from '$/services/routes' import { useSelectedLayoutSegment } from 'next/navigation' export function EvaluationTabSelector({ - evaluationUuid, + evaluation, }: { - evaluationUuid: string + evaluation: Evaluation }) { const router = useNavigate() const selectedSegment = useSelectedLayoutSegment() as EvaluationRoutes | null const pathTo = (evaluationRoute: EvaluationRoutes) => { - const evaluationDetail = ROUTES.evaluations.detail({ uuid: evaluationUuid }) + const evaluationDetail = ROUTES.evaluations.detail({ + uuid: evaluation.uuid, + }) const detail = evaluationDetail[evaluationRoute] ?? evaluationDetail return detail.root } @@ -23,7 +26,7 @@ export function EvaluationTabSelector({
{ - return evaluations?.find((evaluation) => evaluation.uuid === evaluationUuid) - }, [evaluations, evaluationUuid]) - - if (!evaluation) return null +export function EvaluationTitle({ evaluation }: { evaluation: Evaluation }) { return (
{evaluation.name} diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/ConnectedDocumentsTable/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/ConnectedDocumentsTable/index.tsx new file mode 100644 index 000000000..ce94d362f --- /dev/null +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/ConnectedDocumentsTable/index.tsx @@ -0,0 +1,84 @@ +'use client' + +import { HEAD_COMMIT } from '@latitude-data/core/browser' +import type { ConnectedDocumentWithMetadata } from '@latitude-data/core/repositories' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, + Text, +} from '@latitude-data/web-ui' +import { formatCostInMillicents } from '$/app/_lib/formatCostInMillicents' +import { useNavigate } from '$/hooks/useNavigate' +import { ROUTES } from '$/services/routes' + +const ConnectedDocumentTableRow = ({ + document, + onSelect, +}: { + document: ConnectedDocumentWithMetadata + onSelect: () => void +}) => { + return ( + + + {document.path.split('/').pop()} + + + {document.evaluationLogs} + + + {document.totalTokens} + + + + {formatCostInMillicents(document.costInMillicents ?? 0)} + + + + ) +} + +export default function ConnectedDocumentsTable({ + connectedDocumentsWithMetadata, +}: { + connectedDocumentsWithMetadata: ConnectedDocumentWithMetadata[] +}) { + const navigate = useNavigate() + + return ( + + + + Name + Logs evaluated + Tokens + Cost + + + + {connectedDocumentsWithMetadata.map((document) => ( + + navigate.push( + ROUTES.projects + .detail({ id: document.projectId }) + .commits.detail({ uuid: HEAD_COMMIT }) + .documents.detail({ uuid: document.documentUuid }).logs.root, // TODO: Navigate to the document evaluation details page + ) + } + /> + ))} + +
+ ) +} diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx new file mode 100644 index 000000000..190b2a52b --- /dev/null +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/_components/EvaluationStats.tsx @@ -0,0 +1,71 @@ +'use client' + +import { useEffect, useState } from 'react' + +import { readMetadata } from '@latitude-data/compiler' +import { EvaluationDto } from '@latitude-data/core/browser' +import { ConnectedDocumentWithMetadata } from '@latitude-data/core/repositories' +import { Skeleton, Text } from '@latitude-data/web-ui' +import { formatCostInMillicents } from '$/app/_lib/formatCostInMillicents' +import useConnectedDocuments from '$/stores/connectedEvaluations' + +export function Stat({ label, value }: { label: string; value?: string }) { + return ( +
+ {label} + {value == undefined ? ( + + ) : ( + {value} + )} +
+ ) +} + +export default function EvaluationStats({ + evaluation, + connectedDocumentsWithMetadata, +}: { + evaluation: EvaluationDto + connectedDocumentsWithMetadata: ConnectedDocumentWithMetadata[] +}) { + const [model, setModel] = useState() + const { data: connectedDocuments, isLoading: connectedDocumentsLoading } = + useConnectedDocuments({ evaluation }) + + useEffect(() => { + readMetadata({ prompt: evaluation.metadata.prompt }).then((metadata) => { + const metadataModel = (metadata.config['model'] as string) ?? 'Unknown' + setModel(metadataModel) + }) + }, [evaluation.metadata]) + + return ( +
+ + + acc + doc.evaluationLogs, 0) + .toString()} + /> + acc + doc.costInMillicents, + 0, + ), + )} + /> +
+ ) +} diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/layout.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/layout.tsx deleted file mode 100644 index 11bf527d1..000000000 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/layout.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { ReactNode } from 'react' - -import { Text } from '@latitude-data/web-ui' - -export default function DashboardLayout({ children }: { children: ReactNode }) { - return ( - <> - {children} -
- (Really cool dashboard) -
- - ) -} diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/page.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/page.tsx index 118f37351..348e77a98 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/page.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/dashboard/page.tsx @@ -1,3 +1,30 @@ -export default function DashboardPage() { - return null // --> layout.tsx +import { Container } from '@latitude-data/web-ui' +import { + getConnectedDocumentsWithMetadataCached, + getEvaluationByUuidCached, +} from '$/app/(private)/_data-access' + +import ConnectedDocumentsTable from './_components/ConnectedDocumentsTable' +import EvaluationStats from './_components/EvaluationStats' + +export default async function DashboardPage({ + params, +}: { + params: { evaluationUuid: string } +}) { + const evaluation = await getEvaluationByUuidCached(params.evaluationUuid) + const connectedDocumentsWithMetadata = + await getConnectedDocumentsWithMetadataCached(evaluation.id) + + return ( + + + + + ) } diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/layout.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/layout.tsx index 5a803ff9d..fc08df6f3 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/layout.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/layout.tsx @@ -1,5 +1,7 @@ import { ReactNode } from 'react' +import { getEvaluationByUuidCached } from '$/app/(private)/_data-access' + import { EvaluationTabSelector } from './_components/EvaluationTabs' import { EvaluationTitle } from './_components/EvaluationTitle' @@ -10,10 +12,12 @@ export default async function EvaluationLayout({ params: { evaluationUuid: string } children: ReactNode }) { + const evaluation = await getEvaluationByUuidCached(params.evaluationUuid) + return (
- - + +
{children}
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx index 73c499e9f..881e62519 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx @@ -9,10 +9,11 @@ import { Text, Tooltip, } from '@latitude-data/web-ui' +import { formatCostInMillicents } from '$/app/_lib/formatCostInMillicents' import useProviderApiKeys from '$/stores/providerApiKeys' import { format } from 'date-fns' -import { formatCostInMillicents, formatDuration } from '../utils' +import { formatDuration } from '../utils' function MetadataItem({ label, diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx index 6b6a20ce1..ab5567fa5 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogsTable.tsx @@ -12,8 +12,9 @@ import { TableRow, Text, } from '@latitude-data/web-ui' +import { formatCostInMillicents } from '$/app/_lib/formatCostInMillicents' -import { formatCostInMillicents, formatDuration, relativeTime } from './utils' +import { formatDuration, relativeTime } from './utils' export const DocumentLogsTable = ({ documentLogs, diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/utils.ts b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/utils.ts index dc78ce964..6675c8147 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/utils.ts +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/utils.ts @@ -24,8 +24,4 @@ function formatDuration(duration: number) { return `${hours > 0 ? `${hours}h ` : ''}${minutes > 0 ? `${minutes}m ` : ''}${seconds}s` } -function formatCostInMillicents(cost_in_millicents: number) { - return `$ ${cost_in_millicents / 100_000}` -} - -export { relativeTime, formatDuration, formatCostInMillicents } +export { relativeTime, formatDuration } diff --git a/apps/web/src/app/_lib/formatCostInMillicents.ts b/apps/web/src/app/_lib/formatCostInMillicents.ts new file mode 100644 index 000000000..418212909 --- /dev/null +++ b/apps/web/src/app/_lib/formatCostInMillicents.ts @@ -0,0 +1,3 @@ +export function formatCostInMillicents(cost_in_millicents: number) { + return `$ ${cost_in_millicents / 100_000}` +} diff --git a/apps/web/src/stores/connectedEvaluations.ts b/apps/web/src/stores/connectedEvaluations.ts new file mode 100644 index 000000000..ae704c9fc --- /dev/null +++ b/apps/web/src/stores/connectedEvaluations.ts @@ -0,0 +1,51 @@ +'use client' + +import type { DocumentVersion, Evaluation } from '@latitude-data/core/browser' +import { useSession, useToast } from '@latitude-data/web-ui' +import { fetchConnectedDocumentsAction } from '$/actions/connectedEvaluations/fetchConnectedDocuments' +import useSWR, { SWRConfiguration } from 'swr' + +export default function useConnectedDocuments( + { + evaluation, + }: { + evaluation: Evaluation + }, + opts: SWRConfiguration = {}, +) { + const { workspace } = useSession() + const { toast } = useToast() + + const { + data = [], + isLoading, + error, + } = useSWR( + ['connectedDocuments', workspace.id, evaluation.id], + async () => { + const [data, error] = await fetchConnectedDocumentsAction({ + evaluationId: evaluation.id, + }) + + if (error) { + console.error(error) + + toast({ + title: 'Error fetching evaluation connections', + description: error.formErrors?.[0] || error.message, + variant: 'destructive', + }) + throw error + } + + return data + }, + opts, + ) + + return { + data, + isLoading, + error, + } +} diff --git a/packages/core/src/repositories/connectedEvaluationsRepository.ts b/packages/core/src/repositories/connectedEvaluationsRepository.ts new file mode 100644 index 000000000..a7e1d9890 --- /dev/null +++ b/packages/core/src/repositories/connectedEvaluationsRepository.ts @@ -0,0 +1,147 @@ +import { + and, + count, + eq, + getTableColumns, + isNotNull, + max, + sql, + sum, +} from 'drizzle-orm' + +import { ConnectedEvaluation, DocumentVersion } from '../browser' +import { LatitudeError, Result, TypedResult } from '../lib' +import { + connectedEvaluations, + documentLogs, + evaluationResults, + evaluations, + providerLogs, +} from '../schema' +import { DocumentVersionsRepository } from './documentVersionsRepository' +import Repository from './repository' + +const tt = getTableColumns(connectedEvaluations) + +export type ConnectedDocumentWithMetadata = DocumentVersion & { + projectId: number + evaluationLogs: number + totalTokens: number + costInMillicents: number +} + +export class ConnectedEvaluationsRepository extends Repository< + typeof tt, + ConnectedEvaluation +> { + get scope() { + return this.db + .select(tt) + .from(connectedEvaluations) + .innerJoin( + evaluations, + eq(connectedEvaluations.evaluationId, evaluations.id), + ) + .where(eq(evaluations.workspaceId, this.workspaceId)) + .as('connectedEvaluationsScope') + } + + async findByEvaluationId(id: number) { + const result = await this.db + .select() + .from(this.scope) + .where(eq(this.scope.evaluationId, id)) + + return Result.ok(result) + } + + async findByDocumentUuid(uuid: string) { + const result = await this.db + .select() + .from(this.scope) + .where(eq(this.scope.documentUuid, uuid)) + + return Result.ok(result[0]!) + } + + async getConnectedDocumentsWithMetadata( + evaluationId: number, + ): Promise> { + const documentVersionsScope = new DocumentVersionsRepository( + this.workspaceId, + this.db, + ) + + const aggregatedMetadata = this.db + .select({ + evaluationId: evaluationResults.evaluationId, + documentUuid: documentLogs.documentUuid, + evaluationLogs: count(evaluationResults.id).as('evaluation_logs'), + totalTokens: sum(providerLogs.tokens).as('total_tokens'), + costInMillicents: sum(providerLogs.cost_in_millicents).as( + 'cost_in_millicents', + ), + }) + .from(evaluationResults) + .leftJoin( + documentLogs, + eq(documentLogs.id, evaluationResults.documentLogId), + ) + .leftJoin( + providerLogs, + eq(providerLogs.id, evaluationResults.providerLogId), + ) + .groupBy(documentLogs.documentUuid, evaluationResults.evaluationId) + .as('aggregated_metadata') + + const latestDocumentVersions = this.db + .select({ + documentUuid: documentVersionsScope.scope.documentUuid, + latestMergedAt: max(documentVersionsScope.scope.mergedAt).as( + 'latest_merged_at', + ), + }) + .from(documentVersionsScope.scope) + .where(isNotNull(documentVersionsScope.scope.mergedAt)) // Only account for merged documents + .groupBy(documentVersionsScope.scope.documentUuid) + .as('latest_document_versions') + + const result = await this.db + .select({ + ...documentVersionsScope.scope._.selectedFields, + evaluationLogs: + sql`COALESCE(${aggregatedMetadata.evaluationLogs}, 0)` + .mapWith(Number) + .as('evaluation_logs'), + totalTokens: sql`COALESCE(${aggregatedMetadata.totalTokens}, 0)` + .mapWith(Number) + .as('total_tokens'), + costInMillicents: + sql`COALESCE(${aggregatedMetadata.costInMillicents}, 0)` + .mapWith(Number) + .as('cost_in_millicents'), + }) + .from(this.scope) + .innerJoin( + latestDocumentVersions, + eq(this.scope.documentUuid, latestDocumentVersions.documentUuid), + ) + .innerJoin( + documentVersionsScope.scope, + and( + eq(this.scope.documentUuid, documentVersionsScope.scope.documentUuid), + eq( + documentVersionsScope.scope.mergedAt, + latestDocumentVersions.latestMergedAt, + ), + ), + ) + .leftJoin( + aggregatedMetadata, + eq(this.scope.evaluationId, aggregatedMetadata.evaluationId), + ) + .where(eq(this.scope.evaluationId, evaluationId)) + + return Result.ok(result.filter((r) => r.deletedAt == null)) // Only show non-removed documents + } +} diff --git a/packages/core/src/repositories/evaluationResultsRepository/index.ts b/packages/core/src/repositories/evaluationResultsRepository/index.ts index 2dd2c9c7a..a988297a8 100644 --- a/packages/core/src/repositories/evaluationResultsRepository/index.ts +++ b/packages/core/src/repositories/evaluationResultsRepository/index.ts @@ -1,16 +1,10 @@ import { eq, getTableColumns } from 'drizzle-orm' -import { Commit, DocumentLog, EvaluationResult } from '../../browser' +import { EvaluationResult } from '../../browser' import { Result } from '../../lib' import { documentLogs, evaluationResults, evaluations } from '../../schema' import Repository from '../repository' -export type DocumentLogWithMetadata = DocumentLog & { - commit: Commit - tokens: number | null - cost_in_millicents: number | null -} - const tt = getTableColumns(evaluationResults) export class EvaluationResultsRepository extends Repository< diff --git a/packages/core/src/repositories/index.ts b/packages/core/src/repositories/index.ts index 2fc7d35d9..3d27d5c7a 100644 --- a/packages/core/src/repositories/index.ts +++ b/packages/core/src/repositories/index.ts @@ -10,3 +10,4 @@ export * from './documentLogsRepository' export * from './membershipsRepository' export * from './evaluationsRepository' export * from './datasetsRepository' +export * from './connectedEvaluationsRepository'