diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/index.tsx new file mode 100644 index 000000000..0706c0b5b --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/index.tsx @@ -0,0 +1,51 @@ +'use client' + +import { + Commit, + EvaluationAggregationTotals, + EvaluationDto, + EvaluationMeanValue, + EvaluationModalValue, +} from '@latitude-data/core/browser' +import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories' + +import { EvaluationResults } from '../EvaluationResults' +import { MetricsSummary } from '../MetricsSummary' +import { useEvaluationStatus } from './useEvaluationStatus' + +export default function Content({ + commit, + evaluation, + evaluationResults, + documentUuid, + aggregationTotals, + isNumeric, + meanOrModal, +}: { + commit: Commit + evaluation: EvaluationDto + documentUuid: string + evaluationResults: EvaluationResultWithMetadata[] + aggregationTotals: EvaluationAggregationTotals + isNumeric: T + meanOrModal: T extends true ? EvaluationMeanValue : EvaluationModalValue +}) { + const { jobs } = useEvaluationStatus({ evaluation }) + return ( + <> + + + + ) +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useEvaluationStatus.ts b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useEvaluationStatus.ts new file mode 100644 index 000000000..68e7850f9 --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useEvaluationStatus.ts @@ -0,0 +1,72 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { Evaluation } from '@latitude-data/core/browser' +import { useCurrentDocument } from '@latitude-data/web-ui' +import { + useSockets, + type EventArgs, +} from '$/components/Providers/WebsocketsProvider/useSockets' + +import { isEvaluationRunDone } from '../../_lib/isEvaluationRunDone' +import { useRefetchStats } from './useRefetchStats' + +const DISAPERING_IN_MS = 5000 + +export function useEvaluationStatus({ + evaluation, +}: { + evaluation: Evaluation +}) { + const timeoutRef = useRef(null) + const [jobs, setJobs] = useState[]>([]) + const document = useCurrentDocument() + const { refetchStats } = useRefetchStats({ evaluation, document }) + const onMessage = useCallback( + (args: EventArgs<'evaluationStatus'>) => { + if (evaluation.id !== args.evaluationId) return + if (document.documentUuid !== args.documentUuid) return + + const done = isEvaluationRunDone(args) + + if (done) { + refetchStats() + } + + setJobs((prevJobs) => { + const jobIndex = prevJobs.findIndex( + (job) => job.batchId === args.batchId, + ) + + if (jobIndex === -1) { + return [...prevJobs, args] + } else { + const newJobs = [...prevJobs] + newJobs[jobIndex] = args + + if (done) { + setTimeout(() => { + setJobs((currentJobs) => + currentJobs.filter((job) => job.batchId !== args.batchId), + ) + }, DISAPERING_IN_MS) + } + + return newJobs + } + }) + }, + [evaluation.id, document.documentUuid, refetchStats], + ) + + useEffect(() => { + return () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current) + } + } + }, []) + + useSockets({ event: 'evaluationStatus', onMessage }) + + return { jobs } +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useRefetchStats.ts b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useRefetchStats.ts new file mode 100644 index 000000000..d0899f45d --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Content/useRefetchStats.ts @@ -0,0 +1,79 @@ +import { useCallback } from 'react' + +import { + DocumentVersion, + Evaluation, + EvaluationResultableType, +} from '@latitude-data/core/browser' +import { useCurrentCommit } from '@latitude-data/web-ui' +import useEvaluationResultsCounters from '$/stores/evaluationResultCharts/evaluationResultsCounters' +import useEvaluationResultsMeanValue from '$/stores/evaluationResultCharts/evaluationResultsMeanValue' +import useEvaluationResultsModalValue from '$/stores/evaluationResultCharts/evaluationResultsModalValue' +import useAverageResultsAndCostOverCommit from '$/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore' +import useAverageResultOverTime from '$/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore' + +export function useRefetchStats({ + evaluation, + document, +}: { + evaluation: Evaluation + document: DocumentVersion +}) { + const { commit } = useCurrentCommit() + const evaluationId = evaluation.id + const commitUuid = commit.uuid + const documentUuid = document.documentUuid + const isNumeric = + evaluation.configuration.type === EvaluationResultableType.Number + + const { refetch: refetchAverageResulstAndCostsOverCommit } = + useAverageResultsAndCostOverCommit({ + evaluation, + documentUuid, + }) + const { refetch: refetchAverageResultOverTime } = useAverageResultOverTime({ + evaluation, + documentUuid, + }) + const { refetch: refetchMean } = useEvaluationResultsMeanValue({ + commitUuid, + documentUuid, + evaluationId, + }) + const { refetch: refetchModal } = useEvaluationResultsModalValue({ + commitUuid, + documentUuid, + evaluationId, + }) + const { refetch: refetchTotals } = useEvaluationResultsCounters({ + commitUuid, + documentUuid, + evaluationId, + }) + + const refetchStats = useCallback(() => { + console.log('refetchStats') + + Promise.all([ + refetchTotals(), + ...(isNumeric + ? [ + refetchMean(), + refetchAverageResulstAndCostsOverCommit(), + refetchAverageResultOverTime(), + ] + : [refetchModal()]), + ]) + }, [ + isNumeric, + refetchMean, + refetchModal, + refetchTotals, + refetchAverageResulstAndCostsOverCommit, + refetchAverageResultOverTime, + ]) + + return { + refetchStats, + } +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx index 2871779aa..6bbc9c98f 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultsTable.tsx @@ -48,6 +48,9 @@ export const ResultCellContent = ({ return {value as string} } +type EvaluationResultRow = EvaluationResultWithMetadata & { + realtimeAdded?: boolean +} export const EvaluationResultsTable = ({ evaluation, evaluationResults, @@ -55,8 +58,8 @@ export const EvaluationResultsTable = ({ setSelectedResult, }: { evaluation: EvaluationDto - evaluationResults: EvaluationResultWithMetadata[] - selectedResult: EvaluationResultWithMetadata | undefined + evaluationResults: EvaluationResultRow[] + selectedResult: EvaluationResultRow | undefined setSelectedResult: (log: EvaluationResultWithMetadata | undefined) => void }) => { return ( @@ -71,9 +74,9 @@ export const EvaluationResultsTable = ({ - {evaluationResults.map((evaluationResult, idx) => ( + {evaluationResults.map((evaluationResult) => ( setSelectedResult( selectedResult?.id === evaluationResult.id @@ -85,12 +88,18 @@ export const EvaluationResultsTable = ({ 'cursor-pointer border-b-[0.5px] h-12 max-h-12 border-border', { 'bg-secondary': selectedResult?.id === evaluationResult.id, + 'animate-flash': evaluationResult.realtimeAdded, }, )} > - {relativeTime(evaluationResult.createdAt)} + diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationStatusBanner/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationStatusBanner/index.tsx index 40889e016..03920171d 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationStatusBanner/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationStatusBanner/index.tsx @@ -1,82 +1,31 @@ 'use client' -import { useCallback, useEffect, useRef, useState } from 'react' +import { ProgressIndicator } from '@latitude-data/web-ui' +import { isEvaluationRunDone } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone' +import { type EventArgs } from '$/components/Providers/WebsocketsProvider/useSockets' -import { EvaluationDto } from '@latitude-data/core/browser' -import { ProgressIndicator, useCurrentDocument } from '@latitude-data/web-ui' -import { - useSockets, - type EventArgs, -} from '$/components/Providers/WebsocketsProvider/useSockets' - -const DISAPERING_IN_MS = 5000 export function EvaluationStatusBanner({ - evaluation, + jobs, }: { - evaluation: EvaluationDto + jobs: EventArgs<'evaluationStatus'>[] }) { - const timeoutRef = useRef(null) - const [jobs, setJobs] = useState[]>([]) - const document = useCurrentDocument() - - const onMessage = useCallback( - (args: EventArgs<'evaluationStatus'>) => { - if (evaluation.id !== args.evaluationId) return - if (document.documentUuid !== args.documentUuid) return - - setJobs((prevJobs) => { - const jobIndex = prevJobs.findIndex( - (job) => job.batchId === args.batchId, - ) - - if (jobIndex === -1) { - return [...prevJobs, args] - } else { - const newJobs = [...prevJobs] - newJobs[jobIndex] = args - - if (isDone(args)) { - setTimeout(() => { - setJobs((currentJobs) => - currentJobs.filter((job) => job.batchId !== args.batchId), - ) - }, DISAPERING_IN_MS) - } - - return newJobs - } - }) - }, - [evaluation.id, document.documentUuid], - ) - - useEffect(() => { - return () => { - if (timeoutRef.current) { - clearTimeout(timeoutRef.current) - } - } - }, []) - - useSockets({ event: 'evaluationStatus', onMessage }) - return ( <> {jobs.map((job) => (
- {!isDone(job) && ( + {!isEvaluationRunDone(job) && ( {`Running batch evaluation ${job.completed}/${job.total}`} )} - {job.errors > 0 && !isDone(job) && ( + {job.errors > 0 && ( Some evaluations failed to run. We won't retry them automatically to avoid increasing provider costs. Total errors:{' '} {job.errors} )} - {isDone(job) && ( + {isEvaluationRunDone(job) && ( Batch evaluation completed! Total evaluations:{' '} {job.total} ยท Total errors:{' '} @@ -89,7 +38,3 @@ export function EvaluationStatusBanner({ ) } - -function isDone(job: EventArgs<'evaluationStatus'>) { - return job.total === job.completed + job.errors -} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/index.tsx index 74a212dcb..cd1d6a43f 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/index.tsx @@ -1,6 +1,6 @@ 'use client' -import { useEffect, useState } from 'react' +import { useState } from 'react' import { EvaluationDto } from '@latitude-data/core/browser' import { type EvaluationResultWithMetadata } from '@latitude-data/core/repositories' @@ -11,6 +11,10 @@ import { useCurrentDocument, useCurrentProject, } from '@latitude-data/web-ui' +import { + EventArgs, + useSockets, +} from '$/components/Providers/WebsocketsProvider/useSockets' import { DocumentRoutes, ROUTES } from '$/services/routes' import useEvaluationResultsWithMetadata from '$/stores/evaluationResultsWithMetadata' import { useProviderLog } from '$/stores/providerLogs' @@ -20,14 +24,14 @@ import { EvaluationResultInfo } from './EvaluationResultInfo' import { EvaluationResultsTable } from './EvaluationResultsTable' import { EvaluationStatusBanner } from './EvaluationStatusBanner' -const FIVE_SECONDS = 5000 - export function EvaluationResults({ evaluation, evaluationResults: serverData, + jobs, }: { evaluation: EvaluationDto evaluationResults: EvaluationResultWithMetadata[] + jobs: EventArgs<'evaluationStatus'>[] }) { const { project } = useCurrentProject() const { commit } = useCurrentCommit() @@ -47,20 +51,28 @@ export function EvaluationResults({ fallbackData: serverData, }, ) + const onMessage = (args: EventArgs<'evaluationResultCreated'>) => { + if (evaluation.id !== args.evaluationId) return + if (document.documentUuid !== args.documentUuid) return - // FIXME: Listen to websockets to update new evaluation results - useEffect(() => { - const interval = setInterval(() => { - mutate() - }, FIVE_SECONDS) + const createdAt = new Date(args.row.createdAt) + mutate( + (prevData) => { + return [ + { ...args.row, createdAt, realtimeAdded: true }, + ...(prevData ?? []), + ] + }, + { revalidate: false }, + ) + } - return () => clearInterval(interval) - }, [mutate]) + useSockets({ event: 'evaluationResultCreated', onMessage }) return (
Evaluation Results - +
{evaluationResults.length === 0 && ( diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx index a0cf4de21..4eacd1718 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/MeanValuePanel/index.tsx @@ -1,6 +1,6 @@ 'use client' -import { Evaluation, EvaluationMeanValue } from '@latitude-data/core/browser' +import { EvaluationDto, EvaluationMeanValue } from '@latitude-data/core/browser' import { RangeBadge } from '@latitude-data/web-ui' import useEvaluationResultsMeanValue from '$/stores/evaluationResultCharts/evaluationResultsMeanValue' @@ -14,7 +14,7 @@ export default function MeanValuePanel({ }: { commitUuid: string documentUuid: string - evaluation: Evaluation + evaluation: EvaluationDto mean: EvaluationMeanValue }) { const { data } = useEvaluationResultsMeanValue( diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/index.tsx index 92dd0fa61..3cece5925 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/BigNumberPanels/index.tsx @@ -1,93 +1,30 @@ import { Commit, - Evaluation, - EvaluationResultableType, - Workspace, + EvaluationAggregationTotals, + EvaluationDto, + EvaluationMeanValue, + EvaluationModalValue, } from '@latitude-data/core/browser' -import { - getEvaluationMeanValueQuery, - getEvaluationModalValueQuery, - getEvaluationTotalsQuery, -} from '@latitude-data/core/services/evaluationResults/index' import MeanValuePanel from './MeanValuePanel' import ModalValuePanel from './ModalValuePanel' import TotalsPanels from './TotalsPanels' -async function MeanPanel({ - workspaceId, - evaluation, - documentUuid, +export function BigNumberPanels({ commit, -}: { - workspaceId: number - evaluation: Evaluation - documentUuid: string - commit: Commit -}) { - const mean = await getEvaluationMeanValueQuery({ - workspaceId, - evaluation, - documentUuid, - commit, - }) - - return ( - - ) -} - -async function ModalPanel({ - workspaceId, evaluation, documentUuid, - commit, + aggregationTotals, + isNumeric, + meanOrModal, }: { - workspaceId: number - evaluation: Evaluation - documentUuid: string - commit: Commit -}) { - const modal = await getEvaluationModalValueQuery({ - workspaceId, - evaluation, - documentUuid, - commit, - }) - return ( - - ) -} - -export async function BigNumberPanels({ - workspace, - commit, - evaluation, - documentUuid, -}: { - workspace: Workspace + isNumeric: T commit: Commit - evaluation: Evaluation + evaluation: EvaluationDto documentUuid: string + aggregationTotals: EvaluationAggregationTotals + meanOrModal: T extends true ? EvaluationMeanValue : EvaluationModalValue }) { - const aggregationTotals = await getEvaluationTotalsQuery({ - workspaceId: workspace.id, - commit, - evaluation, - documentUuid, - }) - const isNumeric = - evaluation.configuration.type == EvaluationResultableType.Number return (
{isNumeric && ( - )} {!isNumeric && ( - )} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx index 96626ebe9..3c153ab92 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/Numerical/index.tsx @@ -1,5 +1,3 @@ -'use client' - import { Evaluation } from '@latitude-data/core/browser' import { CostOverResultsChart } from './CostOverResults' diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx index 2740550a4..45b27fccc 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/Charts/index.tsx @@ -1,7 +1,5 @@ -'use client' - import { - Evaluation, + EvaluationDto, EvaluationResultableType, } from '@latitude-data/core/browser' @@ -11,7 +9,7 @@ export function EvaluationResultsCharts({ evaluation, documentUuid, }: { - evaluation: Evaluation + evaluation: EvaluationDto documentUuid: string }) { const isNumerical = diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/index.tsx index e3b3a16b4..acee58ec9 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/MetricsSummary/index.tsx @@ -1,18 +1,30 @@ -import { Commit, Evaluation, Workspace } from '@latitude-data/core/browser' +'use client' + +import { + Commit, + EvaluationAggregationTotals, + EvaluationDto, + EvaluationMeanValue, + EvaluationModalValue, +} from '@latitude-data/core/browser' import { BigNumberPanels } from './BigNumberPanels' import { EvaluationResultsCharts } from './Charts' -export function MetricsSummary({ - workspace, +export function MetricsSummary({ commit, evaluation, documentUuid, + aggregationTotals, + meanOrModal, + isNumeric, }: { - workspace: Workspace commit: Commit - evaluation: Evaluation + evaluation: EvaluationDto documentUuid: string + aggregationTotals: EvaluationAggregationTotals + isNumeric: T + meanOrModal: T extends true ? EvaluationMeanValue : EvaluationModalValue }) { return (
@@ -22,10 +34,12 @@ export function MetricsSummary({ />
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone.ts b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone.ts new file mode 100644 index 000000000..dc53de3e3 --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone.ts @@ -0,0 +1,5 @@ +import { type EventArgs } from '$/components/Providers/WebsocketsProvider/useSockets' + +export function isEvaluationRunDone(job: EventArgs<'evaluationStatus'>) { + return job.total === job.completed + job.errors +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx index 7d2a50212..da2c74730 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx @@ -1,8 +1,20 @@ import { ReactNode } from 'react' -import { EvaluationResultableType } from '@latitude-data/core/browser' +import { + Commit, + Evaluation, + EvaluationAggregationTotals, + EvaluationMeanValue, + EvaluationModalValue, + EvaluationResultableType, +} from '@latitude-data/core/browser' import { EvaluationsRepository } from '@latitude-data/core/repositories' import { computeEvaluationResultsWithMetadata } from '@latitude-data/core/services/evaluationResults/computeEvaluationResultsWithMetadata' +import { + getEvaluationMeanValueQuery, + getEvaluationModalValueQuery, + getEvaluationTotalsQuery, +} from '@latitude-data/core/services/evaluationResults/index' import { TableWithHeader, Text } from '@latitude-data/web-ui' import { findCommitCached } from '$/app/(private)/_data-access' import BreadcrumbLink from '$/components/BreadcrumbLink' @@ -11,14 +23,62 @@ import { getCurrentUser } from '$/services/auth/getCurrentUser' import { ROUTES } from '$/services/routes' import { Actions } from './_components/Actions' -import { EvaluationResults } from './_components/EvaluationResults' -import { MetricsSummary } from './_components/MetricsSummary' +import Content from './_components/Content' const TYPE_TEXT: Record = { [EvaluationResultableType.Text]: 'Text', [EvaluationResultableType.Number]: 'Numerical', [EvaluationResultableType.Boolean]: 'Boolean', } + +type ReturnType = T extends true + ? { + aggregationTotals: EvaluationAggregationTotals + meanOrModal: EvaluationMeanValue + } + : { + aggregationTotals: EvaluationAggregationTotals + meanOrModal: EvaluationModalValue + } +async function fetchData({ + workspaceId, + evaluation, + documentUuid, + isNumeric, + commit, +}: { + isNumeric: T + workspaceId: number + evaluation: Evaluation + documentUuid: string + commit: Commit +}): Promise> { + const aggregationTotals = await getEvaluationTotalsQuery({ + workspaceId, + commit, + evaluation, + documentUuid, + }) + + if (isNumeric) { + const mean = await getEvaluationMeanValueQuery({ + workspaceId, + evaluation, + documentUuid, + commit, + }) + return { aggregationTotals, meanOrModal: mean } as ReturnType + } + + const modal = await getEvaluationModalValueQuery({ + workspaceId, + evaluation, + documentUuid, + commit, + }) + return { aggregationTotals, meanOrModal: modal } as ReturnType +} + export default async function ConnectedEvaluationLayout({ params, children, @@ -49,6 +109,15 @@ export default async function ConnectedEvaluationLayout({ draft: commit, limit: 1000, }).then((r) => r.unwrap()) + const isNumeric = + evaluation.configuration.type == EvaluationResultableType.Number + const data = await fetchData({ + workspaceId: workspace.id, + evaluation, + documentUuid: params.documentUuid, + isNumeric, + commit, + }) return (
@@ -94,15 +163,14 @@ export default async function ConnectedEvaluationLayout({ /> } /> - -
) diff --git a/apps/web/src/stores/evaluationResultCharts/evaluationResultsCounters.ts b/apps/web/src/stores/evaluationResultCharts/evaluationResultsCounters.ts index 37a36ea77..2352e0228 100644 --- a/apps/web/src/stores/evaluationResultCharts/evaluationResultsCounters.ts +++ b/apps/web/src/stores/evaluationResultCharts/evaluationResultsCounters.ts @@ -1,4 +1,6 @@ -import { useCurrentProject } from '@latitude-data/web-ui' +import { useCallback } from 'react' + +import { useCurrentProject, useToast } from '@latitude-data/web-ui' import { computeEvaluationResultsCountersAction } from '$/actions/evaluationResults/computeEvaluationResultsCountersAction' import useSWR, { SWRConfiguration } from 'swr' @@ -12,23 +14,32 @@ export default function useEvaluationResultsCounters( documentUuid: string evaluationId: number }, - opts: SWRConfiguration = {}, + { fallbackData }: SWRConfiguration = {}, ) { const { project } = useCurrentProject() + const { toast } = useToast() + const fetcher = useCallback(async () => { + const [data, error] = await computeEvaluationResultsCountersAction({ + projectId: project.id, + commitUuid, + documentUuid, + evaluationId, + }) + + if (error) { + toast({ + title: 'Error fetching evaluation stats', + description: error.formErrors?.[0] || error.message, + variant: 'destructive', + }) + return null + } + return data + }, [commitUuid, documentUuid, evaluationId, project.id]) const { data, isLoading, error, mutate } = useSWR( ['evaluationResultsCounters', commitUuid, documentUuid, evaluationId], - async () => { - const [data, error] = await computeEvaluationResultsCountersAction({ - projectId: project.id, - commitUuid, - documentUuid, - evaluationId, - }) - - if (error) return null - return data - }, - opts, + fetcher, + { fallbackData }, ) return { diff --git a/apps/web/src/stores/evaluationResultCharts/evaluationResultsMeanValue.ts b/apps/web/src/stores/evaluationResultCharts/evaluationResultsMeanValue.ts index 8e837d005..89db73114 100644 --- a/apps/web/src/stores/evaluationResultCharts/evaluationResultsMeanValue.ts +++ b/apps/web/src/stores/evaluationResultCharts/evaluationResultsMeanValue.ts @@ -1,4 +1,6 @@ -import { useCurrentProject } from '@latitude-data/web-ui' +import { useCallback } from 'react' + +import { useCurrentProject, useToast } from '@latitude-data/web-ui' import { computeEvaluationResultsMeanValueAction } from '$/actions/evaluationResults/computeEvaluationResultsMeanValueAction' import useSWR, { SWRConfiguration } from 'swr' @@ -12,24 +14,33 @@ export default function useEvaluationResultsMeanValue( documentUuid: string evaluationId: number }, - opts: SWRConfiguration = {}, + { fallbackData }: SWRConfiguration = {}, ) { + const { toast } = useToast() const { project } = useCurrentProject() - const { data, isLoading, error, mutate } = useSWR( - ['evaluationResultsMeanQuery', commitUuid, documentUuid, evaluationId], - async () => { - const [data, error] = await computeEvaluationResultsMeanValueAction({ - projectId: project.id, - commitUuid, - documentUuid, - evaluationId, - }) + const fetcher = useCallback(async () => { + const [data, error] = await computeEvaluationResultsMeanValueAction({ + projectId: project.id, + commitUuid, + documentUuid, + evaluationId, + }) - if (error) null + if (error) { + toast({ + title: 'Error fetching mean value', + description: error.formErrors?.[0] || error.message, + variant: 'destructive', + }) + return null + } - return data - }, - opts, + return data + }, [commitUuid, documentUuid, evaluationId, project.id]) + const { data, isLoading, error, mutate } = useSWR( + ['evaluationResultsMeanQuery', commitUuid, documentUuid, evaluationId], + fetcher, + { fallbackData }, ) return { diff --git a/apps/web/src/stores/evaluationResultCharts/evaluationResultsModalValue.ts b/apps/web/src/stores/evaluationResultCharts/evaluationResultsModalValue.ts index 1b51b168f..9ed184334 100644 --- a/apps/web/src/stores/evaluationResultCharts/evaluationResultsModalValue.ts +++ b/apps/web/src/stores/evaluationResultCharts/evaluationResultsModalValue.ts @@ -1,4 +1,6 @@ -import { useCurrentProject } from '@latitude-data/web-ui' +import { useCallback } from 'react' + +import { useCurrentProject, useToast } from '@latitude-data/web-ui' import { computeEvaluationResultsModalValueAction } from '$/actions/evaluationResults/computeEvaluationResultsModalValueAction' import useSWR, { SWRConfiguration } from 'swr' @@ -12,24 +14,33 @@ export default function useEvaluationResultsModalValue( documentUuid: string evaluationId: number }, - opts: SWRConfiguration = {}, + { fallbackData }: SWRConfiguration = {}, ) { const { project } = useCurrentProject() - const { data, isLoading, error, mutate } = useSWR( - ['evaluationResultsModalQuery', commitUuid, documentUuid, evaluationId], - async () => { - const [data, error] = await computeEvaluationResultsModalValueAction({ - projectId: project.id, - commitUuid, - documentUuid, - evaluationId, - }) + const { toast } = useToast() + const fetcher = useCallback(async () => { + const [data, error] = await computeEvaluationResultsModalValueAction({ + projectId: project.id, + commitUuid, + documentUuid, + evaluationId, + }) - if (error) null + if (error) { + toast({ + title: 'Error fetching evaluation modal value', + description: error.formErrors?.[0] || error.message, + variant: 'destructive', + }) + return null + } - return data - }, - opts, + return data + }, [commitUuid, documentUuid, evaluationId, project.id]) + const { data, isLoading, error, mutate } = useSWR( + ['evaluationResultsModalQuery', commitUuid, documentUuid, evaluationId], + fetcher, + { fallbackData }, ) return { diff --git a/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore.ts b/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore.ts index 8236ed2c8..8192ba890 100644 --- a/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore.ts +++ b/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore.ts @@ -1,29 +1,28 @@ +import { useCallback } from 'react' + import { Evaluation } from '@latitude-data/core/browser' import { computeAverageResultAndCostOverCommitAction } from '$/actions/evaluationResults/computeAggregatedResults' -import useSWR, { SWRConfiguration } from 'swr' +import useSWR from 'swr' -export default function useAverageResultsAndCostOverCommit( - { - evaluation, - documentUuid, - }: { - evaluation: Evaluation - documentUuid: string - }, - opts: SWRConfiguration = {}, -) { - const { data, isValidating, isLoading, error } = useSWR( - ['averageResultAndCostOverCommit', evaluation.id, documentUuid], - async () => { - const [data, error] = await computeAverageResultAndCostOverCommitAction({ - documentUuid, - evaluationId: evaluation.id, - }) +export default function useAverageResultsAndCostOverCommit({ + evaluation, + documentUuid, +}: { + evaluation: Evaluation + documentUuid: string +}) { + const fetcher = useCallback(async () => { + const [data, error] = await computeAverageResultAndCostOverCommitAction({ + documentUuid, + evaluationId: evaluation.id, + }) - if (error) return [] - return data - }, - opts, + if (error) return [] + return data + }, [documentUuid, evaluation.id]) + const { data, isValidating, isLoading, error, mutate } = useSWR( + ['averageResultAndCostOverCommit', evaluation.id, documentUuid], + fetcher, ) return { @@ -31,5 +30,6 @@ export default function useAverageResultsAndCostOverCommit( isLoading, isValidating, error, + refetch: mutate, } } diff --git a/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore.ts b/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore.ts index 70d252cf4..9a1d656cf 100644 --- a/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore.ts +++ b/apps/web/src/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore.ts @@ -1,29 +1,28 @@ +import { useCallback } from 'react' + import { Evaluation } from '@latitude-data/core/browser' import { computeAverageResultOverTimeAction } from '$/actions/evaluationResults/computeAggregatedResults' -import useSWR, { SWRConfiguration } from 'swr' +import useSWR from 'swr' -export default function useAverageResultOverTime( - { - evaluation, - documentUuid, - }: { - evaluation: Evaluation - documentUuid: string - }, - opts: SWRConfiguration = {}, -) { - const { data, isValidating, isLoading, error } = useSWR( - ['averageResultOverTime', evaluation.id, documentUuid], - async () => { - const [data, error] = await computeAverageResultOverTimeAction({ - documentUuid, - evaluationId: evaluation.id, - }) +export default function useAverageResultOverTime({ + evaluation, + documentUuid, +}: { + evaluation: Evaluation + documentUuid: string +}) { + const fetcher = useCallback(async () => { + const [data, error] = await computeAverageResultOverTimeAction({ + documentUuid, + evaluationId: evaluation.id, + }) - if (error) return [] - return data - }, - opts, + if (error) return [] + return data + }, [documentUuid, evaluation.id]) + const { data, isValidating, isLoading, error, mutate } = useSWR( + ['averageResultOverTime', evaluation.id, documentUuid], + fetcher, ) return { @@ -31,5 +30,6 @@ export default function useAverageResultOverTime( isLoading, isValidating, error, + refetch: mutate, } } diff --git a/apps/web/src/stores/evaluationResultsWithMetadata.ts b/apps/web/src/stores/evaluationResultsWithMetadata.ts index 760099fe5..7381564d6 100644 --- a/apps/web/src/stores/evaluationResultsWithMetadata.ts +++ b/apps/web/src/stores/evaluationResultsWithMetadata.ts @@ -1,6 +1,5 @@ -import { useMemo } from 'react' +import { useCallback } from 'react' -import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories' import { useToast } from '@latitude-data/web-ui' import { computeEvaluationResultsWithMetadataAction } from '$/actions/evaluations/computeEvaluationResultsWithMetadata' import useSWR, { SWRConfiguration } from 'swr' @@ -18,34 +17,33 @@ export default function useEvaluationResultsWithMetadata( commitUuid: string projectId: number }, - opts: SWRConfiguration, + { fallbackData }: SWRConfiguration = {}, ) { const { toast } = useToast() - const { data = EMPTY_ARRAY, ...rest } = useSWR< - EvaluationResultWithMetadata[] - >( - ['evaluationResults', evaluationId, documentUuid, commitUuid, projectId], - async () => { - const [data, error] = await computeEvaluationResultsWithMetadataAction({ - evaluationId, - documentUuid, - commitUuid, - projectId, - }) + const fetcher = useCallback(async () => { + const [data, error] = await computeEvaluationResultsWithMetadataAction({ + evaluationId, + documentUuid, + commitUuid, + projectId, + }) - if (error) { - toast({ - title: 'Error fetching evaluations', - description: error.formErrors?.[0] || error.message, - variant: 'destructive', - }) - throw error - } + if (error) { + toast({ + title: 'Error fetching evaluations', + description: error.formErrors?.[0] || error.message, + variant: 'destructive', + }) + throw error + } - return data - }, - opts, + return data + }, [commitUuid, documentUuid, evaluationId, projectId, toast]) + const { data = EMPTY_ARRAY, mutate } = useSWR( + ['evaluationResults', evaluationId, documentUuid, commitUuid, projectId], + fetcher, + { fallbackData }, ) - return useMemo(() => ({ data, ...rest }), [data, rest]) + return { data, mutate } } diff --git a/apps/web/src/stores/projects.ts b/apps/web/src/stores/projects.ts index dd75e351b..b3eccdaf6 100644 --- a/apps/web/src/stores/projects.ts +++ b/apps/web/src/stores/projects.ts @@ -1,3 +1,5 @@ +import { useCallback } from 'react' + import { Project } from '@latitude-data/core/browser' import { useToast } from '@latitude-data/web-ui' import { createProjectAction } from '$/actions/projects/create' @@ -5,11 +7,11 @@ import { destroyProjectAction } from '$/actions/projects/destroy' import { fetchProjectsAction } from '$/actions/projects/fetch' import { updateProjectAction } from '$/actions/projects/update' import useLatitudeAction from '$/hooks/useLatitudeAction' -import useSWR, { SWRConfiguration } from 'swr' +import useSWR from 'swr' -export default function useProjects(opts?: SWRConfiguration) { +export default function useProjects() { const { toast } = useToast() - const fetcher = async () => { + const fetcher = useCallback(async () => { const [data, error] = await fetchProjectsAction() if (error) { toast({ @@ -21,13 +23,13 @@ export default function useProjects(opts?: SWRConfiguration) { if (!data) return [] return data - } + }, []) const { data = [], mutate, ...rest - } = useSWR('api/projects', fetcher, opts) + } = useSWR('api/projects', fetcher) const { execute: create } = useLatitudeAction(createProjectAction, { onSuccess: ({ data: project }) => { toast({ diff --git a/apps/websockets/src/server.ts b/apps/websockets/src/server.ts index c31b326dd..1875d4855 100644 --- a/apps/websockets/src/server.ts +++ b/apps/websockets/src/server.ts @@ -47,9 +47,7 @@ const io = new Server(server, { }) io.on('connection', (socket: Socket) => { - console.log( - 'Main namespace is not enabled. Connect to /web or /workers instead.', - ) + // Main namespace is not enabled. Connect to /web or /workers instead. socket.disconnect() }) @@ -111,14 +109,12 @@ workers.use(async (socket, next) => { try { const token = socket.handshake.auth?.token if (!token) { - console.log('DEBUG: No token provided') return next(new Error('Authentication error: No token provided')) } const result = await verifyWorkerWebsocketToken(token) if (result.error) { - console.log('DEBUG: JWT verification failed for worker:', result.error) return next(new Error(`Authentication error: ${result.error.message}`)) } @@ -130,21 +126,15 @@ workers.use(async (socket, next) => { }) workers.on('connection', (socket) => { - console.log('DEBUG: Worker connected') - - socket.on('pingFromWorkers', () => { - console.log('DEBUG: Ping from workers') - }) - socket.on('evaluationStatus', (args) => { - console.log('DEBUG: Evaluation STATUS %s', JSON.stringify(args)) const { workspaceId, data } = args const workspace = buildWorkspaceRoom({ workspaceId }) web.to(workspace).emit('evaluationStatus', data) }) - - socket.on('disconnect', () => { - console.log('Worker disconnected') + socket.on('evaluationResultCreated', (args) => { + const { workspaceId, data } = args + const workspace = buildWorkspaceRoom({ workspaceId }) + web.to(workspace).emit('evaluationResultCreated', data) }) }) diff --git a/apps/workers/src/server.ts b/apps/workers/src/server.ts index d0217a865..2f7606158 100644 --- a/apps/workers/src/server.ts +++ b/apps/workers/src/server.ts @@ -1,7 +1,5 @@ import http from 'http' -import { WebsocketClient } from '@latitude-data/core/websockets/workers' - import { captureException, captureMessage } from './utils/sentry' import startWorkers from './workers' @@ -11,12 +9,7 @@ console.log('Workers started') const port = process.env.WORKERS_PORT || 3002 const server = http.createServer(async (req, res) => { - const websockets = await WebsocketClient.getSocket() - - if (req.url === '/ping' && req.method === 'GET') { - websockets.emit('pingFromWorkers') - res.end(JSON.stringify({ status: 'OK', message: 'Pong' })) - } else if (req.url === '/health' && req.method === 'GET') { + if (req.url === '/health' && req.method === 'GET') { res.writeHead(200, { 'Content-Type': 'application/json' }) res.end(JSON.stringify({ status: 'OK', message: 'Workers are healthy' })) } else { diff --git a/packages/core/src/events/handlers/createEvaluationResultJob.test.ts b/packages/core/src/events/handlers/createEvaluationResultJob.test.ts new file mode 100644 index 000000000..4ec63f0b8 --- /dev/null +++ b/packages/core/src/events/handlers/createEvaluationResultJob.test.ts @@ -0,0 +1,169 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { EvaluationRunEvent } from '.' +import { + Commit, + DocumentLog, + DocumentVersion, + Project, + ProviderApiKey, + ProviderLog, + User, + Workspace, +} from '../../browser' +import { EvaluationResultableType, Providers } from '../../constants' +import { NotFoundError } from '../../lib' +import { mergeCommit } from '../../services/commits' +import * as factories from '../../tests/factories' + +const mocks = vi.hoisted(() => { + const mockEmit = vi.fn() + return { + getSocket: vi.fn().mockResolvedValue({ emit: mockEmit }), + mockEmit, + } +}) + +vi.mock('../../websockets/workers', async (importOriginal) => { + const mod = + (await importOriginal()) as typeof import('../../websockets/workers') + return { + ...mod, + WebsocketClient: { + getSocket: mocks.getSocket, + }, + } +}) + +let event: EvaluationRunEvent +let workspace: Workspace +let user: User +let project: Project +let provider: ProviderApiKey +let evaluation: Awaited> +let commit: Commit +let documentVersion: DocumentVersion +let documentLog: DocumentLog +let providerLog: ProviderLog + +describe('createEvaluationResultJob', () => { + beforeEach(async () => { + vi.resetModules() + + const basic = await factories.createProject() + workspace = basic.workspace + user = basic.user + project = basic.project + provider = basic.providers[0]! + const { commit: draft } = await factories.createDraft({ project, user }) + const doc = await factories.createDocumentVersion({ + commit: draft, + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ provider }), + }) + documentVersion = doc.documentVersion + commit = await mergeCommit(draft).then((r) => r.unwrap()) + + evaluation = await factories.createLlmAsJudgeEvaluation({ + workspace, + prompt: factories.helpers.createPrompt({ provider }), + configuration: { + type: EvaluationResultableType.Number, + detail: { + range: { from: 0, to: 100 }, + }, + }, + }) + const { documentLog: log } = await factories.createDocumentLog({ + document: documentVersion, + commit, + }) + documentLog = log + + providerLog = await factories.createProviderLog({ + documentLogUuid: documentLog.uuid, + providerId: provider.id, + providerType: Providers.OpenAI, + }) + event = { + type: 'evaluationRun', + data: { + evaluationId: evaluation.id, + documentUuid: documentVersion.documentUuid, + documentLogUuid: documentLog.uuid, + providerLogUuid: providerLog.uuid, + response: { + object: { result: 33 }, + }, + }, + } as unknown as EvaluationRunEvent + }) + + it('send websocket event for "evaluationResultCreated"', async () => { + const mod = await import('./createEvaluationResultJob') + const { createEvaluationResultJob } = mod + await createEvaluationResultJob({ data: event }) + + expect(mocks.mockEmit).toHaveBeenCalledWith('evaluationResultCreated', { + workspaceId: workspace.id, + data: { + documentUuid: documentVersion.documentUuid, + workspaceId: workspace.id, + evaluationId: evaluation.id, + evaluationResultId: expect.any(Number), + row: { + commit, + id: expect.any(Number), + evaluationId: evaluation.id, + documentLogId: documentLog.id, + providerLogId: providerLog.id, + resultableType: EvaluationResultableType.Number, + resultableId: expect.any(Number), + createdAt: expect.any(Date), + updatedAt: expect.any(Date), + tokens: expect.any(Number), + costInMillicents: expect.any(Number), + result: '33', + }, + }, + }) + }) + + it('should throw NotFoundError when evaluation is not found', async () => { + const mod = await import('./createEvaluationResultJob') + const { createEvaluationResultJob } = mod + const data = { + ...event, + data: { ...event.data, evaluationId: 999 }, + } + await expect(createEvaluationResultJob({ data })).rejects.toThrow( + new NotFoundError('Evaluation not found'), + ) + }) + + it('should throw NotFoundError when documentLogs is not found', async () => { + const mod = await import('./createEvaluationResultJob') + const { createEvaluationResultJob } = mod + const fakeUuid = '00000000-0000-0000-0000-000000000000' + const data = { + ...event, + data: { ...event.data, documentLogUuid: fakeUuid }, + } + await expect(createEvaluationResultJob({ data })).rejects.toThrow( + new NotFoundError('Document log not found'), + ) + }) + + it('should throw NotFoundError when providerLog is not found', async () => { + const mod = await import('./createEvaluationResultJob') + const { createEvaluationResultJob } = mod + const fakeUuid = '00000000-0000-0000-0000-000000000000' + const data = { + ...event, + data: { ...event.data, providerLogUuid: fakeUuid }, + } + await expect(createEvaluationResultJob({ data })).rejects.toThrow( + new NotFoundError('Provider log not found'), + ) + }) +}) diff --git a/packages/core/src/events/handlers/createEvaluationResultJob.ts b/packages/core/src/events/handlers/createEvaluationResultJob.ts index 7126e4ccc..c209edf41 100644 --- a/packages/core/src/events/handlers/createEvaluationResultJob.ts +++ b/packages/core/src/events/handlers/createEvaluationResultJob.ts @@ -1,4 +1,7 @@ +import { eq } from 'drizzle-orm' + import { EvaluationRunEvent } from '.' +import { database } from '../../client' import { ChainObjectResponse } from '../../constants' import { unsafelyFindDocumentLogByUuid, @@ -7,12 +10,15 @@ import { } from '../../data-access' import { NotFoundError } from '../../lib' import { createEvaluationResult } from '../../services/evaluationResults' +import { createEvaluationResultQuery } from '../../services/evaluationResults/_createEvaluationResultQuery' +import { WebsocketClient } from '../../websockets/workers' export const createEvaluationResultJob = async ({ data: event, }: { data: EvaluationRunEvent }) => { + const websockets = await WebsocketClient.getSocket() const { evaluationId, documentLogUuid, providerLogUuid, response } = event.data @@ -25,10 +31,32 @@ export const createEvaluationResultJob = async ({ const providerLog = await unsafelyFindProviderLogByUuid(providerLogUuid) if (!providerLog) throw new NotFoundError('Provider log not found') - await createEvaluationResult({ + const evaluationResult = await createEvaluationResult({ evaluation, documentLog, providerLog, result: (response as ChainObjectResponse).object, }).then((r) => r.unwrap()) + evaluationResult.id + + const { evaluationResultsScope, baseQuery } = createEvaluationResultQuery( + evaluation.workspaceId, + database, + ) + const result = await baseQuery + .where(eq(evaluationResultsScope.id, evaluationResult.id)) + .limit(1) + + const evaluationResultWithMetadata = result[0]! + + websockets.emit('evaluationResultCreated', { + workspaceId: evaluation.workspaceId, + data: { + documentUuid: event.data.documentUuid, + workspaceId: evaluation.workspaceId, + evaluationId: evaluation.id, + evaluationResultId: evaluationResult.id, + row: evaluationResultWithMetadata, + }, + }) } diff --git a/packages/core/src/events/handlers/index.ts b/packages/core/src/events/handlers/index.ts index 781c38870..760796a69 100644 --- a/packages/core/src/events/handlers/index.ts +++ b/packages/core/src/events/handlers/index.ts @@ -39,6 +39,7 @@ export type MembershipCreatedEvent = LatitudeEventGeneric< export type EvaluationRunEvent = LatitudeEventGeneric< 'evaluationRun', { + documentUuid: string evaluationId: number documentLogUuid: string providerLogUuid: string diff --git a/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts b/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts index ce2374290..28a6dc431 100644 --- a/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts +++ b/packages/core/src/services/evaluationResults/aggregations/countersQuery.test.ts @@ -193,7 +193,7 @@ describe('evaluation results aggregations', () => { expect(result.mostCommon).toBe('apple') // Funcky test because of floating point math issues // in Mac OS and CI Linux - expect(result.percentage).toBeGreaterThan(65) + expect(result.percentage).toBeGreaterThan(64) expect(result.percentage).toBeLessThan(67) }) }) diff --git a/packages/core/src/services/evaluations/run.ts b/packages/core/src/services/evaluations/run.ts index e454daea3..5c73f1ad4 100644 --- a/packages/core/src/services/evaluations/run.ts +++ b/packages/core/src/services/evaluations/run.ts @@ -36,9 +36,11 @@ const getResultSchema = (type: EvaluationResultableType): JSONSchema7 => { export const runEvaluation = async ( { documentLog, + documentUuid, evaluation, }: { documentLog: DocumentLog + documentUuid: string evaluation: EvaluationDto }, db = database, @@ -105,12 +107,14 @@ export const runEvaluation = async ( output: 'object', }, }) + if (chainResult.error) return chainResult chainResult.value.response.then((response) => { publisher.publish({ type: 'evaluationRun', data: { + documentUuid, evaluationId: evaluation.id, documentLogUuid: documentLog.uuid, providerLogUuid: response.providerLog.uuid, diff --git a/packages/core/src/websockets/constants.ts b/packages/core/src/websockets/constants.ts index c4d69e32b..c739ed3c0 100644 --- a/packages/core/src/websockets/constants.ts +++ b/packages/core/src/websockets/constants.ts @@ -3,6 +3,8 @@ // All this can be seen in the browser. If you want something private // put in other place. +import { type EvaluationResultWithMetadata } from '../repositories' + const ONE_HOUR = 60 * 60 * 1000 const SEVEN_DAYS = 7 * 24 * ONE_HOUR @@ -35,8 +37,18 @@ type EvaluationStatusArgs = { errors: number enqueued: number } + +type evaluationResultCreatedArgs = { + workspaceId: number + evaluationId: number + documentUuid: string + evaluationResultId: number + row: EvaluationResultWithMetadata +} + export type WebServerToClientEvents = { evaluationStatus: (args: EvaluationStatusArgs) => void + evaluationResultCreated: (args: evaluationResultCreatedArgs) => void joinWorkspace: (args: { workspaceId: number; userId: string }) => void } export type WebClientToServerEvents = { @@ -45,8 +57,11 @@ export type WebClientToServerEvents = { export type WorkersClientToServerEvents = { evaluationStatus: (args: { + workspaceId: number data: EvaluationStatusArgs + }) => void + evaluationResultCreated: (args: { workspaceId: number + data: evaluationResultCreatedArgs }) => void - pingFromWorkers: () => void } diff --git a/packages/core/src/websockets/workers.ts b/packages/core/src/websockets/workers.ts index 09eb95658..22b51a07b 100644 --- a/packages/core/src/websockets/workers.ts +++ b/packages/core/src/websockets/workers.ts @@ -23,15 +23,6 @@ export class WebsocketClient { WebsocketClient.instance = instance return new Promise((resolve) => { websockets.on('connect', () => { - console.log('Workers connected to WebSocket server') - resolve(websockets) - }) - - websockets.on('connect_error', (error) => { - console.error( - 'Error connecting to WebSocket server from WORKERS:', - error, - ) resolve(websockets) }) }) diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts index a37610275..3cdd5948c 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runDocumentJob.test.ts @@ -9,28 +9,43 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { ProgressTracker } from '../../utils/progressTracker' import { runDocumentJob } from './runDocumentJob' -const mocks = vi.hoisted(() => ({ - queues: { - defaultQueue: { - jobs: { - enqueueRunEvaluationJob: vi.fn(), +const mocks = vi.hoisted(() => { + const mockEmit = vi.fn() + return { + mockEmit, + getSocketMock: vi.fn().mockResolvedValue({ emit: mockEmit }), + queues: { + defaultQueue: { + jobs: { + enqueueRunEvaluationJob: vi.fn(), + }, }, - }, - eventsQueue: { - jobs: { - enqueueCreateEventJob: vi.fn(), - enqueuePublishEventJob: vi.fn(), + eventsQueue: { + jobs: { + enqueueCreateEventJob: vi.fn(), + enqueuePublishEventJob: vi.fn(), + }, }, }, + } +}) + +vi.mock('@latitude-data/core/websockets/workers', () => ({ + WebsocketClient: { + getSocket: mocks.getSocketMock, }, })) -// Mock dependencies vi.mock('../../', () => ({ setupJobs: vi.fn().mockImplementation(() => mocks.queues), })) vi.mock('@latitude-data/core/redis') +vi.mock('@latitude-data/core/queues', () => { + return { + queues: vi.fn().mockResolvedValue({}), + } +}) vi.mock('@latitude-data/core/services/commits/runDocumentAtCommit') vi.mock('@latitude-data/env') vi.mock('../../utils/progressTracker') @@ -117,6 +132,15 @@ describe('runDocumentJob', () => { await runDocumentJob(mockJob) + expect(mocks.mockEmit).toHaveBeenCalledWith('evaluationStatus', { + workspaceId: workspace.id, + data: { + batchId: 'batch1', + evaluationId: evaluation.id, + documentUuid: document.documentUuid, + }, + }) + expect(runDocumentAtCommit).toHaveBeenCalled() expect( mocks.queues.defaultQueue.jobs.enqueueRunEvaluationJob, @@ -136,5 +160,13 @@ describe('runDocumentJob', () => { expect(consoleSpy).toHaveBeenCalledWith(testError) expect(ProgressTracker.prototype.incrementErrors).toHaveBeenCalled() + expect(mocks.mockEmit).toHaveBeenCalledWith('evaluationStatus', { + workspaceId: workspace.id, + data: { + batchId: 'batch1', + evaluationId: evaluation.id, + documentUuid: document.documentUuid, + }, + }) }) }) diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.test.ts b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.test.ts new file mode 100644 index 000000000..f7edd284b --- /dev/null +++ b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.test.ts @@ -0,0 +1,152 @@ +import { Job } from 'bullmq' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { runEvaluationJob, type RunEvaluationJobData } from './runEvaluationJob' + +vi.spyOn(console, 'error').mockImplementation(() => undefined) + +vi.mock('@latitude-data/core/queues', () => ({ + queues: vi.fn(() => Promise.resolve({})), // mock queues function +})) + +vi.mock('@latitude-data/core/repositories', () => ({ + DocumentLogsRepository: vi.fn().mockImplementation(() => ({ + findByUuid: vi.fn(() => + Promise.resolve({ unwrap: () => Promise.resolve({}) }), + ), + })), + EvaluationsRepository: vi.fn().mockImplementation(() => ({ + find: vi.fn(() => Promise.resolve({ unwrap: () => Promise.resolve({}) })), + })), +})) + +const runEvaluationMock = vi.hoisted(() => + vi.fn(() => + Promise.resolve({ + unwrap: () => + Promise.resolve({ + response: new Promise((resolve) => { + resolve(null) + }), + }), + }), + ), +) +vi.mock('@latitude-data/core/services/evaluations/run', () => ({ + runEvaluation: runEvaluationMock, +})) + +const mockEmit = vi.hoisted(() => vi.fn()) +const mockGetSocket = vi.hoisted(() => + vi.fn().mockResolvedValue({ emit: mockEmit }), +) +vi.mock('@latitude-data/core/websockets/workers', () => ({ + WebsocketClient: { + getSocket: mockGetSocket, + }, +})) + +vi.mock('@latitude-data/env', () => ({ + env: { NODE_ENV: 'test' }, // Mock environment to be 'test' +})) + +const incrementCompletedMock = vi.hoisted(() => vi.fn()) +const incrementErrorsMock = vi.hoisted(() => vi.fn()) +vi.mock('../../utils/progressTracker', () => { + return { + ProgressTracker: vi.fn().mockImplementation(() => ({ + incrementCompleted: incrementCompletedMock, + incrementErrors: incrementErrorsMock, + getProgress: vi.fn(() => Promise.resolve({ completed: 1, total: 1 })), + })), + } +}) + +let jobData: Job +describe('runEvaluationJob', () => { + beforeEach(() => { + mockGetSocket.mockClear() + jobData = { + id: '1', + data: { + workspaceId: 1, + documentUuid: 'doc-uuid', + documentLogUuid: 'log-uuid', + evaluationId: 2, + batchId: 'batch-123', + skipProgress: false, + }, + } as Job + + runEvaluationMock.mockClear() + }) + + it('calls runEvaluation', async () => { + await runEvaluationJob(jobData) + expect(runEvaluationMock).toHaveBeenCalledWith({ + documentLog: expect.any(Object), + evaluation: expect.any(Object), + documentUuid: 'doc-uuid', + }) + incrementCompletedMock.mockClear() + }) + + it('wait for runEvaluation response to finish', async () => { + const responsePromise = new Promise((resolve) => + setTimeout(() => resolve('expected response'), 100), + ) + const mockedResponse = { + unwrap: () => + Promise.resolve({ + response: responsePromise, + }), + } + + runEvaluationMock.mockResolvedValue(mockedResponse) + + const awaitResponseSpy = vi.fn() + responsePromise.then(awaitResponseSpy) + + await runEvaluationJob(jobData) + expect(awaitResponseSpy).toHaveBeenCalled() + incrementCompletedMock.mockClear() + }) + + it('increment succesfull counter', async () => { + await runEvaluationJob(jobData) + expect(mockGetSocket).toHaveBeenCalledTimes(1) + expect(incrementCompletedMock).toHaveBeenCalledTimes(1) + expect(mockEmit).toHaveBeenCalledWith('evaluationStatus', { + workspaceId: 1, + data: { + batchId: 'batch-123', + evaluationId: 2, + documentUuid: 'doc-uuid', + completed: 1, + total: 1, + }, + }) + incrementCompletedMock.mockClear() + }) + + it('increment error counter', async () => { + runEvaluationMock.mockImplementationOnce(() => { + throw new Error('Some error occurred') + }) + + await runEvaluationJob(jobData) + + expect(mockGetSocket).toHaveBeenCalledTimes(1) + expect(incrementErrorsMock).toHaveBeenCalledTimes(1) + expect(mockEmit).toHaveBeenCalledWith('evaluationStatus', { + workspaceId: 1, + data: { + batchId: 'batch-123', + evaluationId: 2, + documentUuid: 'doc-uuid', + completed: 1, + total: 1, + }, + }) + }) +}) diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts index 807297a0a..75109c0fa 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts @@ -10,7 +10,7 @@ import { Job } from 'bullmq' import { ProgressTracker } from '../../utils/progressTracker' -type RunEvaluationJobData = { +export type RunEvaluationJobData = { workspaceId: number documentUuid: string documentLogUuid: string @@ -35,11 +35,16 @@ export const runEvaluationJob = async (job: Job) => { .find(evaluationId) .then((r) => r.unwrap()) - await runEvaluation({ + const { response } = await runEvaluation({ documentLog, evaluation, + documentUuid, }).then((r) => r.unwrap()) + // Waiting for the reponse is important. It guarantees that the evaluation + // has been created before we notify the client via websockets. + await response + await progressTracker.incrementCompleted() } catch (error) { if (env.NODE_ENV !== 'production') { diff --git a/packages/jobs/src/utils/progressTracker.ts b/packages/jobs/src/utils/progressTracker.ts index ee0b193b4..d6276352e 100644 --- a/packages/jobs/src/utils/progressTracker.ts +++ b/packages/jobs/src/utils/progressTracker.ts @@ -25,18 +25,10 @@ export class ProgressTracker { await this.redis.incr(this.getKey('errors')) } - async decrementTotal() { - await this.redis.decr(this.getKey('total')) - } - async incrementEnqueued() { await this.redis.incr(this.getKey('enqueued')) } - async decrementEnqueued() { - await this.redis.decr(this.getKey('enqueued')) - } - async getProgress() { const [total, completed, errors, enqueued] = await this.redis.mget([ this.getKey('total'), diff --git a/packages/web-ui/tailwind.config.js b/packages/web-ui/tailwind.config.js index 6370236b8..8a1704f50 100644 --- a/packages/web-ui/tailwind.config.js +++ b/packages/web-ui/tailwind.config.js @@ -83,10 +83,15 @@ module.exports = { from: { height: 'var(--radix-accordion-content-height)' }, to: { height: '0' }, }, + 'flash-background': { + '0%': { backgroundColor: 'transparent' }, + '100%': { backgroundColor: 'rgb(var(--accent))' }, + }, }, animation: { 'accordion-down': 'accordion-down 0.2s ease-out', 'accordion-up': 'accordion-up 0.2s ease-out', + 'flash': 'flash-background 1s ease-in-out', }, maxWidth: { modal: '580px',