Skip to content

Commit

Permalink
When a evaluation is run show results in the frontend whenever they're
Browse files Browse the repository at this point in the history
processed
  • Loading branch information
andresgutgon committed Sep 25, 2024
1 parent 07767df commit d5e5428
Show file tree
Hide file tree
Showing 34 changed files with 956 additions and 358 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
'use client'

import {
Commit,
EvaluationAggregationTotals,
EvaluationDto,
EvaluationMeanValue,
EvaluationModalValue,
} from '@latitude-data/core/browser'
import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'

import { EvaluationResults } from '../EvaluationResults'
import { MetricsSummary } from '../MetricsSummary'
import { useEvaluationStatus } from './useEvaluationStatus'

export default function Content<T extends boolean>({
commit,
evaluation,
evaluationResults,
documentUuid,
aggregationTotals,
isNumeric,
meanOrModal,
}: {
commit: Commit
evaluation: EvaluationDto
documentUuid: string
evaluationResults: EvaluationResultWithMetadata[]
aggregationTotals: EvaluationAggregationTotals
isNumeric: T
meanOrModal: T extends true ? EvaluationMeanValue : EvaluationModalValue
}) {
const { jobs } = useEvaluationStatus({ evaluation })
return (
<>
<MetricsSummary
commit={commit}
evaluation={evaluation}
documentUuid={documentUuid}
aggregationTotals={aggregationTotals}
isNumeric={isNumeric}
meanOrModal={meanOrModal}
/>
<EvaluationResults
evaluation={evaluation}
evaluationResults={evaluationResults}
jobs={jobs}
/>
</>
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { useCallback, useEffect, useRef, useState } from 'react'

import { Evaluation } from '@latitude-data/core/browser'
import { useCurrentDocument } from '@latitude-data/web-ui'
import {
useSockets,
type EventArgs,
} from '$/components/Providers/WebsocketsProvider/useSockets'

import { isEvaluationRunDone } from '../../_lib/isEvaluationRunDone'
import { useRefetchStats } from './useRefetchStats'

const DISAPERING_IN_MS = 5000

export function useEvaluationStatus({
evaluation,
}: {
evaluation: Evaluation
}) {
const timeoutRef = useRef<number | null>(null)
const [jobs, setJobs] = useState<EventArgs<'evaluationStatus'>[]>([])
const document = useCurrentDocument()
const { refetchStats } = useRefetchStats({ evaluation, document })
const onMessage = useCallback(
(args: EventArgs<'evaluationStatus'>) => {
if (evaluation.id !== args.evaluationId) return
if (document.documentUuid !== args.documentUuid) return

const done = isEvaluationRunDone(args)

if (done) {
refetchStats()
}

setJobs((prevJobs) => {
const jobIndex = prevJobs.findIndex(
(job) => job.batchId === args.batchId,
)

if (jobIndex === -1) {
return [...prevJobs, args]
} else {
const newJobs = [...prevJobs]
newJobs[jobIndex] = args

if (done) {
setTimeout(() => {
setJobs((currentJobs) =>
currentJobs.filter((job) => job.batchId !== args.batchId),
)
}, DISAPERING_IN_MS)
}

return newJobs
}
})
},
[evaluation.id, document.documentUuid, refetchStats],
)

useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current)
}
}
}, [])

useSockets({ event: 'evaluationStatus', onMessage })

return { jobs }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { useCallback } from 'react'

import {
DocumentVersion,
Evaluation,
EvaluationResultableType,
} from '@latitude-data/core/browser'
import { useCurrentCommit } from '@latitude-data/web-ui'
import useEvaluationResultsCounters from '$/stores/evaluationResultCharts/evaluationResultsCounters'
import useEvaluationResultsMeanValue from '$/stores/evaluationResultCharts/evaluationResultsMeanValue'
import useEvaluationResultsModalValue from '$/stores/evaluationResultCharts/evaluationResultsModalValue'
import useAverageResultsAndCostOverCommit from '$/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore'
import useAverageResultOverTime from '$/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore'

export function useRefetchStats({
evaluation,
document,
}: {
evaluation: Evaluation
document: DocumentVersion
}) {
const { commit } = useCurrentCommit()
const evaluationId = evaluation.id
const commitUuid = commit.uuid
const documentUuid = document.documentUuid
const isNumeric =
evaluation.configuration.type === EvaluationResultableType.Number

const { refetch: refetchAverageResulstAndCostsOverCommit } =
useAverageResultsAndCostOverCommit({
evaluation,
documentUuid,
})
const { refetch: refetchAverageResultOverTime } = useAverageResultOverTime({
evaluation,
documentUuid,
})
const { refetch: refetchMean } = useEvaluationResultsMeanValue({
commitUuid,
documentUuid,
evaluationId,
})
const { refetch: refetchModal } = useEvaluationResultsModalValue({
commitUuid,
documentUuid,
evaluationId,
})
const { refetch: refetchTotals } = useEvaluationResultsCounters({
commitUuid,
documentUuid,
evaluationId,
})

const refetchStats = useCallback(() => {
console.log('refetchStats')

Promise.all([
refetchTotals(),
...(isNumeric
? [
refetchMean(),
refetchAverageResulstAndCostsOverCommit(),
refetchAverageResultOverTime(),
]
: [refetchModal()]),
])
}, [
isNumeric,
refetchMean,
refetchModal,
refetchTotals,
refetchAverageResulstAndCostsOverCommit,
refetchAverageResultOverTime,
])

return {
refetchStats,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,18 @@ export const ResultCellContent = ({
return <Text.H4 noWrap>{value as string}</Text.H4>
}

type EvaluationResultRow = EvaluationResultWithMetadata & {
realtimeAdded?: boolean
}
export const EvaluationResultsTable = ({
evaluation,
evaluationResults,
selectedResult,
setSelectedResult,
}: {
evaluation: EvaluationDto
evaluationResults: EvaluationResultWithMetadata[]
selectedResult: EvaluationResultWithMetadata | undefined
evaluationResults: EvaluationResultRow[]
selectedResult: EvaluationResultRow | undefined
setSelectedResult: (log: EvaluationResultWithMetadata | undefined) => void
}) => {
return (
Expand All @@ -71,9 +74,9 @@ export const EvaluationResultsTable = ({
</TableRow>
</TableHeader>
<TableBody className='max-h-full overflow-y-auto'>
{evaluationResults.map((evaluationResult, idx) => (
{evaluationResults.map((evaluationResult) => (
<TableRow
key={idx}
key={evaluationResult.id}
onClick={() =>
setSelectedResult(
selectedResult?.id === evaluationResult.id
Expand All @@ -85,12 +88,18 @@ export const EvaluationResultsTable = ({
'cursor-pointer border-b-[0.5px] h-12 max-h-12 border-border',
{
'bg-secondary': selectedResult?.id === evaluationResult.id,
'animate-flash': evaluationResult.realtimeAdded,
},
)}
>
<TableCell>
<Text.H4 noWrap>
{relativeTime(evaluationResult.createdAt)}
<time
dateTime={evaluationResult.createdAt.toISOString()}
suppressHydrationWarning
>
{relativeTime(evaluationResult.createdAt)}
</time>
</Text.H4>
</TableCell>
<TableCell>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,82 +1,31 @@
'use client'

import { useCallback, useEffect, useRef, useState } from 'react'
import { ProgressIndicator } from '@latitude-data/web-ui'
import { isEvaluationRunDone } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone'
import { type EventArgs } from '$/components/Providers/WebsocketsProvider/useSockets'

import { EvaluationDto } from '@latitude-data/core/browser'
import { ProgressIndicator, useCurrentDocument } from '@latitude-data/web-ui'
import {
useSockets,
type EventArgs,
} from '$/components/Providers/WebsocketsProvider/useSockets'

const DISAPERING_IN_MS = 5000
export function EvaluationStatusBanner({
evaluation,
jobs,
}: {
evaluation: EvaluationDto
jobs: EventArgs<'evaluationStatus'>[]
}) {
const timeoutRef = useRef<number | null>(null)
const [jobs, setJobs] = useState<EventArgs<'evaluationStatus'>[]>([])
const document = useCurrentDocument()

const onMessage = useCallback(
(args: EventArgs<'evaluationStatus'>) => {
if (evaluation.id !== args.evaluationId) return
if (document.documentUuid !== args.documentUuid) return

setJobs((prevJobs) => {
const jobIndex = prevJobs.findIndex(
(job) => job.batchId === args.batchId,
)

if (jobIndex === -1) {
return [...prevJobs, args]
} else {
const newJobs = [...prevJobs]
newJobs[jobIndex] = args

if (isDone(args)) {
setTimeout(() => {
setJobs((currentJobs) =>
currentJobs.filter((job) => job.batchId !== args.batchId),
)
}, DISAPERING_IN_MS)
}

return newJobs
}
})
},
[evaluation.id, document.documentUuid],
)

useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current)
}
}
}, [])

useSockets({ event: 'evaluationStatus', onMessage })

return (
<>
{jobs.map((job) => (
<div key={job.batchId} className='flex flex-col gap-4'>
{!isDone(job) && (
{!isEvaluationRunDone(job) && (
<ProgressIndicator state='running'>
{`Running batch evaluation ${job.completed}/${job.total}`}
</ProgressIndicator>
)}
{job.errors > 0 && !isDone(job) && (
{job.errors > 0 && (
<ProgressIndicator state='error'>
Some evaluations failed to run. We won't retry them automatically
to avoid increasing provider costs. Total errors:{' '}
<strong>{job.errors}</strong>
</ProgressIndicator>
)}
{isDone(job) && (
{isEvaluationRunDone(job) && (
<ProgressIndicator state='completed'>
Batch evaluation completed! Total evaluations:{' '}
<strong>{job.total}</strong> · Total errors:{' '}
Expand All @@ -89,7 +38,3 @@ export function EvaluationStatusBanner({
</>
)
}

function isDone(job: EventArgs<'evaluationStatus'>) {
return job.total === job.completed + job.errors
}
Loading

0 comments on commit d5e5428

Please sign in to comment.