Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Realtime evaluation result table update #248

Merged
merged 1 commit into from
Sep 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
'use client'

import {
Commit,
EvaluationAggregationTotals,
EvaluationDto,
EvaluationMeanValue,
EvaluationModalValue,
} from '@latitude-data/core/browser'
import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'

import { EvaluationResults } from '../EvaluationResults'
import { MetricsSummary } from '../MetricsSummary'
import { useEvaluationStatus } from './useEvaluationStatus'

export default function Content<T extends boolean>({
commit,
evaluation,
evaluationResults,
documentUuid,
aggregationTotals,
isNumeric,
meanOrModal,
}: {
commit: Commit
evaluation: EvaluationDto
documentUuid: string
evaluationResults: EvaluationResultWithMetadata[]
aggregationTotals: EvaluationAggregationTotals
isNumeric: T
meanOrModal: T extends true ? EvaluationMeanValue : EvaluationModalValue
}) {
const { jobs } = useEvaluationStatus({ evaluation })
return (
<>
<MetricsSummary
commit={commit}
evaluation={evaluation}
documentUuid={documentUuid}
aggregationTotals={aggregationTotals}
isNumeric={isNumeric}
meanOrModal={meanOrModal}
/>
<EvaluationResults
evaluation={evaluation}
evaluationResults={evaluationResults}
jobs={jobs}
/>
</>
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { useCallback, useEffect, useRef, useState } from 'react'

import { Evaluation } from '@latitude-data/core/browser'
import { useCurrentDocument } from '@latitude-data/web-ui'
import {
useSockets,
type EventArgs,
} from '$/components/Providers/WebsocketsProvider/useSockets'

import { isEvaluationRunDone } from '../../_lib/isEvaluationRunDone'
import { useRefetchStats } from './useRefetchStats'

const DISAPERING_IN_MS = 5000

export function useEvaluationStatus({
evaluation,
}: {
evaluation: Evaluation
}) {
const timeoutRef = useRef<number | null>(null)
const [jobs, setJobs] = useState<EventArgs<'evaluationStatus'>[]>([])
const document = useCurrentDocument()
const { refetchStats } = useRefetchStats({ evaluation, document })
const onMessage = useCallback(
(args: EventArgs<'evaluationStatus'>) => {
if (evaluation.id !== args.evaluationId) return
if (document.documentUuid !== args.documentUuid) return

const done = isEvaluationRunDone(args)

if (done) {
refetchStats()
}

setJobs((prevJobs) => {
const jobIndex = prevJobs.findIndex(
(job) => job.batchId === args.batchId,
)

if (jobIndex === -1) {
return [...prevJobs, args]
} else {
const newJobs = [...prevJobs]
newJobs[jobIndex] = args

if (done) {
setTimeout(() => {
setJobs((currentJobs) =>
currentJobs.filter((job) => job.batchId !== args.batchId),
)
}, DISAPERING_IN_MS)
}

return newJobs
}
})
},
[evaluation.id, document.documentUuid, refetchStats],
)

useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current)
}
}
}, [])

useSockets({ event: 'evaluationStatus', onMessage })

return { jobs }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { useCallback } from 'react'

import {
DocumentVersion,
Evaluation,
EvaluationResultableType,
} from '@latitude-data/core/browser'
import { useCurrentCommit } from '@latitude-data/web-ui'
import useEvaluationResultsCounters from '$/stores/evaluationResultCharts/evaluationResultsCounters'
import useEvaluationResultsMeanValue from '$/stores/evaluationResultCharts/evaluationResultsMeanValue'
import useEvaluationResultsModalValue from '$/stores/evaluationResultCharts/evaluationResultsModalValue'
import useAverageResultsAndCostOverCommit from '$/stores/evaluationResultCharts/numericalResults/averageResultAndCostOverCommitStore'
import useAverageResultOverTime from '$/stores/evaluationResultCharts/numericalResults/averageResultOverTimeStore'

export function useRefetchStats({
evaluation,
document,
}: {
evaluation: Evaluation
document: DocumentVersion
}) {
const { commit } = useCurrentCommit()
const evaluationId = evaluation.id
const commitUuid = commit.uuid
const documentUuid = document.documentUuid
const isNumeric =
evaluation.configuration.type === EvaluationResultableType.Number

const { refetch: refetchAverageResulstAndCostsOverCommit } =
useAverageResultsAndCostOverCommit({
evaluation,
documentUuid,
})
const { refetch: refetchAverageResultOverTime } = useAverageResultOverTime({
evaluation,
documentUuid,
})
const { refetch: refetchMean } = useEvaluationResultsMeanValue({
commitUuid,
documentUuid,
evaluationId,
})
const { refetch: refetchModal } = useEvaluationResultsModalValue({
commitUuid,
documentUuid,
evaluationId,
})
const { refetch: refetchTotals } = useEvaluationResultsCounters({
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this refetch actually the mutate function? if so i wouldn't rename it it's better to expose the real name so devs understand what they can do with this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I knew this will come out. I don't agree in this case to be honest. I think fetch express better the intention here

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this function just triggers a new request to update the data, I agree with @andresgutgon here ☝️

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but it doesn't just do that, you can also pass it data to mutate the store

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In general yes, but in this use case is a refetch. We're talking semantics here

commitUuid,
documentUuid,
evaluationId,
})

const refetchStats = useCallback(() => {
console.log('refetchStats')

Promise.all([
refetchTotals(),
...(isNumeric
? [
refetchMean(),
refetchAverageResulstAndCostsOverCommit(),
refetchAverageResultOverTime(),
]
: [refetchModal()]),
])
}, [
isNumeric,
refetchMean,
refetchModal,
refetchTotals,
refetchAverageResulstAndCostsOverCommit,
refetchAverageResultOverTime,
])

return {
refetchStats,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,18 @@ export const ResultCellContent = ({
return <Text.H4 noWrap>{value as string}</Text.H4>
}

type EvaluationResultRow = EvaluationResultWithMetadata & {
realtimeAdded?: boolean
}
export const EvaluationResultsTable = ({
evaluation,
evaluationResults,
selectedResult,
setSelectedResult,
}: {
evaluation: EvaluationDto
evaluationResults: EvaluationResultWithMetadata[]
selectedResult: EvaluationResultWithMetadata | undefined
evaluationResults: EvaluationResultRow[]
selectedResult: EvaluationResultRow | undefined
setSelectedResult: (log: EvaluationResultWithMetadata | undefined) => void
}) => {
return (
Expand All @@ -71,9 +74,9 @@ export const EvaluationResultsTable = ({
</TableRow>
</TableHeader>
<TableBody className='max-h-full overflow-y-auto'>
{evaluationResults.map((evaluationResult, idx) => (
{evaluationResults.map((evaluationResult) => (
<TableRow
key={idx}
key={evaluationResult.id}
onClick={() =>
setSelectedResult(
selectedResult?.id === evaluationResult.id
Expand All @@ -85,12 +88,18 @@ export const EvaluationResultsTable = ({
'cursor-pointer border-b-[0.5px] h-12 max-h-12 border-border',
{
'bg-secondary': selectedResult?.id === evaluationResult.id,
'animate-flash': evaluationResult.realtimeAdded,
},
)}
>
<TableCell>
<Text.H4 noWrap>
{relativeTime(evaluationResult.createdAt)}
<time
dateTime={evaluationResult.createdAt.toISOString()}
suppressHydrationWarning
>
{relativeTime(evaluationResult.createdAt)}
</time>
</Text.H4>
</TableCell>
<TableCell>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,82 +1,31 @@
'use client'

import { useCallback, useEffect, useRef, useState } from 'react'
import { ProgressIndicator } from '@latitude-data/web-ui'
import { isEvaluationRunDone } from '$/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_lib/isEvaluationRunDone'
import { type EventArgs } from '$/components/Providers/WebsocketsProvider/useSockets'

import { EvaluationDto } from '@latitude-data/core/browser'
import { ProgressIndicator, useCurrentDocument } from '@latitude-data/web-ui'
import {
useSockets,
type EventArgs,
} from '$/components/Providers/WebsocketsProvider/useSockets'

const DISAPERING_IN_MS = 5000
export function EvaluationStatusBanner({
evaluation,
jobs,
}: {
evaluation: EvaluationDto
jobs: EventArgs<'evaluationStatus'>[]
}) {
const timeoutRef = useRef<number | null>(null)
const [jobs, setJobs] = useState<EventArgs<'evaluationStatus'>[]>([])
const document = useCurrentDocument()

const onMessage = useCallback(
(args: EventArgs<'evaluationStatus'>) => {
if (evaluation.id !== args.evaluationId) return
if (document.documentUuid !== args.documentUuid) return

setJobs((prevJobs) => {
const jobIndex = prevJobs.findIndex(
(job) => job.batchId === args.batchId,
)

if (jobIndex === -1) {
return [...prevJobs, args]
} else {
const newJobs = [...prevJobs]
newJobs[jobIndex] = args

if (isDone(args)) {
setTimeout(() => {
setJobs((currentJobs) =>
currentJobs.filter((job) => job.batchId !== args.batchId),
)
}, DISAPERING_IN_MS)
}

return newJobs
}
})
},
[evaluation.id, document.documentUuid],
)

useEffect(() => {
return () => {
if (timeoutRef.current) {
clearTimeout(timeoutRef.current)
}
}
}, [])

useSockets({ event: 'evaluationStatus', onMessage })

return (
<>
{jobs.map((job) => (
<div key={job.batchId} className='flex flex-col gap-4'>
{!isDone(job) && (
{!isEvaluationRunDone(job) && (
<ProgressIndicator state='running'>
{`Running batch evaluation ${job.completed}/${job.total}`}
</ProgressIndicator>
)}
{job.errors > 0 && !isDone(job) && (
{job.errors > 0 && (
<ProgressIndicator state='error'>
Some evaluations failed to run. We won't retry them automatically
to avoid increasing provider costs. Total errors:{' '}
<strong>{job.errors}</strong>
</ProgressIndicator>
)}
{isDone(job) && (
{isEvaluationRunDone(job) && (
<ProgressIndicator state='completed'>
Batch evaluation completed! Total evaluations:{' '}
<strong>{job.total}</strong> · Total errors:{' '}
Expand All @@ -89,7 +38,3 @@ export function EvaluationStatusBanner({
</>
)
}

function isDone(job: EventArgs<'evaluationStatus'>) {
return job.total === job.completed + job.errors
}
Loading
Loading