-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(evaluations): add eject functionality for evaluations (#620)
- Implement `ejectEvaluationAction` to handle the ejection of evaluations to advanced mode. - Create `ejectEvaluation` service to manage the ejection process, including updating metadata and result configurations. - Add unit tests for `ejectEvaluationAction` to ensure proper handling of authorized and unauthorized scenarios. - Refactor evaluation editors to use a common `EvaluationEditorLayout` component for consistent UI and functionality. - Introduce `useProviderModel` hook to manage provider and model selection logic across different evaluation editors. This change allows users to convert simple evaluations into advanced evaluations with custom prompts, enhancing the flexibility and customization of the evaluation process.
- Loading branch information
Showing
10 changed files
with
577 additions
and
363 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| import { | ||
| EvaluationMetadataType, | ||
| EvaluationResultableType, | ||
| ProviderApiKey, | ||
| Providers, | ||
| } from '@latitude-data/core/browser' | ||
| import * as factories from '@latitude-data/core/factories' | ||
| import { Result } from '@latitude-data/core/lib/Result' | ||
| import { EvaluationsRepository } from '@latitude-data/core/repositories' | ||
| import { beforeEach, describe, expect, it, vi } from 'vitest' | ||
|
|
||
| import { ejectEvaluationAction } from './eject' | ||
|
|
||
| const mocks = vi.hoisted(() => { | ||
| return { | ||
| getSession: vi.fn(), | ||
| } | ||
| }) | ||
|
|
||
| vi.mock('$/services/auth/getSession', () => ({ | ||
| getSession: mocks.getSession, | ||
| })) | ||
|
|
||
| vi.mock('@latitude-data/core/repositories/evaluationsRepository') | ||
|
|
||
| describe('ejectEvaluationAction', () => { | ||
| describe('unauthorized', () => { | ||
| let evaluationId: number | ||
|
|
||
| beforeEach(async () => { | ||
| const { workspace, userData } = await factories.createWorkspace() | ||
| const provider = await factories.createProviderApiKey({ | ||
| workspace, | ||
| type: Providers.OpenAI, | ||
| name: 'Test Provider', | ||
| user: userData, | ||
| }) | ||
| const evaluation = await factories.createLlmAsJudgeEvaluation({ | ||
| workspace, | ||
| user: userData, | ||
| prompt: factories.helpers.createPrompt({ provider }), | ||
| }) | ||
| evaluationId = evaluation.id | ||
| }) | ||
|
|
||
| it('errors when the user is not authenticated', async () => { | ||
| mocks.getSession.mockReturnValue(null) | ||
|
|
||
| const [_, error] = await ejectEvaluationAction({ | ||
| id: evaluationId, | ||
| }) | ||
|
|
||
| expect(error!.name).toEqual('UnauthorizedError') | ||
| }) | ||
| }) | ||
|
|
||
| describe('authorized', () => { | ||
| let evaluation: any | ||
| let user: any | ||
| let workspace: any | ||
| let provider: ProviderApiKey | ||
|
|
||
| beforeEach(async () => { | ||
| const { workspace: createdWorkspace, userData } = | ||
| await factories.createWorkspace() | ||
|
|
||
| workspace = createdWorkspace | ||
| user = userData | ||
| provider = await factories.createProviderApiKey({ | ||
| workspace, | ||
| type: Providers.OpenAI, | ||
| name: 'Test Provider', | ||
| user, | ||
| }) | ||
| evaluation = await factories.createEvaluation({ | ||
| workspace, | ||
| user, | ||
| metadataType: EvaluationMetadataType.LlmAsJudgeSimple, | ||
| metadata: { | ||
| providerApiKeyId: provider.id, | ||
| model: 'gpt-4o', | ||
| objective: 'Test objective', | ||
| additionalInstructions: 'Test additional instructions', | ||
| }, | ||
| resultType: EvaluationResultableType.Boolean, | ||
| resultConfiguration: { | ||
| trueValueDescription: 'Test true value description', | ||
| falseValueDescription: 'Test false value description', | ||
| }, | ||
| }) | ||
|
|
||
| mocks.getSession.mockReturnValue({ | ||
| user: userData, | ||
| }) | ||
| }) | ||
|
|
||
| it('successfully ejects an evaluation', async () => { | ||
| // @ts-expect-error - Mocking the repository | ||
| vi.mocked(EvaluationsRepository).mockImplementation(() => ({ | ||
| find: vi.fn().mockResolvedValue(Result.ok(evaluation)), | ||
| })) | ||
|
|
||
| const [data, error] = await ejectEvaluationAction({ | ||
| id: evaluation.id, | ||
| }) | ||
|
|
||
| expect(error).toBeNull() | ||
| expect(data).toEqual(expect.objectContaining({ id: evaluation.id })) | ||
| }) | ||
|
|
||
| it('throws an error if evaluation is not found', async () => { | ||
| // @ts-expect-error - Mocking the repository | ||
| vi.mocked(EvaluationsRepository).mockImplementation(() => ({ | ||
| find: vi | ||
| .fn() | ||
| .mockResolvedValue(Result.error(new Error('Evaluation not found'))), | ||
| })) | ||
|
|
||
| const [_, error] = await ejectEvaluationAction({ | ||
| id: 999, | ||
| }) | ||
|
|
||
| expect(error).not.toBeNull() | ||
| expect(error!.message).toEqual('Evaluation not found') | ||
| }) | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| 'use server' | ||
|
|
||
| import { EvaluationsRepository } from '@latitude-data/core/repositories' | ||
| import { ejectEvaluation } from '@latitude-data/core/services/evaluations/eject' | ||
| import { z } from 'zod' | ||
|
|
||
| import { authProcedure } from '../procedures' | ||
|
|
||
| export const ejectEvaluationAction = authProcedure | ||
| .createServerAction() | ||
| .input(z.object({ id: z.coerce.number() })) | ||
| .handler(async ({ input, ctx }) => { | ||
| const scope = new EvaluationsRepository(ctx.workspace.id) | ||
| const evaluation = await scope.find(input.id).then((r) => r.unwrap()) | ||
| const result = await ejectEvaluation(evaluation) | ||
|
|
||
| return result.unwrap() | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.