diff --git a/.tmuxinator.yml b/.tmuxinator.yml index 931565ce1..9a52119a0 100644 --- a/.tmuxinator.yml +++ b/.tmuxinator.yml @@ -3,6 +3,6 @@ name: latitude-llm windows: - web: cd apps/web - - apps: pnpm dev --filter='./apps/*' + - apps: pnpm dev --filter='./apps/*' --filter='./packages/**' - docker: docker compose up - studio: cd packages/core && pnpm db:studio diff --git a/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.test.ts b/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.test.ts index ee3c58727..307234f45 100644 --- a/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.test.ts +++ b/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.test.ts @@ -15,8 +15,7 @@ import { testConsumeStream } from 'test/helpers' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ - addMessages: vi.fn(async ({ providerLogHandler }) => { - providerLogHandler({ uuid: 'fake-provider-log-uuid' }) + addMessages: vi.fn(async () => { const stream = new ReadableStream({ start(controller) { controller.enqueue({ @@ -43,9 +42,7 @@ const mocks = vi.hoisted(() => ({ }), queues: { defaultQueue: { - jobs: { - enqueueCreateProviderLogJob: vi.fn(), - }, + jobs: {}, }, }, })) @@ -149,26 +146,5 @@ describe('POST /add-message', () => { providerLogHandler: expect.any(Function), }) }) - - it('enqueue the provider log of the new message', async () => { - const res = await app.request(route, { - method: 'POST', - body, - headers, - }) - - expect(mocks.queues) - expect(res.status).toBe(200) - expect(res.body).toBeInstanceOf(ReadableStream) - await testConsumeStream(res.body as ReadableStream) - - expect( - mocks.queues.defaultQueue.jobs.enqueueCreateProviderLogJob, - ).toHaveBeenCalledWith({ - uuid: 'fake-provider-log-uuid', - source: LogSources.Playground, - apiKeyId: apiKey.id, - }) - }) }) }) diff --git a/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.ts b/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.ts index 8fdc34d98..0f26823af 100644 --- a/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.ts +++ b/apps/gateway/src/routes/api/v1/chats/handlers/addMessage.ts @@ -1,9 +1,9 @@ import { zValidator } from '@hono/zod-validator' import { LogSources } from '@latitude-data/core/browser' import { addMessages } from '@latitude-data/core/services/documentLogs/index' +import { createProviderLog } from '@latitude-data/core/services/providerLogs/create' import { messageSchema } from '$/common/messageSchema' import { pipeToStream } from '$/common/pipeToStream' -import { queues } from '$/jobs' import { Factory } from 'hono/factory' import { streamSSE } from 'hono/streaming' import { z } from 'zod' @@ -28,13 +28,12 @@ export const addMessageHandler = factory.createHandlers( workspace, documentLogUuid, messages, - providerLogHandler: (log) => { - // TODO: Review why this is possibly undefined now - queues.defaultQueue.jobs.enqueueCreateProviderLogJob!({ + providerLogHandler: async (log) => { + await createProviderLog({ ...log, source, apiKeyId: apiKey.id, - }) + }).then((r) => r.unwrap()) }, }).then((r) => r.unwrap()) diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts index 0587cd265..04fbc4ae4 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.ts @@ -1,6 +1,7 @@ import { zValidator } from '@hono/zod-validator' import { LogSources } from '@latitude-data/core/browser' import { runDocumentAtCommit } from '@latitude-data/core/services/commits/runDocumentAtCommit' +import { createProviderLog } from '@latitude-data/core/services/providerLogs/create' import { pipeToStream } from '$/common/pipeToStream' import { queues } from '$/jobs' import { Factory } from 'hono/factory' @@ -42,16 +43,17 @@ export const runHandler = factory.createHandlers( commit, parameters, providerLogHandler: (log) => { - queues.defaultQueue.jobs.enqueueCreateProviderLogJob({ + createProviderLog({ ...log, source, apiKeyId: apiKey.id, - }) + }).then((r) => r.unwrap()) }, }).then((r) => r.unwrap()) await pipeToStream(stream, result.stream) + // TODO: review if this is needed and why it's not in addMessages handler? queues.defaultQueue.jobs.enqueueCreateDocumentLogJob({ commit, data: { diff --git a/apps/web/src/actions/documents/create.ts b/apps/web/src/actions/documents/create.ts index 24d9a2907..7d0a2c427 100644 --- a/apps/web/src/actions/documents/create.ts +++ b/apps/web/src/actions/documents/create.ts @@ -11,14 +11,14 @@ export const createDocumentVersionAction = withProject .input( z.object({ path: z.string(), - commitId: z.number(), + commitUuid: z.string(), }), { type: 'json' }, ) .handler(async ({ input, ctx }) => { const commitsScope = new CommitsRepository(ctx.project.workspaceId) const commit = await commitsScope - .getCommitById(input.commitId) + .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project }) .then((r) => r.unwrap()) const result = await createNewDocument({ diff --git a/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts b/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts index 03a0bd513..980a51d7d 100644 --- a/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts +++ b/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts @@ -48,7 +48,7 @@ describe('destroyDocumentAction', async () => { it('fails when the user is not authenticated', async () => { const [_, error] = await destroyDocumentAction({ projectId: project.id, - commitId: draft.id, + commitUuid: draft.uuid, documentUuid: document.documentUuid, }) @@ -72,7 +72,7 @@ describe('destroyDocumentAction', async () => { const otherWorkspaceDocument = allDocs[0]! const [_, error] = await destroyDocumentAction({ projectId: otherWorkspaceProject.id, - commitId: otherCommit.id, + commitUuid: otherCommit.uuid, documentUuid: otherWorkspaceDocument.documentUuid, }) expect(error?.name).toEqual('NotFoundError') @@ -81,7 +81,7 @@ describe('destroyDocumentAction', async () => { it('fails when trying to remove a document from a merged commit', async () => { const [_, error] = await destroyDocumentAction({ projectId: project.id, - commitId: merged.id, + commitUuid: merged.uuid, documentUuid: document.documentUuid, }) expect(error?.name).toEqual('BadRequestError') @@ -90,7 +90,7 @@ describe('destroyDocumentAction', async () => { it('creates a soft deleted documents in draft document', async () => { await destroyDocumentAction({ projectId: project.id, - commitId: draft.id, + commitUuid: draft.uuid, documentUuid: document.documentUuid, }) // TODO: move to core diff --git a/apps/web/src/actions/documents/destroyDocumentAction/index.ts b/apps/web/src/actions/documents/destroyDocumentAction/index.ts index a6dc7db14..a5c10589b 100644 --- a/apps/web/src/actions/documents/destroyDocumentAction/index.ts +++ b/apps/web/src/actions/documents/destroyDocumentAction/index.ts @@ -10,13 +10,13 @@ import { z } from 'zod' export const destroyDocumentAction = withProject .createServerAction() - .input(z.object({ documentUuid: z.string(), commitId: z.number() }), { + .input(z.object({ documentUuid: z.string(), commitUuid: z.string() }), { type: 'json', }) .handler(async ({ input, ctx }) => { const commitsScope = new CommitsRepository(ctx.project.workspaceId) const commit = await commitsScope - .getCommitById(input.commitId) + .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project }) .then((r) => r.unwrap()) const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId) const document = await docsScope diff --git a/apps/web/src/actions/documents/destroyFolderAction.ts b/apps/web/src/actions/documents/destroyFolderAction.ts index 2b349fd4d..ddc72f91e 100644 --- a/apps/web/src/actions/documents/destroyFolderAction.ts +++ b/apps/web/src/actions/documents/destroyFolderAction.ts @@ -8,13 +8,13 @@ import { withProject } from '../procedures' export const destroyFolderAction = withProject .createServerAction() - .input(z.object({ path: z.string(), commitId: z.number() }), { + .input(z.object({ path: z.string(), commitUuid: z.string() }), { type: 'json', }) .handler(async ({ input, ctx }) => { const commitsScope = new CommitsRepository(ctx.project.workspaceId) const commit = await commitsScope - .getCommitById(input.commitId) + .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project }) .then((r) => r.unwrap()) const result = await destroyFolder({ path: input.path, diff --git a/apps/web/src/actions/documents/getDocumentsAtCommitAction.ts b/apps/web/src/actions/documents/getDocumentsAtCommitAction.ts index 3c2b2d878..deb3e4b03 100644 --- a/apps/web/src/actions/documents/getDocumentsAtCommitAction.ts +++ b/apps/web/src/actions/documents/getDocumentsAtCommitAction.ts @@ -10,10 +10,10 @@ import { withProject } from '../procedures' export const getDocumentsAtCommitAction = withProject .createServerAction() - .input(z.object({ commitId: z.number() })) + .input(z.object({ commitUuid: z.string() })) .handler(async ({ input, ctx }) => { const commit = await new CommitsRepository(ctx.project.workspaceId) - .getCommitById(input.commitId) + .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project }) .then((r) => r.unwrap()) const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId) const result = await docsScope.getDocumentsAtCommit(commit) diff --git a/apps/web/src/actions/documents/updateContent.test.ts b/apps/web/src/actions/documents/updateContent.test.ts index b7e1340ca..c0a9af846 100644 --- a/apps/web/src/actions/documents/updateContent.test.ts +++ b/apps/web/src/actions/documents/updateContent.test.ts @@ -1,6 +1,7 @@ import { DocumentVersion, Project, SafeUser } from '@latitude-data/core/browser' import * as factories from '@latitude-data/core/factories' import { updateDocument } from '@latitude-data/core/services/documents/update' +import { findCommitById } from 'node_modules/@latitude-data/core/src/data-access/commits' import { beforeEach, describe, expect, it, vi } from 'vitest' import { updateDocumentContentAction } from './updateContent' @@ -32,10 +33,13 @@ describe('updateDocumentAction', async () => { }) it('errors when the user is not authenticated', async () => { + const commit = await findCommitById({ id: doc1.commitId }).then((r) => + r.unwrap(), + ) const [_, error] = await updateDocumentContentAction({ projectId, documentUuid: doc1.documentUuid, - commitId: doc1.commitId, + commitUuid: commit.uuid, content: 'foo2', }) @@ -78,7 +82,7 @@ describe('updateDocumentAction', async () => { const [data, error] = await updateDocumentContentAction({ projectId: project.id, documentUuid: doc1.documentUuid, - commitId: draft.id, + commitUuid: draft.uuid, content: 'foo3', }) @@ -96,7 +100,7 @@ describe('updateDocumentAction', async () => { const [data, error] = await updateDocumentContentAction({ projectId: project.id, documentUuid: doc1.documentUuid, - commitId: draft.id, + commitUuid: draft.uuid, content: 'foo2', }) diff --git a/apps/web/src/actions/documents/updateContent.ts b/apps/web/src/actions/documents/updateContent.ts index e508a8bfc..d1aa1c20e 100644 --- a/apps/web/src/actions/documents/updateContent.ts +++ b/apps/web/src/actions/documents/updateContent.ts @@ -14,7 +14,7 @@ export const updateDocumentContentAction = withProject .input( z.object({ documentUuid: z.string(), - commitId: z.number(), + commitUuid: z.string(), content: z.string(), }), { type: 'json' }, @@ -22,7 +22,7 @@ export const updateDocumentContentAction = withProject .handler(async ({ input, ctx }) => { const commitsScope = new CommitsRepository(ctx.project.workspaceId) const commit = await commitsScope - .getCommitById(input.commitId) + .getCommitByUuid({ uuid: input.commitUuid, project: ctx.project }) .then((r) => r.unwrap()) const docsScope = new DocumentVersionsRepository(ctx.project.workspaceId) const document = await docsScope diff --git a/apps/web/src/actions/providerLogs/fetch.ts b/apps/web/src/actions/providerLogs/fetch.ts new file mode 100644 index 000000000..066611c2c --- /dev/null +++ b/apps/web/src/actions/providerLogs/fetch.ts @@ -0,0 +1,34 @@ +'use server' + +import { ProviderLogsRepository } from '@latitude-data/core/repositories' +import { z } from 'zod' + +import { authProcedure } from '../procedures' + +export const getProviderLogsAction = authProcedure + .createServerAction() + .input( + z.object({ + documentUuid: z.string().optional(), + documentLogUuid: z.string().optional(), + }), + ) + .handler(async ({ input, ctx }) => { + const { documentUuid, documentLogUuid } = input + const scope = new ProviderLogsRepository(ctx.workspace.id) + + let result + if (documentLogUuid) { + result = await scope + .findByDocumentLogUuid(documentLogUuid, { limit: 1000 }) + .then((r) => r.unwrap()) + } else if (documentUuid) { + result = await scope + .findByDocumentUuid(documentUuid) + .then((r) => r.unwrap()) + } else { + result = await scope.findAll({ limit: 1000 }).then((r) => r.unwrap()) + } + + return result + }) diff --git a/apps/web/src/actions/providerLogs/getProviderLogsForDocumentLogAction.ts b/apps/web/src/actions/providerLogs/getProviderLogsForDocumentLogAction.ts deleted file mode 100644 index ac7ab44ee..000000000 --- a/apps/web/src/actions/providerLogs/getProviderLogsForDocumentLogAction.ts +++ /dev/null @@ -1,17 +0,0 @@ -'use server' - -import { ProviderLogsRepository } from '@latitude-data/core/repositories' -import { z } from 'zod' - -import { authProcedure } from '../procedures' - -export const getProviderLogsForDocumentLogAction = authProcedure - .createServerAction() - .input(z.object({ documentLogUuid: z.string() })) - .handler(async ({ input, ctx }) => { - const providerLogsScope = new ProviderLogsRepository(ctx.workspace.id) - - return await providerLogsScope - .findByDocumentLogUuid(input.documentLogUuid) - .then((r) => r.unwrap()) - }) diff --git a/apps/web/src/actions/sdk/addMessagesAction.ts b/apps/web/src/actions/sdk/addMessagesAction.ts index 0a353fbcd..e73fe8461 100644 --- a/apps/web/src/actions/sdk/addMessagesAction.ts +++ b/apps/web/src/actions/sdk/addMessagesAction.ts @@ -28,7 +28,7 @@ export async function addMessagesAction({ const sdk = await createSdk().then((r) => r.unwrap()) const stream = createStreamableValue() - const response = sdk.addMessges({ + const response = sdk.addMessages({ params: { documentLogUuid, messages, source: LogSources.Playground }, onMessage: (chainEvent) => { stream.update(chainEvent) diff --git a/apps/web/src/app/(private)/_data-access/index.ts b/apps/web/src/app/(private)/_data-access/index.ts index afb23e7b5..b33ab0c9f 100644 --- a/apps/web/src/app/(private)/_data-access/index.ts +++ b/apps/web/src/app/(private)/_data-access/index.ts @@ -9,6 +9,7 @@ import { DocumentVersionsRepository, EvaluationsRepository, ProjectsRepository, + ProviderLogsRepository, } from '@latitude-data/core/repositories/index' import { getCurrentUser } from '$/services/auth/getCurrentUser' import { notFound } from 'next/navigation' @@ -163,3 +164,9 @@ export const getEvaluationByUuidCached = cache(async (uuid: string) => { return evaluation }) + +export const getProviderLogCached = cache(async (uuid: string) => { + const { workspace } = await getCurrentUser() + const scope = new ProviderLogsRepository(workspace.id) + return await scope.findByUuid(uuid).then((r) => r.unwrap()) +}) diff --git a/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx b/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx new file mode 100644 index 000000000..1c78f5232 --- /dev/null +++ b/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx @@ -0,0 +1,97 @@ +import { useEffect, useRef, useState } from 'react' + +import { + Chain, + Conversation, + ConversationMetadata, +} from '@latitude-data/compiler' +import { + Button, + ErrorMessage, + Message, + Text, + Tooltip, + useAutoScroll, +} from '@latitude-data/web-ui' + +export default function Preview({ + metadata, + parameters, + runPrompt, +}: { + metadata: ConversationMetadata | undefined + parameters: Record + runPrompt: () => void +}) { + const [conversation, setConversation] = useState( + undefined, + ) + const [completed, setCompleted] = useState(true) + const [error, setError] = useState(undefined) + const containerRef = useRef(null) + useAutoScroll(containerRef, { startAtBottom: true }) + + useEffect(() => { + if (!metadata) return + if (metadata.errors.length > 0) return + + const chain = new Chain({ + prompt: metadata.resolvedPrompt, + parameters, + }) + + chain + .step() + .then(({ conversation, completed }) => { + setError(undefined) + setConversation(conversation) + setCompleted(completed) + }) + .catch((error) => { + setConversation(undefined) + setCompleted(true) + setError(error) + }) + }, [metadata, parameters]) + + return ( +
+ Preview + {(conversation?.messages ?? []) + .filter((message) => message.role === 'assistant') + .map((message, index) => ( + + ))} + {error !== undefined && } + {!completed && ( +
+ + Showing the first step. Other steps will show after running. + +
+ )} + +
+ {error || (metadata?.errors.length ?? 0) > 0 ? ( + + Run prompt + + } + > + There are errors in your prompt. Please fix them before running. + + ) : ( + + )} +
+
+ ) +} diff --git a/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx b/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx new file mode 100644 index 000000000..1bf9b0913 --- /dev/null +++ b/apps/web/src/app/(private)/evaluations/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx @@ -0,0 +1,168 @@ +'use client' + +import { useCallback, useEffect, useMemo, useState } from 'react' + +import { ConversationMetadata } from '@latitude-data/compiler' +import { EvaluationDto } from '@latitude-data/core/browser' +import { + Badge, + Button, + Icons, + Input, + Text, + TextArea, +} from '@latitude-data/web-ui' +import { ROUTES } from '$/services/routes' +import useProviderLogs from '$/stores/providerLogs' +import Link from 'next/link' +import { useSearchParams } from 'next/navigation' + +import { Header } from '../Header' +import Preview from './Preview' + +type InputDto = { + value: string + type: 'textarea' | 'input' +} + +function convertParams(inputs: Record) { + return Object.fromEntries( + Object.entries(inputs).map(([key, inputDto]) => { + let value + try { + value = JSON.parse(inputDto.value) + } catch (e) { + // Do nothing + } + return [key, value] + }), + ) +} + +export default function Playground({ + evaluation, + metadata, +}: { + evaluation: EvaluationDto + metadata: ConversationMetadata +}) { + const [mode, setMode] = useState<'preview' | 'chat'>('preview') + const [inputs, setInputs] = useState>({}) + const parameters = useMemo(() => convertParams(inputs), [inputs]) + const searchParams = useSearchParams() + const providerLogUuid = searchParams.get('providerLogUuid') + const { data } = useProviderLogs() + const providerLog = useMemo( + () => data?.find((log) => log.uuid === providerLogUuid), + [data, providerLogUuid], + ) + + const setInput = useCallback((param: string, value: InputDto) => { + setInputs((inputs) => ({ ...inputs, [param]: value })) + }, []) + + useEffect(() => { + if (providerLog) { + setInput('messages', { + type: 'textarea', + value: JSON.stringify(providerLog.messages), + }) + } + }, [setInput, providerLog]) + + useEffect(() => { + if (!metadata) return + + // Remove only inputs that are no longer in the metadata, and add new ones + // Leave existing inputs as they are + setInputs( + Object.fromEntries( + Array.from(metadata.parameters).map((param) => { + if (param in inputs) return [param, inputs[param]!] + + return [param, { value: '', type: 'input' }] + }), + ), + ) + }, [metadata]) + + return ( + <> +
+ {mode === 'chat' && ( + + )} +
+
+
+
+ Variables + + Import data from logs + +
+ {Object.keys(inputs).length > 0 ? ( + Object.entries(inputs).map(([param, inputDto], idx) => ( +
+ {{{param}}} +
+ {inputDto.type === 'textarea' ? ( +