diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts index 9a6ed4096..e75389d6a 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/get.test.ts @@ -3,6 +3,7 @@ import { createDocumentVersion, createDraft, createProject, + helpers, } from '@latitude-data/core/factories' import { DocumentVersionsRepository } from '@latitude-data/core/repositories' import { apiKeys } from '@latitude-data/core/schema' @@ -28,7 +29,7 @@ describe('GET documents', () => { describe('authorized', () => { it('succeeds', async () => { - const { workspace, user, project } = await createProject() + const { workspace, user, project, providers } = await createProject() // TODO: move to core const apikey = await database.query.apiKeys.findFirst({ where: eq(apiKeys.workspaceId, workspace.id), @@ -38,7 +39,11 @@ describe('GET documents', () => { project, user, }) - const document = await createDocumentVersion({ commit, path }) + const document = await createDocumentVersion({ + commit, + path, + content: helpers.createPrompt({ provider: providers[0]! }), + }) await mergeCommit(commit).then((r) => r.unwrap()) diff --git a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts index 5aa835802..b757d0e42 100644 --- a/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts +++ b/apps/gateway/src/routes/api/v1/projects/:projectId/commits/:commitUuid/documents/handlers/run.test.ts @@ -10,6 +10,7 @@ import { createDocumentVersion, createDraft, createProject, + helpers, } from '@latitude-data/core/factories' import { Result } from '@latitude-data/core/lib/Result' import { apiKeys } from '@latitude-data/core/schema' @@ -74,7 +75,12 @@ describe('POST /run', () => { describe('authorized', () => { beforeEach(async () => { - const { workspace: wsp, user, project: prj } = await createProject() + const { + workspace: wsp, + user, + project: prj, + providers, + } = await createProject() project = prj workspace = wsp // TODO: move to core @@ -90,14 +96,7 @@ describe('POST /run', () => { const document = await createDocumentVersion({ commit: cmt, path, - content: ` - --- - provider: openai - model: gpt-4o - --- - - Ignore all the rest and just return "Hello". - `, + content: helpers.createPrompt({ provider: providers[0]! }), }) commit = await mergeCommit(cmt).then((r) => r.unwrap()) diff --git a/apps/web/src/actions/commits/publishDraftCommit.test.ts b/apps/web/src/actions/commits/publishDraftCommit.test.ts index 227bff14e..65824a36a 100644 --- a/apps/web/src/actions/commits/publishDraftCommit.test.ts +++ b/apps/web/src/actions/commits/publishDraftCommit.test.ts @@ -1,11 +1,13 @@ -import type { - Commit, - DocumentVersion, - Project, - User, - Workspace, +import { + Providers, + type Commit, + type DocumentVersion, + type Project, + type User, + type Workspace, } from '@latitude-data/core/browser' import * as factories from '@latitude-data/core/factories' +import { helpers } from '@latitude-data/core/factories' import { updateDocument } from '@latitude-data/core/services/documents/update' import { publishDraftCommitAction } from '$/actions/commits/publishDraftCommitAction' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -33,7 +35,10 @@ describe('publishDraftCommitAction', () => { project: prj, documents: docs, } = await factories.createProject({ - documents: { doc1: 'content' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: helpers.createPrompt({ provider: 'openai', content: 'content' }), + }, }) user = usr workspace = wp diff --git a/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts b/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts index 3f277b9c0..be1f2279e 100644 --- a/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts +++ b/apps/web/src/actions/documents/destroyDocumentAction/index.test.ts @@ -2,10 +2,15 @@ import { Commit, DocumentVersion, Project, + Providers, User, } from '@latitude-data/core/browser' import { database } from '@latitude-data/core/client' -import { createDraft, createProject } from '@latitude-data/core/factories' +import { + createDraft, + createProject, + helpers, +} from '@latitude-data/core/factories' import { documentVersions } from '@latitude-data/core/schema' import { and, eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -34,7 +39,10 @@ describe('destroyDocumentAction', async () => { commit: cmt, documents: allDocs, } = await createProject({ - documents: { doc1: 'Doc 1' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: helpers.createPrompt({ provider: 'openai', content: 'Doc 1' }), + }, }) const { commit } = await createDraft({ project: prj, user }) merged = cmt @@ -67,7 +75,10 @@ describe('destroyDocumentAction', async () => { commit: otherCommit, documents: allDocs, } = await createProject({ - documents: { doc1: 'Doc 1' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: helpers.createPrompt({ provider: 'openai', content: 'Doc 1' }), + }, }) const otherWorkspaceDocument = allDocs[0]! const [_, error] = await destroyDocumentAction({ diff --git a/apps/web/src/actions/documents/updateContent.test.ts b/apps/web/src/actions/documents/updateContent.test.ts index c3a6a0f60..1cb96c3bd 100644 --- a/apps/web/src/actions/documents/updateContent.test.ts +++ b/apps/web/src/actions/documents/updateContent.test.ts @@ -1,4 +1,9 @@ -import { DocumentVersion, Project, User } from '@latitude-data/core/browser' +import { + DocumentVersion, + Project, + Providers, + User, +} from '@latitude-data/core/browser' import * as factories from '@latitude-data/core/factories' import { updateDocument } from '@latitude-data/core/services/documents/update' import { findCommitById } from 'node_modules/@latitude-data/core/src/data-access/commits' @@ -24,8 +29,12 @@ describe('updateDocumentAction', async () => { const { workspace } = await factories.createWorkspace() const { documents, project } = await factories.createProject({ workspace, + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { - doc1: 'foo', + doc1: factories.helpers.createPrompt({ + provider: 'openai', + content: 'foo', + }), }, }) doc1 = documents.filter((d) => d.path === 'doc1')[0]! @@ -57,8 +66,12 @@ describe('updateDocumentAction', async () => { const { documents, project: projectData } = await factories.createProject( { workspace, + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { - doc1: 'foo', + doc1: factories.helpers.createPrompt({ + provider: 'openai', + content: 'foo', + }), }, }, ) diff --git a/apps/web/src/actions/evaluations/connect.test.ts b/apps/web/src/actions/evaluations/connect.test.ts index e7974eb4a..d0554c93d 100644 --- a/apps/web/src/actions/evaluations/connect.test.ts +++ b/apps/web/src/actions/evaluations/connect.test.ts @@ -44,7 +44,13 @@ describe('connectEvaluationsAction', () => { beforeEach(async () => { const setup = await factories.createProject({ - documents: { 'test-doc': 'Test content' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + 'test-doc': factories.helpers.createPrompt({ + provider: 'openai', + content: 'Test content', + }), + }, }) workspace = setup.workspace user = setup.user diff --git a/apps/web/src/actions/evaluations/runBatch.test.ts b/apps/web/src/actions/evaluations/runBatch.test.ts index 6a5b060cb..384bf5c8a 100644 --- a/apps/web/src/actions/evaluations/runBatch.test.ts +++ b/apps/web/src/actions/evaluations/runBatch.test.ts @@ -4,6 +4,7 @@ import { DocumentVersion, EvaluationDto, Project, + Providers, User, Workspace, } from '@latitude-data/core/browser' @@ -67,7 +68,13 @@ describe('runBatchAction', () => { vi.clearAllMocks() const setup = await factories.createProject({ - documents: { 'test-doc': 'Test content' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + 'test-doc': factories.helpers.createPrompt({ + provider: 'openai', + content: 'Test content', + }), + }, }) workspace = setup.workspace user = setup.user diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx index 062781a89..87ede39cb 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx @@ -3,6 +3,7 @@ import { Suspense, useCallback, useEffect, useMemo, useState } from 'react' import { ConversationMetadata, readMetadata } from '@latitude-data/compiler' +import { promptConfigSchema } from '@latitude-data/core/browser' import { Button, DocumentTextEditor, @@ -10,6 +11,7 @@ import { } from '@latitude-data/web-ui' import EditorHeader from '$/components/EditorHeader' import useEvaluations from '$/stores/evaluations' +import useProviderApiKeys from '$/stores/providerApiKeys' import Playground from './Playground' import { EVALUATION_PARAMETERS } from './Playground/Chat' @@ -26,8 +28,13 @@ export default function EvaluationEditor({ () => data.find((e) => e.uuid === evaluationUuid), [evaluationUuid, data], ) + const { data: providers } = useProviderApiKeys() const [value, setValue] = useState(defaultPrompt) const [metadata, setMetadata] = useState() + const configSchema = useMemo( + () => promptConfigSchema({ providers: providers ?? [] }), + [providers], + ) const save = useCallback( (val: string) => { update({ @@ -49,8 +56,9 @@ export default function EvaluationEditor({ readMetadata({ prompt: value, withParameters: EVALUATION_PARAMETERS, + configSchema, }).then(setMetadata) - }, [value]) + }, [value, configSchema]) if (!evaluation) return null diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx index 517c74931..598dd825a 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx @@ -6,6 +6,7 @@ import { Suspense, useCallback, useEffect, + useMemo, useState, } from 'react' @@ -14,7 +15,10 @@ import { readMetadata, Document as RefDocument, } from '@latitude-data/compiler' -import { DocumentVersion } from '@latitude-data/core/browser' +import { + DocumentVersion, + promptConfigSchema, +} from '@latitude-data/core/browser' import { DocumentTextEditor, DocumentTextEditorFallback, @@ -25,6 +29,7 @@ import { type AddMessagesActionFn } from '$/actions/sdk/addMessagesAction' import type { RunDocumentActionFn } from '$/actions/sdk/runDocumentAction' import EditorHeader from '$/components/EditorHeader' import useDocumentVersions from '$/stores/documentVersions' +import useProviderApiKeys from '$/stores/providerApiKeys' import { useDebouncedCallback } from 'use-debounce' import Playground from './Playground' @@ -50,6 +55,7 @@ export default function DocumentEditor({ }) { const { commit } = useCurrentCommit() const { project } = useCurrentProject() + const { data: providers } = useProviderApiKeys() const { data: _documents, updateContent } = useDocumentVersions( { @@ -63,6 +69,10 @@ export default function DocumentEditor({ const [value, setValue] = useState(document.content) const [isSaved, setIsSaved] = useState(true) const [metadata, setMetadata] = useState() + const configSchema = useMemo( + () => promptConfigSchema({ providers }), + [providers], + ) const debouncedSave = useDebouncedCallback( (val: string) => { @@ -123,8 +133,9 @@ export default function DocumentEditor({ prompt: value, fullPath: document.path, referenceFn: readDocument, + configSchema, }).then(setMetadata) - }, [readDocument]) + }, [readDocument, configSchema]) const isMerged = commit.mergedAt !== null return ( diff --git a/apps/web/src/components/EditorHeader/index.tsx b/apps/web/src/components/EditorHeader/index.tsx index a0b34fab8..8137d7758 100644 --- a/apps/web/src/components/EditorHeader/index.tsx +++ b/apps/web/src/components/EditorHeader/index.tsx @@ -14,7 +14,6 @@ import { Text, useLocalStorage, } from '@latitude-data/web-ui' -import { useWritePromptProvider } from '$/components/EditorHeader/useWritePromptProvider' import useProviderApiKeys from '$/stores/providerApiKeys' const CUSTOM_MODEL = 'custom-model' @@ -49,7 +48,6 @@ function selectModel({ export default function EditorHeader({ title, - prompt, metadata, onChangePrompt, rightActions, @@ -68,30 +66,32 @@ export default function EditorHeader({ const model = config?.model as string return { providerName, model } }, [metadata?.config]) + const { data: providerApiKeys, isLoading } = useProviderApiKeys() + const { value: showLineNumbers, setValue: setShowLineNumbers } = useLocalStorage({ key: AppLocalStorage.editorLineNumbers, defaultValue: true, }) + const { value: wrapText, setValue: setWrapText } = useLocalStorage({ key: AppLocalStorage.editorWrapText, defaultValue: true, }) + const { value: showMinimap, setValue: setShowMinimap } = useLocalStorage({ key: AppLocalStorage.editorMinimap, defaultValue: false, }) - const { onProviderDataChange } = useWritePromptProvider({ - prompt, - onChangePrompt, - }) + const providersByName = useMemo(() => { return providerApiKeys.reduce((acc, data) => { acc[data.name] = data return acc }, {} as IProviderByName) }, [isLoading, providerApiKeys]) + const [selectedProvider, setProvider] = useState() const [selectedModel, setModel] = useState(() => selectModel({ @@ -99,6 +99,7 @@ export default function EditorHeader({ providersByName, }), ) + const providerOptions = useMemo(() => { return providerApiKeys.map((apiKey) => ({ label: apiKey.name, @@ -110,6 +111,7 @@ export default function EditorHeader({ ? providersByName[selectedProvider]?.provider : undefined, }) + useEffect(() => { if (isLoading) return @@ -124,6 +126,7 @@ export default function EditorHeader({ providersByName, promptMetadata?.providerName, ]) + useEffect(() => { if (isLoading) return @@ -142,8 +145,11 @@ export default function EditorHeader({ promptMetadata?.providerName, promptMetadata?.model, ]) + const onSelectProvider = useCallback( (value: string) => { + if (!metadata) return + const provider = providersByName[value]! if (!provider) return @@ -152,25 +158,27 @@ export default function EditorHeader({ PROVIDER_MODELS[provider.provider] ?? {}, )[0] setModel(firstModel) - onProviderDataChange({ - name: provider.name, - model: firstModel, - }) + + const config = metadata.config + config.provider = provider.name + config.model = firstModel + onChangePrompt(metadata.setConfig(config)) }, - [providersByName, onProviderDataChange], + [providersByName, metadata], ) + const onModelChange = useCallback( (value: string) => { + if (!metadata) return if (value === CUSTOM_MODEL) return if (!selectedProvider) return setModel(value) - onProviderDataChange({ - name: selectedProvider, - model: value, - }) + const config = metadata.config + config.model = value + onChangePrompt(metadata.setConfig(config)) }, - [onProviderDataChange, selectedProvider], + [selectedProvider, metadata], ) return ( @@ -206,16 +214,19 @@ export default function EditorHeader({ { - it('parse and empty prompt and set provider', () => { - const onChangePromptMock = vi.fn() - const { result } = renderHook(() => - useWritePromptProvider({ - prompt: '', - onChangePrompt: onChangePromptMock, - }), - ) - act(() => { - result.current.onProviderDataChange({ name: 'openai', model: 'gpt-4o' }) - }) - expect(onChangePromptMock).toHaveBeenCalledWith( - trimStartSpace(` - --- - provider: openai - model: gpt-4o - --- - `), - ) - }) - - it('parse a prompt with a provider and set a new provider', () => { - const onChangePromptMock = vi.fn() - const { result } = renderHook(() => - useWritePromptProvider({ - prompt: trimStartSpace(` - --- - provider: openai - model: gpt-4o - --- - `), - onChangePrompt: onChangePromptMock, - }), - ) - act(() => { - result.current.onProviderDataChange({ - name: 'mistral', - model: 'open-mistral-nemo-2407', - }) - }) - expect(onChangePromptMock).toHaveBeenCalledWith( - trimStartSpace(`--- - provider: mistral - model: open-mistral-nemo-2407 - --- - `), - ) - }) - - it('parse a prompt with only a provider', () => { - const onChangePromptMock = vi.fn() - const { result } = renderHook(() => - useWritePromptProvider({ - prompt: trimStartSpace(` - --- - provider: openai - --- - `), - onChangePrompt: onChangePromptMock, - }), - ) - act(() => { - result.current.onProviderDataChange({ - name: 'mistral', - model: 'open-mistral-nemo-2407', - }) - }) - expect(onChangePromptMock).toHaveBeenCalledWith( - trimStartSpace(`--- - provider: mistral - model: open-mistral-nemo-2407 - --- - `), - ) - }) - - it('parse a prompt with temperature but not model or provider', () => { - const onChangePromptMock = vi.fn() - const { result } = renderHook(() => - useWritePromptProvider({ - prompt: trimStartSpace(` - --- - temperature: 0.5 - --- - `), - onChangePrompt: onChangePromptMock, - }), - ) - act(() => { - result.current.onProviderDataChange({ - name: 'mistral', - model: 'open-mistral-nemo-2407', - }) - }) - expect(onChangePromptMock).toHaveBeenCalledWith( - trimStartSpace(`--- - temperature: 0.5 - provider: mistral - model: open-mistral-nemo-2407 - --- - `), - ) - }) - - it('with a comment before yaml header', () => { - const onChangePromptMock = vi.fn() - const { result } = renderHook(() => - useWritePromptProvider({ - prompt: trimStartSpace(` - /* This is a comment */ - --- - temperature: 0.5 - --- - `), - onChangePrompt: onChangePromptMock, - }), - ) - act(() => { - result.current.onProviderDataChange({ - name: 'mistral', - model: 'open-mistral-nemo-2407', - }) - }) - expect(onChangePromptMock).toHaveBeenCalledWith( - trimStartSpace(`/* This is a comment */ - --- - temperature: 0.5 - provider: mistral - model: open-mistral-nemo-2407 - --- - `), - ) - }) -}) diff --git a/apps/web/src/components/EditorHeader/useWritePromptProvider/index.ts b/apps/web/src/components/EditorHeader/useWritePromptProvider/index.ts deleted file mode 100644 index bab772c38..000000000 --- a/apps/web/src/components/EditorHeader/useWritePromptProvider/index.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { useCallback, useMemo } from 'react' - -import { Config } from '@latitude-data/compiler' -import yaml from 'yaml' - -const findSeparator = (line: string) => line.trim() === '---' -export function trimStartSpace(text: string) { - return text - .split('\n') - .map((line) => line.trimStart()) - .filter((line, index, arr) => !(index === arr.length - 1 && line === '')) - .join('\n') - .replace(/^\n+/, '') -} - -type ParsedDoc = { - beforeYaml: string - config: Config - afterYaml: string -} -export function useWritePromptProvider({ - prompt, - onChangePrompt, -}: { - prompt: string - onChangePrompt: (prompt: string) => void -}) { - const parsedDoc = useMemo(() => { - const promptLines = prompt.split('\n') - const start = promptLines.findIndex(findSeparator) - let parsedConfig: Config = {} - - if (start === -1) { - return { - beforeYaml: '', - config: parsedConfig, - afterYaml: prompt, - } - } - - const endRelative = promptLines.slice(start + 1).findIndex(findSeparator) - - if (endRelative === -1) { - return { - beforeYaml: '', - config: parsedConfig, - afterYaml: prompt, - } - } - - const absoluteEndIndex = start + 1 + endRelative - const yamlSection = promptLines - .slice(start + 1, absoluteEndIndex) - .join('\n') - const beforeYaml = promptLines.slice(0, start).join('\n') - const afterYaml = promptLines.slice(absoluteEndIndex + 1).join('\n') - - try { - parsedConfig = yaml.parse(yamlSection) as Config - } catch { - parsedConfig = {} - } - - return { - beforeYaml, - config: parsedConfig, - afterYaml, - } - }, [prompt]) - - const onProviderDataChange = useCallback( - ({ name, model }: { name: string; model: string | undefined }) => { - const config = parsedDoc.config - - if (config.provider === name && config.model === model) return - if (config.provider !== name) { - config.provider = name - } - if (config.model !== model) { - config.model = model - } - - const newPrompt = trimStartSpace(`${parsedDoc.beforeYaml} - --- - ${trimStartSpace(yaml.stringify(config))} - --- - ${parsedDoc.afterYaml}`) - - onChangePrompt(newPrompt) - }, - [parsedDoc, onChangePrompt], - ) - - return { - onProviderDataChange, - } -} diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 82731bbe7..8b69cb322 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -23,7 +23,8 @@ "acorn": "^8.9.0", "code-red": "^1.0.3", "locate-character": "^3.0.0", - "yaml": "^2.4.5" + "yaml": "^2.4.5", + "zod": "^3.23.8" }, "devDependencies": { "@latitude-data/eslint-config": "workspace:*", diff --git a/packages/compiler/src/compiler/base/nodes/config.ts b/packages/compiler/src/compiler/base/nodes/config.ts index 977d8af70..cbd2941c6 100644 --- a/packages/compiler/src/compiler/base/nodes/config.ts +++ b/packages/compiler/src/compiler/base/nodes/config.ts @@ -1,4 +1,5 @@ import { Config as ConfigNode } from '$compiler/parser/interfaces' +import yaml from 'yaml' import { CompileNodeContext } from '../types' @@ -6,5 +7,5 @@ export async function compile({ node, setConfig, }: CompileNodeContext): Promise { - setConfig(node.value) + setConfig(yaml.parse(node.value)) } diff --git a/packages/compiler/src/compiler/compile.test.ts b/packages/compiler/src/compiler/compile.test.ts index d590f60d4..8f76a359b 100644 --- a/packages/compiler/src/compiler/compile.test.ts +++ b/packages/compiler/src/compiler/compile.test.ts @@ -69,24 +69,6 @@ describe('config section', async () => { baz: ['qux', 'quux'], }) }) - - it('does not compile the prompt as YAML when it is not the first element in the prompt', async () => { - const prompt = ` - Lorem ipsum - --- - foo: bar - baz: - - qux - - quux - --- - ` - const result = await render({ - prompt: removeCommonIndent(prompt), - parameters: {}, - }) - - expect(result.config).toEqual({}) - }) }) describe('comments', async () => { diff --git a/packages/compiler/src/compiler/config.ts b/packages/compiler/src/compiler/config.ts deleted file mode 100644 index 03b66a58a..000000000 --- a/packages/compiler/src/compiler/config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Fragment } from '$compiler/parser/interfaces' -import type { Config } from '$compiler/types' - -export function readConfig(fragment: Fragment): Config { - for (const node of fragment.children) { - if (node.type === 'Config') { - return node.value - } - } - return {} -} diff --git a/packages/compiler/src/compiler/index.ts b/packages/compiler/src/compiler/index.ts index 1c337b118..dc3ded224 100644 --- a/packages/compiler/src/compiler/index.ts +++ b/packages/compiler/src/compiler/index.ts @@ -1,4 +1,5 @@ import { Conversation, ConversationMetadata } from '$compiler/types' +import { z } from 'zod' import { Chain } from './chain' import { @@ -37,16 +38,19 @@ export function readMetadata({ fullPath, referenceFn, withParameters, + configSchema, }: { prompt: string fullPath?: string referenceFn?: ReferencePromptFn withParameters?: string[] + configSchema?: z.ZodType }): Promise { return new ReadMetadata({ document: { path: fullPath ?? '', content: prompt }, referenceFn, withParameters, + configSchema, }).run() } diff --git a/packages/compiler/src/compiler/readMetadata.test.ts b/packages/compiler/src/compiler/readMetadata.test.ts index c3677f0f8..110f67787 100644 --- a/packages/compiler/src/compiler/readMetadata.test.ts +++ b/packages/compiler/src/compiler/readMetadata.test.ts @@ -1,6 +1,7 @@ import { CUSTOM_TAG_END, CUSTOM_TAG_START } from '$compiler/constants' import CompileError from '$compiler/error/error' import { describe, expect, it } from 'vitest' +import { z } from 'zod' import { readMetadata } from '.' import { Document } from './readMetadata' @@ -253,8 +254,8 @@ describe('config', async () => { }) }) - it('does not compile the prompt as YAML when it is not the first element in the prompt', async () => { - const prompt = ` + it('fails when there is content before the config section', async () => { + const prompt = removeCommonIndent(` Lorem ipsum --- foo: bar @@ -262,13 +263,157 @@ describe('config', async () => { - qux - quux --- - ` + `) + + const metadata = await readMetadata({ prompt }) + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('invalid-config-placement') + }) + + it('fails when the config is not valid YAML', async () => { + const prompt = removeCommonIndent(` + --- + foo: bar + baa + --- + `) + + const metadata = await readMetadata({ prompt }) + + expect(metadata.config).toEqual({ foo: 'bar', baa: null }) + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('invalid-config') + }) + + it('fails when there are multiple config sections', async () => { + const prompt = removeCommonIndent(` + --- + foo: bar + --- + --- + baz: qux + --- + `) + + const metadata = await readMetadata({ prompt }) + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('config-already-declared') + }) + + it('fails when a schema is provided and there is no config section', async () => { + const prompt = removeCommonIndent(` + Lorem ipsum + `) const metadata = await readMetadata({ - prompt: removeCommonIndent(prompt), + prompt, + configSchema: z.object({ + foo: z.string(), + }), }) - expect(metadata.config).toEqual({}) + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('config-not-found') + }) + + it('fails when the configSchema is not validated', async () => { + const prompt = removeCommonIndent(` + --- + foo: 2 + --- + `) + + const metadata = await readMetadata({ + prompt, + configSchema: z.object({ + foo: z.string(), + }), + }) + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('invalid-config') + }) + + it('does not fail when the config schema is validated', async () => { + const prompt = removeCommonIndent(` + --- + foo: bar + --- + `) + + const metadata = await readMetadata({ + prompt, + configSchema: z.object({ + foo: z.string(), + }), + }) + + expect(metadata.errors.length).toBe(0) + }) + + it('returns the correct positions of parsing errors', async () => { + const prompt = removeCommonIndent(` + /* + Lorem ipsum + */ + --- + foo: bar + baa + --- + `) + + const expectedErrorPosition = prompt.indexOf('baa') + + const metadata = await readMetadata({ prompt }) + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('invalid-config') + expect(metadata.errors[0]!.pos).toBe(expectedErrorPosition) + }) + + it('returns the correct positions of schema errors', async () => { + const prompt = removeCommonIndent(` + --- + foo: bar + --- + `) + + const metadata = await readMetadata({ + prompt, + configSchema: z.object({ + foo: z.number(), + }), + }) + const expectedErrorPosition = prompt.indexOf('bar') + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('invalid-config') + expect(metadata.errors[0]!.pos).toBe(expectedErrorPosition) + }) + + it('fails when the config section is defined inside an if block', async () => { + const prompt = removeCommonIndent(` + ${CUSTOM_TAG_START}#if true${CUSTOM_TAG_END} + --- + foo: bar + --- + ${CUSTOM_TAG_START}/if${CUSTOM_TAG_END} + `) + + const metadata = await readMetadata({ prompt }) + + expect(metadata.errors.length).toBe(1) + expect(metadata.errors[0]).toBeInstanceOf(CompileError) + expect(metadata.errors[0]!.code).toBe('config-outside-root') }) }) diff --git a/packages/compiler/src/compiler/readMetadata.ts b/packages/compiler/src/compiler/readMetadata.ts index 2cf54f170..d52af0f5e 100644 --- a/packages/compiler/src/compiler/readMetadata.ts +++ b/packages/compiler/src/compiler/readMetadata.ts @@ -18,12 +18,13 @@ import type { } from '$compiler/parser/interfaces' import { Config, ConversationMetadata, MessageRole } from '$compiler/types' import { Node as LogicalExpression } from 'estree' -import yaml from 'yaml' +import yaml, { Node as YAMLItem } from 'yaml' +import { z } from 'zod' -import { readConfig } from './config' import { updateScopeContextForNode } from './logic' import { ScopeContext } from './scope' import { + findYAMLItemPosition, isChainStepTag, isContentTag, isMessageTag, @@ -52,9 +53,13 @@ export class ReadMetadata { private referenceFn?: ReferencePromptFn private fullPath: string private withParameters?: string[] + private configSchema?: z.ZodType + private config?: Config + private configPosition?: { start: number; end: number } private resolvedPrompt: string private resolvedPromptOffset: number = 0 + private hasContent: boolean = false private accumulatedToolCalls: ToolCallTag[] = [] private errors: CompileError[] = [] @@ -66,15 +71,18 @@ export class ReadMetadata { document, referenceFn, withParameters, + configSchema, }: { document: Document referenceFn?: ReferencePromptFn withParameters?: string[] + configSchema?: z.ZodType }) { this.rawText = document.content this.referenceFn = referenceFn this.fullPath = document.path this.withParameters = withParameters + this.configSchema = configSchema this.resolvedPrompt = document.content } @@ -89,7 +97,6 @@ export class ReadMetadata { } let fragment: Fragment - let config: Config = {} try { fragment = parse(this.rawText) @@ -108,13 +115,36 @@ export class ReadMetadata { scopeContext, isInsideMessageTag: false, isInsideContentTag: false, + isRoot: true, }) - config = readConfig(fragment) - let resolvedPrompt = this.resolvedPrompt - if (config && Object.keys(config).length > 0) { - const configYaml = yaml.stringify(config) - resolvedPrompt = `---\n${configYaml}---\n${this.resolvedPrompt}` + if (this.configSchema && !this.config) { + this.baseNodeError(errors.missingConfig, fragment, { start: 0, end: 0 }) + } + + const resolvedPrompt = + Object.keys(this.config ?? {}).length > 0 + ? '---\n' + + yaml.stringify(this.config, { indent: 2 }) + + '---\n' + + this.resolvedPrompt + : this.resolvedPrompt + + const setConfig = (config: Config) => { + const start = this.configPosition?.start ?? 0 + const end = this.configPosition?.end ?? 0 + + if (Object.keys(config).length === 0) { + return this.rawText.slice(0, start) + this.rawText.slice(end) + } + + return ( + this.rawText.slice(0, start) + + '---\n' + + yaml.stringify(config, { indent: 2 }) + + '---\n' + + this.rawText.slice(end) + ) } return { @@ -123,8 +153,9 @@ export class ReadMetadata { ...(scopeContext.onlyPredefinedVariables ?? new Set([])), ]), resolvedPrompt, - config, + config: this.config ?? {}, errors: this.errors, + setConfig, } } @@ -190,11 +221,13 @@ export class ReadMetadata { scopeContext, isInsideMessageTag, isInsideContentTag, + isRoot = false, }: { node: TemplateNode scopeContext: ScopeContext isInsideMessageTag: boolean isInsideContentTag: boolean + isRoot?: boolean }): Promise { if (node.type === 'Fragment') { for await (const childNode of node.children ?? []) { @@ -203,27 +236,91 @@ export class ReadMetadata { scopeContext, isInsideMessageTag, isInsideContentTag, + isRoot, }) } return } - if (node.type === 'Config' || node.type === 'Comment') { + if (node.type === 'Comment' || node.type === 'Config') { /* Remove from the resolved prompt */ const start = node.start! + this.resolvedPromptOffset const end = node.end! + this.resolvedPromptOffset this.resolvedPrompt = this.resolvedPrompt.slice(0, start) + this.resolvedPrompt.slice(end) this.resolvedPromptOffset -= end - start + } + + if (node.type === 'Config') { + if (this.config) { + this.baseNodeError(errors.configAlreadyDeclared, node) + } + if (!isRoot) { + this.baseNodeError(errors.configOutsideRoot, node) + } + if (this.hasContent) { + this.baseNodeError(errors.invalidConfigPlacement, node) + } + + this.configPosition = { start: node.start!, end: node.end! } + + const parsedYaml = yaml.parseDocument(node.value, { + keepSourceTokens: true, + }) + + const CONFIG_START_OFFSET = 3 // The config is always offsetted by 3 characters due to the `---` + + if (parsedYaml.errors.length) { + parsedYaml.errors.forEach((error) => { + const [errorStart, errorEnd] = error.pos + this.baseNodeError(errors.invalidConfig(error.message), node, { + start: node.start! + CONFIG_START_OFFSET + errorStart, + end: node.start! + CONFIG_START_OFFSET + errorEnd, + }) + }) + } + + const parsedObj = parsedYaml.toJS() + + try { + this.configSchema?.parse(parsedObj) + } catch (err) { + if (err instanceof z.ZodError) { + err.errors.forEach((error) => { + const issue = error.message + + const [errorStart, errorEnd] = findYAMLItemPosition( + parsedYaml.contents as YAMLItem, + error.path, + )! + + this.baseNodeError(errors.invalidConfig(issue), node, { + start: node.start! + CONFIG_START_OFFSET + errorStart, + end: node.start! + CONFIG_START_OFFSET + errorEnd + 1, + }) + }) + } + } + + this.config = parsedObj return } if (node.type === 'Text') { + if (node.data.trim()) { + this.hasContent = true + } + /* do nothing */ + return + } + + if (node.type === 'Comment') { /* do nothing */ return } if (node.type === 'MustacheTag') { + this.hasContent = true const expression = node.expression await this.updateScopeContext({ node: expression, scopeContext }) return @@ -299,6 +396,7 @@ export class ReadMetadata { } if (node.type === 'ElementTag') { + this.hasContent = true if (isToolCallTag(node)) { if (isInsideContentTag) { this.baseNodeError(errors.toolCallTagInsideContent, node) @@ -545,14 +643,15 @@ export class ReadMetadata { private baseNodeError( { code, message }: { code: string; message: string }, node: BaseNode, + customPos?: { start: number; end: number }, ): void { try { error(message, { name: 'CompileError', code, source: this.rawText || '', - start: node.start || 0, - end: node.end || undefined, + start: customPos?.start || node.start || 0, + end: customPos?.end || node.end || undefined, }) } catch (error) { this.errors.push(error as CompileError) diff --git a/packages/compiler/src/compiler/utils.ts b/packages/compiler/src/compiler/utils.ts index 45518ba54..53ee8d9a5 100644 --- a/packages/compiler/src/compiler/utils.ts +++ b/packages/compiler/src/compiler/utils.ts @@ -13,6 +13,7 @@ import { ToolCallTag, } from '$compiler/parser/interfaces' import { ContentType, MessageRole } from '$compiler/types' +import { Scalar, Node as YAMLItem, YAMLMap, YAMLSeq } from 'yaml' export function isIterable(obj: unknown): obj is Iterable { return (obj as Iterable)?.[Symbol.iterator] !== undefined @@ -69,3 +70,28 @@ export function tagAttributeIsLiteral(tag: ElementTag, name: string): boolean { if (attr.value === true) return true return attr.value.every((v) => v.type === 'Text') } + +type YAMLItemRange = [number, number] | undefined +export function findYAMLItemPosition( + parent: YAMLItem, + path: (string | number)[], +): YAMLItemRange { + const parentRange: YAMLItemRange = parent.range + ? [parent.range[0], parent.range[1]] + : undefined + + if (path.length === 0 || !('items' in parent)) return parentRange + + let child: YAMLItem | undefined + if (parent instanceof YAMLMap) { + child = parent.items.find((i) => { + return (i.key as Scalar)?.value === path[0]! + })?.value as YAMLItem | undefined + } + if (parent instanceof YAMLSeq && typeof path[0] === 'number') { + child = parent.items[Number(path[0])] as YAMLItem | undefined + } + + if (!child) return parentRange + return findYAMLItemPosition(child, path.slice(1)) ?? parentRange +} diff --git a/packages/compiler/src/error/error.ts b/packages/compiler/src/error/error.ts index bbe1422f0..56128ce31 100644 --- a/packages/compiler/src/error/error.ts +++ b/packages/compiler/src/error/error.ts @@ -65,8 +65,14 @@ function getCodeFrame( export function error(message: string, props: CompileErrorProps): never { const error = new CompileError(message) error.name = props.name - const start = locate(props.source, props.start, { offsetLine: 1 }) - const end = locate(props.source, props.end || props.start, { offsetLine: 1 }) + const start = locate(props.source, props.start, { + offsetLine: 1, + offsetColumn: 1, + }) + const end = locate(props.source, props.end ?? props.start, { + offsetLine: 1, + offsetColumn: 1, + }) error.code = props.code error.start = start error.end = end diff --git a/packages/compiler/src/error/errors.ts b/packages/compiler/src/error/errors.ts index bb1714d0d..32eb443f9 100644 --- a/packages/compiler/src/error/errors.ts +++ b/packages/compiler/src/error/errors.ts @@ -90,16 +90,33 @@ export default { code: 'invalid-logic-block-placement', message: `${CUSTOM_TAG_START}#${name}${CUSTOM_TAG_END} block cannot be ${location}`, }), - invalidConfig: (message: string) => ({ - code: 'invalid-config', - message: `Invalid config: ${message}`, - }), unexpectedTagClose: (name: string) => ({ code: 'unexpected-tag-close', message: `Unexpected closing tag for ${name}`, }), /* COMPILER ERRORS */ + missingConfig: { + code: 'config-not-found', + message: + 'A configuration section is required. For example:\n\n---\nprovider: openai\nmodel: gpt-4\n---\n', + }, + configAlreadyDeclared: { + code: 'config-already-declared', + message: 'Cannot declare config twice', + }, + configOutsideRoot: { + code: 'config-outside-root', + message: 'Config can only be declared at the root of the prompt', + }, + invalidConfig: (message: string) => ({ + code: 'invalid-config', + message, + }), + invalidConfigPlacement: { + code: 'invalid-config-placement', + message: 'Config must be defined before any content', + }, unsupportedBaseNodeType: (type: string) => ({ code: 'unsupported-base-node-type', message: `Unsupported base node type: ${type}`, diff --git a/packages/compiler/src/parser/index.ts b/packages/compiler/src/parser/index.ts index ea1f4d7d0..689d00a4d 100644 --- a/packages/compiler/src/parser/index.ts +++ b/packages/compiler/src/parser/index.ts @@ -57,7 +57,7 @@ export class Parser { code: `unclosed-block`, message: `Block was left open`, }, - current.start!, + current.start! + 1, ) } if (state !== fragment) { diff --git a/packages/compiler/src/parser/interfaces.ts b/packages/compiler/src/parser/interfaces.ts index 0e4d9741b..a3711b07c 100644 --- a/packages/compiler/src/parser/interfaces.ts +++ b/packages/compiler/src/parser/interfaces.ts @@ -22,7 +22,7 @@ export type Fragment = BaseNode & { export type Config = BaseNode & { type: 'Config' - value: Record + value: string } export type Text = BaseNode & { diff --git a/packages/compiler/src/parser/state/config.ts b/packages/compiler/src/parser/state/config.ts index 3a4e5f8f4..062d6d532 100644 --- a/packages/compiler/src/parser/state/config.ts +++ b/packages/compiler/src/parser/state/config.ts @@ -1,7 +1,5 @@ -import PARSER_ERRORS from '$compiler/error/errors' import { Parser } from '$compiler/parser' import type { Config } from '$compiler/parser/interfaces' -import yaml from 'yaml' export function config(parser: Parser) { const start = parser.index @@ -14,19 +12,12 @@ export function config(parser: Parser) { parser.eat('---', true) parser.eat('\n') - let parsedData - try { - parsedData = yaml.parse(data) - } catch (error) { - parser.error(PARSER_ERRORS.invalidConfig((error as Error).message), start) - } - const node = { start, end: parser.index, type: 'Config', raw: data, - value: parsedData, + value: data, } as Config parser.current().children!.push(node) diff --git a/packages/compiler/src/parser/state/fragment.ts b/packages/compiler/src/parser/state/fragment.ts index 9b5d83fa6..8a012b64c 100644 --- a/packages/compiler/src/parser/state/fragment.ts +++ b/packages/compiler/src/parser/state/fragment.ts @@ -18,11 +18,7 @@ export default function fragment(parser: Parser): (parser: Parser) => void { return multiLineComment } if (parser.match('---')) { - // Only parse config if it's the first thing in the file - const isFirst = parser.template.slice(0, parser.index).trim() === '' // Ignore any whitespace - if (isFirst) { - return config - } + return config } return text diff --git a/packages/compiler/src/parser/state/text.ts b/packages/compiler/src/parser/state/text.ts index 39c4ed076..0c80cc585 100644 --- a/packages/compiler/src/parser/state/text.ts +++ b/packages/compiler/src/parser/state/text.ts @@ -14,9 +14,7 @@ export function text(parser: Parser) { if (isEscaping) data = data.slice(0, -1) // Remove the escape character if (!isEscaping && parser.match('---')) { - // Only parse config if it's the first thing in the file - const isFirst = parser.template.slice(0, parser.index).trim() === '' // Ignore any whitespace - if (isFirst) break + break } if ( diff --git a/packages/compiler/src/types/index.ts b/packages/compiler/src/types/index.ts index 8dbfcfa21..5e139a442 100644 --- a/packages/compiler/src/types/index.ts +++ b/packages/compiler/src/types/index.ts @@ -14,6 +14,7 @@ export type ConversationMetadata = { config: Config errors: CompileError[] parameters: Set // Variables used in the prompt that have not been defined in runtime + setConfig: (config: Config) => string } export * from './message' diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index f679c2309..72a35bc7d 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -1,3 +1,7 @@ +import { z } from 'zod' + +import { ProviderApiKey } from './browser' + export function objectToString(object: any) { try { return JSON.stringify(object) @@ -5,3 +9,25 @@ export function objectToString(object: any) { return 'Error: Provider returned an object that could not be stringified' } } + +export function promptConfigSchema({ + providers, +}: { + providers: ProviderApiKey[] +}) { + const providerNames = providers.map((provider) => provider.name) + + return z.object({ + provider: z + .string({ + required_error: `You must select a provider.\nFor example: 'provider: ${providerNames[0] ?? ''}'`, + }) + .refine((p) => providers.find((provider) => provider.name === p), { + message: `Provider not available. You must use one of the following:\n${providerNames.map((p) => `'${p}'`).join(', ')}`, + }), + model: z.string({ + required_error: `You must select the model.\nFor example: 'model: 'gpt-4o'`, + }), + temperature: z.number().min(0).max(2).optional(), + }) +} diff --git a/packages/core/src/lib/documentSchema.ts b/packages/core/src/lib/documentSchema.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts b/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts index e3fb46bd6..7f2181cde 100644 --- a/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts +++ b/packages/core/src/repositories/commitsRepository/getChangedDocumentsInDraft.test.ts @@ -6,12 +6,14 @@ import { DocumentVersion, ModifiedDocumentType, Project, + Providers, } from '../../browser' import { destroyDocument, updateDocument } from '../../services/documents' import { createDocumentVersion, createDraft, createProject, + helpers, } from '../../tests/factories' let project: Project @@ -27,11 +29,15 @@ describe('publishDraftCommit', () => { user, documents: docs, } = await createProject({ + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { folder1: { - doc1: 'content1', + doc1: helpers.createPrompt({ + provider: 'openai', + content: 'content1', + }), }, - doc2: 'content2', + doc2: helpers.createPrompt({ provider: 'openai', content: 'content2' }), }, }) project = prj @@ -61,7 +67,10 @@ describe('publishDraftCommit', () => { it('show changed documents', async () => { await updateDocument({ document: documents['folder1/doc1']!, - content: 'content1.1', + content: helpers.createPrompt({ + provider: 'openai', + content: 'content1.1', + }), commit: draftCommit, }).then((r) => r.unwrap()) @@ -83,7 +92,10 @@ describe('publishDraftCommit', () => { const { documentVersion: newDoc } = await createDocumentVersion({ commit: draftCommit, path: 'folder1/doc3', - content: 'content3', + content: helpers.createPrompt({ + provider: 'openai', + content: 'content3', + }), }) const changes = await repo @@ -123,12 +135,18 @@ describe('publishDraftCommit', () => { it('show documents with number of errors sorted by errors', async () => { await updateDocument({ document: documents['folder1/doc1']!, - content: 'Content doc1 changed', + content: helpers.createPrompt({ + provider: 'openai', + content: 'Content doc1 changed', + }), commit: draftCommit, }).then((r) => r.unwrap()) await updateDocument({ document: documents['doc2']!, - content: 'WRONG', + content: helpers.createPrompt({ + provider: 'openai', + content: 'WRONG', + }), commit: draftCommit, }).then((r) => r.unwrap()) diff --git a/packages/core/src/repositories/commitsRepository/index.test.ts b/packages/core/src/repositories/commitsRepository/index.test.ts index 92fe68cdf..5d1f54bad 100644 --- a/packages/core/src/repositories/commitsRepository/index.test.ts +++ b/packages/core/src/repositories/commitsRepository/index.test.ts @@ -1,31 +1,44 @@ import { beforeEach, describe, expect, it } from 'vitest' -import type { Project, User } from '../../browser' +import type { Project, ProviderApiKey, User } from '../../browser' import { CommitStatus } from '../../constants' import { mergeCommit } from '../../services/commits' import { createNewDocument } from '../../services/documents' import * as factories from '../../tests/factories' import { CommitsRepository } from './index' -async function createDraftsCommits(project: Project, user: User) { +async function createDraftsCommits( + project: Project, + user: User, + provider: ProviderApiKey, +) { const results = [] for (let i = 0; i < 10; i++) { const draft = await factories.createDraft({ project, user }) await createNewDocument({ commit: draft.commit, path: `${i}/foo`, - content: 'foo', + content: factories.helpers.createPrompt({ provider }), }) results.push(draft) } return results } + let project: Project let repository: CommitsRepository describe('Commits by project', () => { beforeEach(async () => { - let { project: firstProject, user } = await factories.createProject() - const drafsCommits = await createDraftsCommits(firstProject, user) + let { + project: firstProject, + user, + providers, + } = await factories.createProject() + const drafsCommits = await createDraftsCommits( + firstProject, + user, + providers[0]!, + ) await Promise.all([ mergeCommit(drafsCommits[0]!.commit), mergeCommit(drafsCommits[1]!.commit), diff --git a/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts b/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts index a48debe0c..87e738b1a 100644 --- a/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts +++ b/packages/core/src/repositories/documentLogsRepository/getDocumentLogsWithMetadata.test.ts @@ -1,32 +1,21 @@ import { describe, expect, it } from 'vitest' -import { Providers } from '../../constants' import { mergeCommit } from '../../services/commits' import { updateDocument } from '../../services/documents' import * as factories from '../../tests/factories' import { DocumentLogsRepository } from './index' -const documentContent = (content: string) => ` ---- -provider: foo -model: bar ---- -${content} -` - describe('getDocumentLogsWithMetadata', () => { it('return all logs from merged commits', async () => { - const { workspace, project, user } = await factories.createProject() - await factories.createProviderApiKey({ - workspace, - user, - name: 'foo', - type: Providers.OpenAI, - }) + const { project, user, providers } = await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit: commit1, - content: documentContent('VERSION_1'), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_1', + }), }) await mergeCommit(commit1).then((r) => r.unwrap()) @@ -34,7 +23,10 @@ describe('getDocumentLogsWithMetadata', () => { await updateDocument({ commit: commit2, document: doc, - content: documentContent('VERSION_2'), + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_2', + }), }) await mergeCommit(commit2).then((r) => r.unwrap()) @@ -59,17 +51,15 @@ describe('getDocumentLogsWithMetadata', () => { }) it('includes logs from specified draft', async () => { - const { workspace, project, user } = await factories.createProject() - await factories.createProviderApiKey({ - workspace, - user, - name: 'foo', - type: Providers.OpenAI, - }) + const { project, user, providers } = await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit: commit1, - content: documentContent('VERSION_1'), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_1', + }), }) await mergeCommit(commit1).then((r) => r.unwrap()) @@ -77,7 +67,11 @@ describe('getDocumentLogsWithMetadata', () => { await updateDocument({ commit: commit2, document: doc, - content: documentContent('VERSION_2'), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_2', + }), }) await mergeCommit(commit2).then((r) => r.unwrap()) @@ -85,7 +79,10 @@ describe('getDocumentLogsWithMetadata', () => { await updateDocument({ commit: draft, document: doc, - content: documentContent('VERSION_3'), + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_3', + }), }) const { documentLog: log1 } = await factories.createDocumentLog({ @@ -115,17 +112,15 @@ describe('getDocumentLogsWithMetadata', () => { }) it('does not include logs from non-specified drafts', async () => { - const { workspace, project, user } = await factories.createProject() - await factories.createProviderApiKey({ - workspace, - user, - name: 'foo', - type: Providers.OpenAI, - }) + const { project, user, providers } = await factories.createProject() const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit: commit1, - content: documentContent('VERSION_1'), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_1', + }), }) await mergeCommit(commit1).then((r) => r.unwrap()) @@ -133,14 +128,20 @@ describe('getDocumentLogsWithMetadata', () => { await updateDocument({ commit: draft1, document: doc, - content: documentContent('VERSION_2'), + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_2', + }), }) const { commit: draft2 } = await factories.createDraft({ project, user }) await updateDocument({ commit: draft2, document: doc, - content: documentContent('VERSION_3'), + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_3', + }), }) const { documentLog: log1 } = await factories.createDocumentLog({ @@ -170,17 +171,15 @@ describe('getDocumentLogsWithMetadata', () => { }) it('returns a sum of tokens and cost', async () => { - const { workspace, project, user } = await factories.createProject() - await factories.createProviderApiKey({ - workspace, - user, - name: 'foo', - type: Providers.OpenAI, - }) + const { project, user, providers } = await factories.createProject() const { commit } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit, - content: documentContent('\n'), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: '\n', + }), }) await mergeCommit(commit).then((r) => r.unwrap()) diff --git a/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts b/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts index abbf412a4..bc0c79406 100644 --- a/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts +++ b/packages/core/src/repositories/documentVersionsRepository/getDocumentAtCommit.test.ts @@ -8,11 +8,15 @@ import { DocumentVersionsRepository } from './index' describe('getDocumentAtCommit', () => { it('return doc from merged commit', async () => { - const { project, user } = await factories.createProject() + const { project, user, providers } = await factories.createProject() const { commit } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit: commit, - content: 'VERSION_1', + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_1', + }), }) const mergedCommit = await mergeCommit(commit).then((r) => r.unwrap()) const documentUuid = doc.documentUuid @@ -25,11 +29,11 @@ describe('getDocumentAtCommit', () => { }) const document = result.unwrap() - expect(omit(document, 'id', 'updatedAt')).toEqual({ - ...omit(doc, 'id', 'updatedAt'), + expect(omit(document, 'id', 'updatedAt', 'resolvedContent')).toEqual({ + ...omit(doc, 'id', 'updatedAt', 'resolvedContent'), projectId: project.id, mergedAt: mergedCommit.mergedAt, - resolvedContent: 'VERSION_1', }) + expect(document.resolvedContent).toEqual(doc.content) }) }) diff --git a/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts b/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts index 73b4b9590..09e51de17 100644 --- a/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts +++ b/packages/core/src/repositories/documentVersionsRepository/getDocumentsAtCommit.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it } from 'vitest' import type { Commit, DocumentVersion, Project } from '../../browser' -import { HEAD_COMMIT } from '../../constants' +import { HEAD_COMMIT, Providers } from '../../constants' import { mergeCommit } from '../../services/commits' import { updateDocument } from '../../services/documents' import * as factories from '../../tests/factories' @@ -25,7 +25,10 @@ describe('getDocumentsAtCommit', () => { commit, documents: allDocs, } = await ctx.factories.createProject({ - documents: { doc1: 'Doc 1' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: ctx.factories.helpers.createPrompt({ provider: 'openai' }), + }, }) const documentsScope = new DocumentVersionsRepository(project.workspaceId) @@ -39,7 +42,17 @@ describe('getDocumentsAtCommit', () => { it('get docs from HEAD without soft deleted', async (ctx) => { const { commit, project, documents, user } = await ctx.factories.createProject({ - documents: { doc1: 'Doc 1', doc2: 'Doc 2' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1', + }), + doc2: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2', + }), + }, }) const documentsScope = new DocumentVersionsRepository(project.workspaceId) const { commit: draft } = await factories.createDraft({ project, user }) @@ -51,12 +64,13 @@ describe('getDocumentsAtCommit', () => { .getDocumentsAtCommit(draft) .then((r) => r.unwrap()) const contents = filteredDocs.map((d) => d.content) - expect(contents).toEqual(['Doc 1']) + expect(contents.length).toBe(1) + expect(contents[0]).toContain('Doc 1') }) describe('documents for each commit', () => { - beforeEach(async () => { - const { project, user } = await factories.createProject() + beforeEach(async (ctx) => { + const { project, user, providers } = await ctx.factories.createProject() const documentsScope = new DocumentVersionsRepository(project.workspaceId) const { commit: commit1 } = await factories.createDraft({ project, user }) const { commit: commit2 } = await factories.createDraft({ project, user }) @@ -65,7 +79,11 @@ describe('getDocumentsAtCommit', () => { // Initial document const { documentVersion: doc1 } = await factories.createDocumentVersion({ commit: commit1, - content: 'VERSION_1', + path: 'doc1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_1', + }), }) await mergeCommit(commit1).then((r) => r.unwrap()) @@ -73,7 +91,10 @@ describe('getDocumentsAtCommit', () => { const doc2 = await updateDocument({ commit: commit2, document: doc1!, - content: 'VERSION_2', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_2', + }), }).then((r) => r.unwrap()) await mergeCommit(commit2).then((r) => r.unwrap()) @@ -81,7 +102,10 @@ describe('getDocumentsAtCommit', () => { const doc3 = await updateDocument({ commit: commit3, document: doc1!, - content: 'VERSION_3_draft', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'VERSION_3_draft', + }), }).then((r) => r.unwrap()) documentsByContent = { @@ -104,7 +128,7 @@ describe('getDocumentsAtCommit', () => { .then((r) => r.unwrap()) expect(documents.length).toBe(1) - expect(documents[0]!.content).toBe('VERSION_1') + expect(documents[0]!.content).toContain('VERSION_1') }) it('get docs from version 2', async () => { @@ -115,7 +139,7 @@ describe('getDocumentsAtCommit', () => { .then((r) => r.unwrap()) expect(documents.length).toBe(1) - expect(documents[0]!.content).toBe('VERSION_2') + expect(documents[0]!.content).toContain('VERSION_2') }) it('get docs from version 3', async () => { @@ -126,7 +150,7 @@ describe('getDocumentsAtCommit', () => { .then((r) => r.unwrap()) expect(documents.length).toBe(1) - expect(documents[0]!.content).toBe('VERSION_3_draft') + expect(documents[0]!.content).toContain('VERSION_3_draft') }) it('get docs from HEAD', async () => { @@ -144,20 +168,24 @@ describe('getDocumentsAtCommit', () => { .then((r) => r.unwrap()) expect(documents.length).toBe(1) - expect(documents[0]!.content).toBe('VERSION_2') + expect(documents[0]!.content).toContain('VERSION_2') }) }) describe('documents from previous commits', () => { - beforeEach(async () => { - const { project, user } = await factories.createProject() + beforeEach(async (ctx) => { + const { project, user, providers } = await ctx.factories.createProject() const documentsScope = new DocumentVersionsRepository(project.workspaceId) // Doc 1 const { commit: commit1 } = await factories.createDraft({ project, user }) const { documentVersion: doc1 } = await factories.createDocumentVersion({ commit: commit1, - content: 'Doc_1_commit_1', + path: 'doc1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc_1_commit_1', + }), }) const mergedCommit1 = await mergeCommit(commit1).then((r) => r.unwrap()) @@ -165,7 +193,11 @@ describe('getDocumentsAtCommit', () => { const { commit: commit2 } = await factories.createDraft({ project, user }) const { documentVersion: doc2 } = await factories.createDocumentVersion({ commit: commit2, - content: 'Doc_2_commit_2', + path: 'doc2', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc_2_commit_2', + }), }) const mergedCommit2 = await mergeCommit(commit2).then((r) => r.unwrap()) @@ -174,7 +206,10 @@ describe('getDocumentsAtCommit', () => { const doc3 = await updateDocument({ commit: commit3, document: doc2, - content: 'Doc_2_commit_3_draft', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc_2_commit_3_draft', + }), }).then((r) => r.unwrap()) documentsByContent = { @@ -200,8 +235,10 @@ describe('getDocumentsAtCommit', () => { .getDocumentsAtCommit(commit) .then((r) => r.unwrap()) - const contents = documents.map((d) => d.content) - expect(contents).toEqual(['Doc_1_commit_1']) + const contents = documents + .sort((a, b) => a.path.localeCompare(b.path)) + .map((d) => d.content) + expect(contents[0]).toContain('Doc_1_commit_1') }) it('get docs from commit 2', async () => { @@ -210,8 +247,11 @@ describe('getDocumentsAtCommit', () => { .getDocumentsAtCommit(commit) .then((r) => r.unwrap()) - const contents = documents.map((d) => d.content).sort() - expect(contents).toEqual(['Doc_1_commit_1', 'Doc_2_commit_2']) + const contents = documents + .sort((a, b) => a.path.localeCompare(b.path)) + .map((d) => d.content) + expect(contents[0]).toContain('Doc_1_commit_1') + expect(contents[1]).toContain('Doc_2_commit_2') }) it('get docs from commit 3', async () => { @@ -220,8 +260,11 @@ describe('getDocumentsAtCommit', () => { .getDocumentsAtCommit(commit) .then((r) => r.unwrap()) - const contents = documents.map((d) => d.content).sort() - expect(contents).toEqual(['Doc_1_commit_1', 'Doc_2_commit_3_draft']) + const contents = documents + .sort((a, b) => a.path.localeCompare(b.path)) + .map((d) => d.content) + expect(contents[0]).toContain('Doc_1_commit_1') + expect(contents[1]).toContain('Doc_2_commit_3_draft') }) it('get docs from HEAD', async () => { @@ -237,8 +280,11 @@ describe('getDocumentsAtCommit', () => { .getDocumentsAtCommit(commit) .then((r) => r.unwrap()) - const contents = documents.map((d) => d.content).sort() - expect(contents).toEqual(['Doc_1_commit_1', 'Doc_2_commit_2']) + const contents = documents + .sort((a, b) => a.path.localeCompare(b.path)) + .map((d) => d.content) + expect(contents[0]).toContain('Doc_1_commit_1') + expect(contents[1]).toContain('Doc_2_commit_2') }) }) }) diff --git a/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts b/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts index 0e9ce153f..aa18627bf 100644 --- a/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts +++ b/packages/core/src/repositories/evaluationResultsRepository/findByDocumentUuid.test.ts @@ -1,22 +1,17 @@ import { describe, expect, it } from 'vitest' import { EvaluationResultsRepository } from '.' -import { EvaluationResultableType, Providers } from '../../constants' +import { EvaluationResultableType } from '../../constants' import { mergeCommit } from '../../services/commits' import * as factories from '../../tests/factories' describe('findEvaluationResultsByDocumentUuid', () => { it('return evaluation results', async () => { - const { workspace, project, user } = await factories.createProject() - const provider = await factories.createProviderApiKey({ - workspace, - type: Providers.OpenAI, - name: 'openai', - user, - }) + const { workspace, project, user, providers } = + await factories.createProject() const evaluation = await factories.createLlmAsJudgeEvaluation({ workspace, - prompt: factories.helpers.createPrompt({ provider }), + prompt: factories.helpers.createPrompt({ provider: providers[0]! }), configuration: { type: EvaluationResultableType.Number, detail: { @@ -28,7 +23,8 @@ describe('findEvaluationResultsByDocumentUuid', () => { const { commit: draft } = await factories.createDraft({ project, user }) const { documentVersion: doc } = await factories.createDocumentVersion({ commit: draft, - content: factories.helpers.createPrompt({ provider }), + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ provider: providers[0]! }), }) const commit = await mergeCommit(draft).then((r) => r.unwrap()) diff --git a/packages/core/src/services/commits/merge.test.ts b/packages/core/src/services/commits/merge.test.ts index 09cd93168..ba9d14d97 100644 --- a/packages/core/src/services/commits/merge.test.ts +++ b/packages/core/src/services/commits/merge.test.ts @@ -2,6 +2,7 @@ import { eq } from 'drizzle-orm' import { describe, expect, it } from 'vitest' import { database } from '../../client' +import { Providers } from '../../constants' import { findHeadCommit } from '../../data-access/commits' import { documentVersions } from '../../schema' import { createNewDocument, updateDocument } from '../documents' @@ -9,13 +10,13 @@ import { mergeCommit } from './merge' describe('mergeCommit', () => { it('merges a commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, providers } = await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ commit, path: 'foo', - content: 'foo', + content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), }) const mergedCommit = await mergeCommit(commit).then((r) => r.unwrap()) @@ -35,13 +36,13 @@ describe('mergeCommit', () => { }) it('recomputes all changes in the commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, providers } = await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ commit, path: 'foo', - content: 'foo', + content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), }) const currentChanges = await database.query.documentVersions.findMany({ @@ -60,17 +61,20 @@ describe('mergeCommit', () => { expect(mergedChanges.length).toBe(1) expect(mergedChanges[0]!.path).toBe('foo') - expect(mergedChanges[0]!.resolvedContent).toBe('foo') + expect(mergedChanges[0]!.resolvedContent).toBeDefined() }) it('fails when trying to merge a commit with syntax errors', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, providers } = await ctx.factories.createProject() const { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ commit, path: 'foo', - content: ' { }) it('detects with a new document version does not actually change anything', async (ctx) => { + const originalContent = ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'foo', + }) + const { project, user, documents } = await ctx.factories.createProject({ + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { - foo: 'foo', + foo: originalContent, }, }) @@ -95,13 +105,16 @@ describe('mergeCommit', () => { await updateDocument({ commit, document: documents[0]!, - content: 'bar', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'bar', + }), }) await updateDocument({ commit, document: documents[0]!, - content: 'foo', // back to the original content + content: originalContent, // back to the original content }) const res = await mergeCommit(commit) @@ -109,7 +122,7 @@ describe('mergeCommit', () => { }) it('increases the version number of the commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, providers } = await ctx.factories.createProject() const { commit: commit1 } = await ctx.factories.createDraft({ project, user, @@ -118,7 +131,10 @@ describe('mergeCommit', () => { const doc = await createNewDocument({ commit: commit1, path: 'foo1', - content: 'foo1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'foo1', + }), }).then((r) => r.unwrap()) const mergedCommit1 = await mergeCommit(commit1).then((r) => r.unwrap()) @@ -132,7 +148,10 @@ describe('mergeCommit', () => { await updateDocument({ document: doc, commit: commit2, - content: 'foo2', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'foo2', + }), }).then((r) => r.unwrap()) const mergedCommit2 = await mergeCommit(commit2).then((r) => r.unwrap()) diff --git a/packages/core/src/services/commits/runDocumentAtCommit.test.ts b/packages/core/src/services/commits/runDocumentAtCommit.test.ts index 7eda4a7ad..502bd08b9 100644 --- a/packages/core/src/services/commits/runDocumentAtCommit.test.ts +++ b/packages/core/src/services/commits/runDocumentAtCommit.test.ts @@ -1,3 +1,4 @@ +import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' import { @@ -6,11 +7,12 @@ import { LogSources, ProviderApiKey, Providers, - User, Workspace, } from '../../browser' +import { database } from '../../client' import { publisher } from '../../events/publisher' -import { createProject, createProviderApiKey } from '../../tests/factories' +import { providerApiKeys } from '../../schema' +import { createProject } from '../../tests/factories' import { testConsumeStream } from '../../tests/helpers' import { runDocumentAtCommit } from './index' @@ -30,9 +32,15 @@ describe('runDocumentAtCommit', () => { }) it('fails if provider api key is not found', async () => { - const { workspace, document, commit } = await buildData({ + const { workspace, document, commit, provider } = await buildData({ doc1Content: dummyDoc1Content, }) + + await database + .update(providerApiKeys) + .set({ name: 'another-name' }) + .where(eq(providerApiKeys.id, provider.id)) + const result = await runDocumentAtCommit({ workspace, document, @@ -49,26 +57,20 @@ describe('runDocumentAtCommit', () => { beforeEach(async () => { const { workspace: wsp, - user: usr, document: doc, commit: cmt, + provider: prv, } = await buildData({ doc1Content: dummyDoc1Content, }) - user = usr document = doc commit = cmt workspace = wsp - provider = await createProviderApiKey({ - workspace, - type: Providers.OpenAI, - name: 'openai', - user, - }) + provider = prv }) it('fails if document is not found in commit', async () => { - const { document } = await buildData() + const { document } = await buildData({ doc1Content: dummyDoc1Content }) const result = await runDocumentAtCommit({ workspace, document, @@ -261,23 +263,26 @@ This is a test document ` -async function buildData({ doc1Content = '' }: { doc1Content?: string } = {}) { - const { workspace, documents, commit, user } = await createProject({ - documents: { - doc1: doc1Content, +async function buildData({ doc1Content }: { doc1Content: string }) { + const { workspace, documents, commit, user, providers } = await createProject( + { + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: doc1Content, + }, }, - }) + ) return { workspace, document: documents[0]!, commit, user, + provider: providers[0]!, } } let document: DocumentVersion let commit: Commit let workspace: Workspace -let user: User let provider: ProviderApiKey diff --git a/packages/core/src/services/documentLogs/addMessages/index.test.ts b/packages/core/src/services/documentLogs/addMessages/index.test.ts index 2668f40c3..f9ee12f01 100644 --- a/packages/core/src/services/documentLogs/addMessages/index.test.ts +++ b/packages/core/src/services/documentLogs/addMessages/index.test.ts @@ -12,11 +12,7 @@ import { User, Workspace, } from '../../../browser' -import { - createDocumentLog, - createProject, - createProviderApiKey, -} from '../../../tests/factories' +import { createDocumentLog, createProject } from '../../../tests/factories' import { testConsumeStream } from '../../../tests/helpers' import { addMessages } from './index' @@ -84,7 +80,9 @@ async function buildData({ documents, commit: cmt, user: usr, + providers, } = await createProject({ + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { doc1: doc1Content, }, @@ -93,12 +91,7 @@ async function buildData({ commit = cmt workspace = wsp user = usr - providerApiKey = await createProviderApiKey({ - workspace, - type: Providers.OpenAI, - name: 'openai', - user, - }) + providerApiKey = providers[0]! const { providerLogs } = await createDocumentLog({ commit, document, diff --git a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts index 2d407f5e8..86a02dafd 100644 --- a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts @@ -1,43 +1,26 @@ import { beforeEach, describe, expect, it } from 'vitest' -import { - DocumentLog, - ProviderApiKey, - ProviderLog, - Providers, - Workspace, -} from '../../browser' +import { DocumentLog, ProviderApiKey, ProviderLog } from '../../browser' import { NotFoundError } from '../../lib' import * as factories from '../../tests/factories' import { computeDocumentLogWithMetadata } from './computeDocumentLogWithMetadata' describe('computeDocumentLogWithMetadata', () => { let documentLog: DocumentLog - let workspace: Workspace let provider: ProviderApiKey let providerLogs: ProviderLog[] beforeEach(async () => { const setup = await factories.createProject() - workspace = setup.workspace - provider = await factories.createProviderApiKey({ - workspace, - type: Providers.OpenAI, - name: 'foo', - user: setup.user, - }) + provider = setup.providers[0]! const { commit } = await factories.createDraft({ project: setup.project, user: setup.user, }) const { documentVersion } = await factories.createDocumentVersion({ commit, - content: ` - --- - provider: ${provider.name} - model: 'gpt-4o-mini' - --- - `, + path: 'folder1/doc1', + content: factories.helpers.createPrompt({ provider, content: 'Doc 1' }), }) const { documentLog: dl, providerLogs: _providerLogs } = await factories.createDocumentLog({ diff --git a/packages/core/src/services/documents/create.test.ts b/packages/core/src/services/documents/create.test.ts index d84b6ac83..1905dcf05 100644 --- a/packages/core/src/services/documents/create.test.ts +++ b/packages/core/src/services/documents/create.test.ts @@ -46,31 +46,12 @@ describe('createNewDocument', () => { }) it('fails when trying to create a document in a merged commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() - let { commit } = await ctx.factories.createDraft({ project, user }) - await createNewDocument({ - commit, - path: 'foo', - content: 'foo', - }) - commit = await mergeCommit(commit).then((r) => r.unwrap()) - - const result = await createNewDocument({ - commit, - path: 'foo', - }) - - expect(result.ok).toBe(false) - expect(result.error!.message).toBe('Cannot modify a merged commit') - }) - - it('fails when trying to create a document in a merged commit', async (ctx) => { - const { project, user } = await ctx.factories.createProject() + const { project, user, providers } = await ctx.factories.createProject() let { commit } = await ctx.factories.createDraft({ project, user }) await createNewDocument({ commit, path: 'foo', - content: 'foo', + content: ctx.factories.helpers.createPrompt({ provider: providers[0]! }), }) commit = await mergeCommit(commit).then((r) => r.unwrap()) diff --git a/packages/core/src/services/documents/destroyFolder.test.ts b/packages/core/src/services/documents/destroyFolder.test.ts index 54195d26f..60cf50602 100644 --- a/packages/core/src/services/documents/destroyFolder.test.ts +++ b/packages/core/src/services/documents/destroyFolder.test.ts @@ -2,6 +2,7 @@ import { and, eq } from 'drizzle-orm' import { describe, expect, it } from 'vitest' import { database } from '../../client' +import { Providers } from '../../constants' import { NotFoundError } from '../../lib' import { documentVersions } from '../../schema' import * as factories from '../../tests/factories' @@ -23,13 +24,16 @@ describe('removing folders', () => { expect(result.error).toEqual(new NotFoundError('Folder does not exist')) }) - it('throws error if commit is merged', async () => { - const { project, user } = await factories.createProject() + it('throws error if commit is merged', async (ctx) => { + const { project, user, providers } = await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) await createNewDocument({ commit: draft, path: 'foo', - content: 'foo', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'foo', + }), }) const mergedCommit = await mergeCommit(draft).then((r) => r.unwrap()) @@ -42,32 +46,56 @@ describe('removing folders', () => { expect(result.error).toEqual(new Error('Cannot modify a merged commit')) }) - it('destroy folder that were in draft document but not in previous merged commits', async () => { - const { project, user } = await factories.createProject() + it('destroy folder that were in draft document but not in previous merged commits', async (ctx) => { + const { project, user, providers } = await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) await factories.createDocumentVersion({ commit: draft, path: 'root-folder/some-folder/doc1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 1', + }), }) await factories.createDocumentVersion({ commit: draft, path: 'root-folder/some-folder/doc2', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 2', + }), }) await factories.createDocumentVersion({ commit: draft, path: 'root-folder/some-folder/inner-folder/doc42', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 42', + }), }) await factories.createDocumentVersion({ commit: draft, path: 'root-folder/other-nested-folder/doc3', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 3', + }), }) await factories.createDocumentVersion({ commit: draft, path: 'root-folder/some-foldernoisadoc', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 3', + }), }) await factories.createDocumentVersion({ commit: draft, path: 'other-foler/doc4', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 4', + }), }) await destroyFolder({ @@ -89,12 +117,19 @@ describe('removing folders', () => { ]) }) - it('create soft deleted documents that were present in merged commits and were deleted in this draft commit', async () => { + it('create soft deleted documents that were present in merged commits and were deleted in this draft commit', async (ctx) => { const { project, user } = await factories.createProject({ + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { 'some-folder': { - doc2: 'Doc 2', - doc1: 'Doc 1', + doc2: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2', + }), + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1', + }), }, }, }) @@ -118,12 +153,19 @@ describe('removing folders', () => { expect(paths).toEqual(['some-folder/doc1', 'some-folder/doc2']) }) - it('existing documents in this commit draft are marked as deleted', async () => { + it('existing documents in this commit draft are marked as deleted', async (ctx) => { const { project, user, documents } = await factories.createProject({ + providers: [{ type: Providers.OpenAI, name: 'openai' }], documents: { 'some-folder': { - doc2: 'Doc 2', - doc1: 'Doc 1', + doc2: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2', + }), + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1', + }), }, }, }) diff --git a/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts b/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts index 33bf258b2..66708775f 100644 --- a/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts +++ b/packages/core/src/services/documents/destroyOrSoftDeleteDocuments.test.ts @@ -2,19 +2,23 @@ import { and, eq } from 'drizzle-orm' import { describe, expect, it } from 'vitest' import { database } from '../../client' +import { Providers } from '../../constants' import { documentVersions } from '../../schema' import * as factories from '../../tests/factories' import { destroyOrSoftDeleteDocuments } from './destroyOrSoftDeleteDocuments' import { updateDocument } from './update' describe('destroyOrSoftDeleteDocuments', () => { - it('remove documents that were not present in merged commits', async () => { - const { project, user } = await factories.createProject() + it('remove documents that were not present in merged commits', async (ctx) => { + const { project, user, providers } = await factories.createProject() const { commit: draft } = await factories.createDraft({ project, user }) const { documentVersion: draftDocument } = await factories.createDocumentVersion({ commit: draft, path: 'doc1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + }), }) await destroyOrSoftDeleteDocuments({ @@ -29,13 +33,16 @@ describe('destroyOrSoftDeleteDocuments', () => { expect(documents.length).toBe(0) }) - it('mark as deleted documents that were present in merged commits and not in the draft commit', async () => { + it('mark as deleted documents that were present in merged commits and not in the draft commit', async (ctx) => { const { project, user, documents: allDocs, } = await factories.createProject({ - documents: { doc1: 'Doc 1' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: ctx.factories.helpers.createPrompt({ provider: 'openai' }), + }, }) const document = allDocs[0]! const { commit: draft } = await factories.createDraft({ project, user }) @@ -54,13 +61,16 @@ describe('destroyOrSoftDeleteDocuments', () => { expect(drafDocument!.deletedAt).not.toBe(null) }) - it('mark as deleted documents present in the draft commit', async () => { + it('mark as deleted documents present in the draft commit', async (ctx) => { const { project, user, documents: allDocs, } = await factories.createProject({ - documents: { doc1: 'Doc 1' }, + providers: [{ type: Providers.OpenAI, name: 'openai' }], + documents: { + doc1: ctx.factories.helpers.createPrompt({ provider: 'openai' }), + }, }) const document = allDocs[0]! const { commit: draft } = await factories.createDraft({ project, user }) diff --git a/packages/core/src/services/documents/recomputeChanges/index.ts b/packages/core/src/services/documents/recomputeChanges/index.ts index 9b72b50d3..3d2de3577 100644 --- a/packages/core/src/services/documents/recomputeChanges/index.ts +++ b/packages/core/src/services/documents/recomputeChanges/index.ts @@ -8,10 +8,16 @@ import { } from '@latitude-data/compiler' import { eq } from 'drizzle-orm' -import { Commit, DocumentVersion } from '../../../browser' +import { + Commit, + DocumentVersion, + promptConfigSchema, + ProviderApiKey, +} from '../../../browser' import { database } from '../../../client' import { Result, Transaction, TypedResult } from '../../../lib' import { assertCommitIsDraft } from '../../../lib/assertCommitIsDraft' +import { ProviderApiKeysRepository } from '../../../repositories' import { documentVersions } from '../../../schema' import { getHeadDocumentsAndDraftDocumentsForCommit } from './getHeadDocumentsAndDraftDocuments' import { getMergedAndDraftDocuments } from './getMergedAndDraftDocuments' @@ -19,9 +25,11 @@ import { getMergedAndDraftDocuments } from './getMergedAndDraftDocuments' async function resolveDocumentChanges({ originalDocuments, newDocuments, + providers, }: { originalDocuments: DocumentVersion[] newDocuments: DocumentVersion[] + providers: ProviderApiKey[] }): Promise<{ documents: DocumentVersion[] errors: Record @@ -43,12 +51,15 @@ async function resolveDocumentChanges({ } } + const configSchema = promptConfigSchema({ providers }) + const newDocumentsWithUpdatedHash = await Promise.all( newDocuments.map(async (d) => { const metadata = await readMetadata({ prompt: d.content ?? '', fullPath: d.path, referenceFn: getDocumentContent, + configSchema, }) if (metadata.errors.length > 0) { errors[d.documentUuid] = metadata.errors @@ -137,10 +148,15 @@ export async function recomputeChanges( documentsInDrafCommit, }) + const providersScope = new ProviderApiKeysRepository(workspaceId, tx) + const providersResult = await providersScope.findAll() + if (providersResult.error) return Result.error(providersResult.error) + const { documents: documentsToUpdate, errors } = await resolveDocumentChanges({ originalDocuments: mergedDocuments, newDocuments: draftDocuments, + providers: providersResult.value, }) const newDraftDocuments = ( diff --git a/packages/core/src/services/documents/update.test.ts b/packages/core/src/services/documents/update.test.ts index 27ba447b0..5f53eade6 100644 --- a/packages/core/src/services/documents/update.test.ts +++ b/packages/core/src/services/documents/update.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' +import { Providers } from '../../constants' import { CommitsRepository, DocumentVersionsRepository, @@ -11,8 +12,17 @@ describe('updateDocument', () => { it('modifies a document that was created in a previous commit', async (ctx) => { const { workspace, project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { - doc1: 'Doc 1 commit 1', + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1 commit 1', + }), }, }) @@ -22,7 +32,10 @@ describe('updateDocument', () => { await updateDocument({ commit, document: documents[0]!, - content: 'Doc 1 commit 2', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1 commit 2', + }), }).then((r) => r.unwrap()) await recomputeChanges({ draft: commit, workspaceId: workspace.id }) @@ -33,23 +46,30 @@ describe('updateDocument', () => { expect(changedDocuments.length).toBe(1) expect(changedDocuments[0]!.path).toBe('doc1') - expect(changedDocuments[0]!.content).toBe('Doc 1 commit 2') + expect(changedDocuments[0]!.content).toContain('Doc 1 commit 2') }) it('modifies a document that was created in the same commit', async (ctx) => { - const { workspace, project, user } = await ctx.factories.createProject() + const { workspace, project, user, providers } = + await ctx.factories.createProject() const docsScope = new DocumentVersionsRepository(project.workspaceId) const { commit } = await ctx.factories.createDraft({ project, user }) const { documentVersion: doc } = await ctx.factories.createDocumentVersion({ commit: commit, path: 'doc1', - content: 'Doc 1 v1', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 1 v1', + }), }) await updateDocument({ commit, document: doc, - content: 'Doc 1 v2', + content: ctx.factories.helpers.createPrompt({ + provider: providers[0]!, + content: 'Doc 1 v2', + }), }).then((r) => r.unwrap()) await recomputeChanges({ draft: commit, workspaceId: workspace.id }) @@ -60,17 +80,29 @@ describe('updateDocument', () => { expect(changedDocuments.length).toBe(1) expect(changedDocuments[0]!.path).toBe('doc1') - expect(changedDocuments[0]!.content).toBe('Doc 1 v2') + expect(changedDocuments[0]!.content).toContain('Doc 1 v2') }) it('modifying a document creates a change to all other documents that reference it', async (ctx) => { const { workspace, project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { referenced: { - doc: 'The document that is being referenced', + doc: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'The document that is being referenced', + }), }, - unmodified: '', + unmodified: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: '', + }), }, }) @@ -81,7 +113,10 @@ describe('updateDocument', () => { await updateDocument({ commit: draft, document: referencedDoc, - content: 'The document that is being referenced v2', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'The document that is being referenced v2', + }), }).then((r) => r.unwrap()) await recomputeChanges({ draft, workspaceId: workspace.id }) @@ -100,11 +135,23 @@ describe('updateDocument', () => { it('renaming a document creates a change to all other documents that reference it', async (ctx) => { const { workspace, project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { referenced: { - doc: 'The document that is being referenced', + doc: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'The document that is being referenced', + }), }, - main: '', + main: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: '', + }), }, }) const docsScope = new DocumentVersionsRepository(project.workspaceId) @@ -134,11 +181,23 @@ describe('updateDocument', () => { it('undoing a change to a document removes it from the list of changed documents', async (ctx) => { const { workspace, project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { referenced: { - doc: 'The document that is being referenced', + doc: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'The document that is being referenced', + }), }, - unmodified: '', + unmodified: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: '', + }), }, }) const docsScope = new DocumentVersionsRepository(project.workspaceId) @@ -149,7 +208,10 @@ describe('updateDocument', () => { await updateDocument({ commit, document: referencedDoc, - content: 'The document that is being referenced v2', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'The document that is being referenced v2', + }), }).then((r) => r.unwrap()) await recomputeChanges({ draft: commit, workspaceId: workspace.id }) @@ -181,9 +243,21 @@ describe('updateDocument', () => { it('fails when renaming a document with a path that already exists', async (ctx) => { const { project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { - doc1: 'Doc 1', - doc2: 'Doc 2', + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1', + }), + doc2: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2', + }), }, }) @@ -204,8 +278,17 @@ describe('updateDocument', () => { it('fails when trying to create a document in a merged commit', async (ctx) => { const { project, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { - foo: 'foo', + foo: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'foo', + }), }, }) const commitsScope = new CommitsRepository(project.workspaceId) @@ -218,7 +301,10 @@ describe('updateDocument', () => { const result = await updateDocument({ commit: commit!, document: fooDoc, - content: 'bar', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'bar', + }), }) expect(result.ok).toBe(false) @@ -228,9 +314,21 @@ describe('updateDocument', () => { it('invalidates the resolvedContent for all documents in the commit', async (ctx) => { const { workspace, project, user, documents } = await ctx.factories.createProject({ + providers: [ + { + type: Providers.OpenAI, + name: 'openai', + }, + ], documents: { - doc1: 'Doc 1', - doc2: 'Doc 2', + doc1: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1', + }), + doc2: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2', + }), }, }) const docsScope = new DocumentVersionsRepository(project.workspaceId) @@ -242,7 +340,10 @@ describe('updateDocument', () => { await updateDocument({ commit, document: doc1, - content: 'Doc 1 v2', + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 1 v2', + }), }).then((r) => r.unwrap()) await recomputeChanges({ draft: commit, workspaceId: workspace.id }) @@ -250,8 +351,11 @@ describe('updateDocument', () => { await updateDocument({ commit, document: doc2, - content: 'Doc 2 v2', - }) + content: ctx.factories.helpers.createPrompt({ + provider: 'openai', + content: 'Doc 2 v2', + }), + }).then((r) => r.unwrap()) const commitDocs = await docsScope .getDocumentsAtCommit(commit) diff --git a/packages/core/src/services/evaluationResults/create.test.ts b/packages/core/src/services/evaluationResults/create.test.ts index 89cd74667..caa5f4e1f 100644 --- a/packages/core/src/services/evaluationResults/create.test.ts +++ b/packages/core/src/services/evaluationResults/create.test.ts @@ -29,6 +29,7 @@ async function setupTest(configurationType: EvaluationResultableType) { const { commit } = await factories.createDraft({ project, user }) const { documentVersion } = await factories.createDocumentVersion({ commit, + path: 'folder1/doc1', content: ` --- provider: ${provider!.name} diff --git a/packages/core/src/tests/factories/documents.ts b/packages/core/src/tests/factories/documents.ts index 15f713140..454a97b06 100644 --- a/packages/core/src/tests/factories/documents.ts +++ b/packages/core/src/tests/factories/documents.ts @@ -1,4 +1,3 @@ -import { faker } from '@faker-js/faker' import { and, eq } from 'drizzle-orm' import { type Commit } from '../../browser' @@ -9,15 +8,8 @@ import { updateDocument } from '../../services/documents/update' export type IDocumentVersionData = { commit: Commit - path?: string - content?: string -} - -function makeRandomDocumentVersionData() { - return { - path: faker.commerce.productName().replace(/\s/g, '_').toLowerCase(), - content: faker.lorem.paragraphs().toLowerCase(), - } + path: string + content: string } export async function markAsSoftDelete( @@ -35,16 +27,7 @@ export async function markAsSoftDelete( ) } -export async function createDocumentVersion( - documentData: IDocumentVersionData, -) { - const randomData = makeRandomDocumentVersionData() - const data = { - ...randomData, - content: randomData.content, - ...documentData, - } - +export async function createDocumentVersion(data: IDocumentVersionData) { let result = await createNewDocument({ commit: data.commit, path: data.path, diff --git a/packages/core/src/tests/factories/helpers.ts b/packages/core/src/tests/factories/helpers.ts index aedc00398..90022ffaa 100644 --- a/packages/core/src/tests/factories/helpers.ts +++ b/packages/core/src/tests/factories/helpers.ts @@ -18,17 +18,20 @@ const randomSentence = () => { function createPrompt({ provider, model, + content, steps, }: { - provider: ProviderApiKey + provider: ProviderApiKey | string model?: string + content?: string steps?: number }) { const prompt = ` --- -provider: ${provider.name} +provider: ${typeof provider === 'string' ? provider : provider.name} model: ${model ?? faker.internet.domainName()} --- +${content ?? ''} ${Array.from({ length: steps ?? 1 }) .map(() => randomSentence()) .join('\n\n')} diff --git a/packages/core/src/tests/factories/projects.ts b/packages/core/src/tests/factories/projects.ts index b485dd9bd..b0d8f5dbb 100644 --- a/packages/core/src/tests/factories/projects.ts +++ b/packages/core/src/tests/factories/projects.ts @@ -1,6 +1,6 @@ import { faker } from '@faker-js/faker' -import type { DocumentVersion, Providers, User, Workspace } from '../../browser' +import { DocumentVersion, Providers, User, Workspace } from '../../browser' import { unsafelyGetUser } from '../../data-access' import { CommitsRepository } from '../../repositories' import { mergeCommit } from '../../services/commits' @@ -72,8 +72,12 @@ export async function createProject(projectData: Partial = {}) { const commitsScope = new CommitsRepository(workspace.id) let commit = (await commitsScope.getFirstCommitForProject(project)).unwrap() + const providersToCreate = + projectData.providers == undefined + ? [{ type: Providers.OpenAI, name: faker.internet.domainName() }] + : projectData.providers const providers = await Promise.all( - projectData.providers?.map(({ type, name }) => + providersToCreate.map(({ type, name }) => createProviderApiKey({ workspace, user, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f66e2e9b..7c796a939 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -565,6 +565,9 @@ importers: yaml: specifier: ^2.4.5 version: 2.5.1 + zod: + specifier: ^3.23.8 + version: 3.23.8 devDependencies: '@latitude-data/eslint-config': specifier: workspace:*