diff --git a/apps/gateway/package.json b/apps/gateway/package.json index ce8f4287d..15e567a0e 100644 --- a/apps/gateway/package.json +++ b/apps/gateway/package.json @@ -27,7 +27,6 @@ "@t3-oss/env-core": "^0.10.1", "drizzle-orm": "^0.33.0", "hono": "^4.5.3", - "jet-paths": "^1.0.6", "lodash-es": "^4.17.21", "zod": "^3.23.8", "rate-limiter-flexible": "^5.0.3" diff --git a/apps/gateway/src/common/messageSchema.ts b/apps/gateway/src/common/messageSchema.ts deleted file mode 100644 index 5e2e3c3c6..000000000 --- a/apps/gateway/src/common/messageSchema.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { ContentType, MessageRole } from '@latitude-data/compiler' -import { z } from 'zod' - -const userContentSchema = z.array( - z - .object({ - type: z.literal(ContentType.text), - text: z.string(), - }) - .or( - z.object({ - type: z.literal(ContentType.image), - image: z - .string() - .or(z.instanceof(Uint8Array)) - .or(z.instanceof(Buffer)) - .or(z.instanceof(ArrayBuffer)) - .or(z.instanceof(URL)), - }), - ), -) - -// TODO: We could unify the way this schema is build. -// We could have this zod schema as base in a common package and -// infer compiler's types from it -export const messageSchema = z - .object({ - role: z.literal(MessageRole.system), - content: z.string(), - }) - .or( - z.object({ - role: z.literal(MessageRole.user), - name: z.string().optional(), - content: userContentSchema, - }), - ) - .or( - z.object({ - role: z.literal(MessageRole.assistant), - content: z.string().or( - z.array( - z.object({ - type: z.literal(ContentType.toolCall), - toolCallId: z.string(), - toolName: z.string(), - args: z.record(z.any()), - }), - ), - ), - toolCalls: z.array( - z.object({ - id: z.string(), - name: z.string(), - arguments: z.record(z.any()), - }), - ), - }), - ) - .or( - z.object({ - role: z.literal(MessageRole.tool), - content: z.array( - z.object({ - type: z.literal(ContentType.toolResult), - toolCallId: z.string(), - toolName: z.string(), - result: z.string(), - isError: z.boolean().optional(), - }), - ), - }), - ) diff --git a/apps/gateway/src/common/routes.ts b/apps/gateway/src/common/routes.ts deleted file mode 100644 index 61b659ecf..000000000 --- a/apps/gateway/src/common/routes.ts +++ /dev/null @@ -1,32 +0,0 @@ -const ROUTES = { - Base: '', - Api: { - Base: '/api', - V1: { - Base: '/v1', - Conversations: { - Base: '/conversations', - Chat: '/:conversationUuid/chat', - }, - Documents: { - Base: '/projects/:projectId/versions/:versionUuid/documents', - Get: '/:documentPath{.+}', - Run: '/run', - }, - }, - V2: { - Base: '/v2', - Conversations: { - Base: '/conversations', - Chat: '/:conversationUuid/chat', - }, - Documents: { - Base: '/projects/:projectId/versions/:versionUuid/documents', - Get: '/:documentPath{.+}', - Run: '/run', - }, - }, - }, -} - -export default ROUTES diff --git a/apps/gateway/src/middlewares/errorHandler.ts b/apps/gateway/src/middlewares/errorHandler.ts index 4fa2ea6d2..e0d8fd69f 100644 --- a/apps/gateway/src/middlewares/errorHandler.ts +++ b/apps/gateway/src/middlewares/errorHandler.ts @@ -40,8 +40,8 @@ const errorHandlerMiddleware = (err: Error) => { if (err instanceof HTTPException) { return Response.json( { - name: ApiErrorCodes.HTTPExeption, - errorCode: ApiErrorCodes.HTTPExeption, + name: ApiErrorCodes.HTTPException, + errorCode: ApiErrorCodes.HTTPException, message: err.message, details: { cause: err.cause }, }, diff --git a/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts b/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts index dd5497b15..ef6c39af1 100644 --- a/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts +++ b/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/handlers/chat.ts @@ -1,9 +1,8 @@ import { zValidator } from '@hono/zod-validator' -import { LogSources } from '@latitude-data/core/browser' +import { LogSources, messagesSchema } from '@latitude-data/core/browser' import { streamToGenerator } from '@latitude-data/core/lib/streamToGenerator' import { addMessages } from '@latitude-data/core/services/documentLogs/index' import { captureException } from '@sentry/node' -import { messageSchema } from '$/common/messageSchema' import { Factory } from 'hono/factory' import { streamSSE } from 'hono/streaming' import { z } from 'zod' @@ -13,7 +12,7 @@ import { chainEventPresenter } from '../../../projects/[projectId]/versions/[ver const factory = new Factory() const schema = z.object({ - messages: z.array(messageSchema), + messages: messagesSchema, __internal: z .object({ source: z.nativeEnum(LogSources).optional(), diff --git a/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts b/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts index 0097fb7db..fd5d47fb1 100644 --- a/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts +++ b/apps/gateway/src/routes/api/v1/conversations/[conversationUuid]/index.ts @@ -1,8 +1,7 @@ -import ROUTES from '$/common/routes' import { Hono } from 'hono' import { chatHandler } from './handlers/chat' export const chatsRouter = new Hono() -chatsRouter.post(ROUTES.Api.V1.Conversations.Chat, ...chatHandler) +chatsRouter.post('/:conversationUuid/chat', ...chatHandler) diff --git a/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts b/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts index 7f0cc80a5..d7b0488ce 100644 --- a/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts +++ b/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/index.ts @@ -1,12 +1,13 @@ -import ROUTES from '$/common/routes' import { Hono } from 'hono' import { getHandler } from './handlers/get' import { runHandler } from './handlers/run' +import { logsRouterV1 } from './logs' const router = new Hono() -router.get(ROUTES.Api.V1.Documents.Get, ...getHandler) -router.post(ROUTES.Api.V1.Documents.Run, ...runHandler) +router.get('/:documentPath{.+}', ...getHandler) +router.post('/run', ...runHandler) +router.route('/logs', logsRouterV1) export { router as documentsRouter } diff --git a/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts b/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts new file mode 100644 index 000000000..c86c159b8 --- /dev/null +++ b/apps/gateway/src/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts @@ -0,0 +1,8 @@ +import { postHandler } from '$/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post' +import { Hono } from 'hono' + +const router = new Hono() + +router.post('/', ...postHandler) + +export { router as logsRouterV1 } diff --git a/apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts b/apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts index e36f7679c..1472edaa0 100644 --- a/apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts +++ b/apps/gateway/src/routes/api/v2/conversations/[conversationUuid]/index.ts @@ -1,7 +1,6 @@ -import ROUTES from '$/common/routes' import { chatHandler } from '$/routes/api/v1/conversations/[conversationUuid]/handlers/chat' import { Hono } from 'hono' export const chatsRouter = new Hono() -chatsRouter.post(ROUTES.Api.V2.Conversations.Chat, ...chatHandler) +chatsRouter.post('/:conversationUuid/chat', ...chatHandler) diff --git a/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts index c1c08c800..0ac69e9c6 100644 --- a/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts +++ b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/index.ts @@ -1,12 +1,13 @@ -import ROUTES from '$/common/routes' import { getHandler } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/get' import { Hono } from 'hono' import { runHandler } from './handlers/run' +import { logsRouterV2 } from './logs' const router = new Hono() -router.get(ROUTES.Api.V2.Documents.Get, ...getHandler) -router.post(ROUTES.Api.V2.Documents.Run, ...runHandler) +router.get('/:documentPath{.+}', ...getHandler) +router.post('/run', ...runHandler) +router.route('/logs', logsRouterV2) export { router as documentsRouter } diff --git a/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post.ts b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post.ts new file mode 100644 index 000000000..265b69c07 --- /dev/null +++ b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/handlers/post.ts @@ -0,0 +1,50 @@ +import { zValidator } from '@hono/zod-validator' +import { LogSources, messagesSchema } from '@latitude-data/core/browser' +import { generateUUIDIdentifier } from '@latitude-data/core/lib/generateUUID' +import { createDocumentLog } from '@latitude-data/core/services/documentLogs/create' +import { getData } from '$/routes/api/v1/projects/[projectId]/versions/[versionUuid]/documents/handlers/_shared' +import { Factory } from 'hono/factory' +import { z } from 'zod' + +const factory = new Factory() + +const schema = z.object({ + path: z.string(), + messages: messagesSchema, + response: z.string().optional(), +}) + +export const postHandler = factory.createHandlers( + zValidator('json', schema), + async (c) => { + const { projectId, versionUuid } = c.req.param() + const { path, messages, response } = c.req.valid('json') + const workspace = c.get('workspace') + const { document, commit } = await getData({ + workspace, + projectId: Number(projectId!), + commitUuid: versionUuid!, + documentPath: path!, + }).then((r) => r.unwrap()) + + const documentLog = await createDocumentLog({ + data: { + uuid: generateUUIDIdentifier(), + documentUuid: document.documentUuid, + resolvedContent: document.content, + source: LogSources.API, + parameters: {}, + providerLog: { + messages, + responseText: + response ?? + (messages[messages.length - 1].content?.text || + messages[messages.length - 1].content), + }, + }, + commit, + }).then((r) => r.unwrap()) + + return c.json(documentLog) + }, +) diff --git a/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts new file mode 100644 index 000000000..94d12e74d --- /dev/null +++ b/apps/gateway/src/routes/api/v2/projects/[projectId]/versions/[versionUuid]/documents/logs/index.ts @@ -0,0 +1,9 @@ +import { Hono } from 'hono' + +import { postHandler } from './handlers/post' + +const router = new Hono() + +router.post('/', ...postHandler) + +export { router as logsRouterV2 } diff --git a/apps/gateway/src/routes/app.ts b/apps/gateway/src/routes/app.ts index 3d614340f..5ec73525c 100644 --- a/apps/gateway/src/routes/app.ts +++ b/apps/gateway/src/routes/app.ts @@ -1,10 +1,8 @@ -import ROUTES from '$/common/routes' import authMiddleware from '$/middlewares/auth' import errorHandlerMiddleware from '$/middlewares/errorHandler' import rateLimitMiddleware from '$/middlewares/rateLimit' import { Hono } from 'hono' import { logger } from 'hono/logger' -import jetPaths from 'jet-paths' import { chatsRouter as chatsRouterV1 } from './api/v1/conversations/[conversationUuid]' import { documentsRouter as documentsRouterV1 } from './api/v1/projects/[projectId]/versions/[versionUuid]/documents' @@ -26,10 +24,19 @@ app.use(rateLimitMiddleware()) app.use(authMiddleware()) // Routers -app.route(jetPaths(ROUTES).Api.V1.Documents.Base, documentsRouterV1) -app.route(jetPaths(ROUTES).Api.V1.Conversations.Base, chatsRouterV1) -app.route(jetPaths(ROUTES).Api.V2.Documents.Base, documentsRouterV2) -app.route(jetPaths(ROUTES).Api.V2.Conversations.Base, chatsRouterV2) +// v1 +app.route( + '/api/v1/projects/:projectId/versions/:versionUuid/documents', + documentsRouterV1, +) +app.route('/api/v1/conversations', chatsRouterV1) + +// v2 +app.route( + '/api/v2/projects/:projectId/versions/:versionUuid/documents', + documentsRouterV2, +) +app.route('/api/v2/conversations', chatsRouterV2) // Must be the last one! app.onError(errorHandlerMiddleware) diff --git a/apps/gateway/test/support/paths.ts b/apps/gateway/test/support/paths.ts deleted file mode 100644 index c9c3e80af..000000000 --- a/apps/gateway/test/support/paths.ts +++ /dev/null @@ -1,6 +0,0 @@ -import ROUTES from '$/common/routes' -import jetPaths from 'jet-paths' - -const paths = jetPaths(ROUTES) - -export default paths diff --git a/apps/web/src/actions/datasets/create.ts b/apps/web/src/actions/datasets/create.ts index c4034acf6..62da648bf 100644 --- a/apps/web/src/actions/datasets/create.ts +++ b/apps/web/src/actions/datasets/create.ts @@ -1,27 +1,17 @@ 'use server' +import { + DELIMITER_VALUES, + DELIMITERS_KEYS, + MAX_SIZE, + MAX_UPLOAD_SIZE_IN_MB, +} from '@latitude-data/core/browser' import { DatasetsRepository } from '@latitude-data/core/repositories' import { createDataset } from '@latitude-data/core/services/datasets/create' import { z } from 'zod' import { authProcedure } from '../procedures' -const DELIMITERS_KEYS = [ - 'comma', - 'semicolon', - 'tab', - 'space', - 'custom', -] as const -const DELIMITER_VALUES = { - comma: ',', - semicolon: ';', - tab: '\t', - space: ' ', -} - -const MAX_SIZE = 15 -const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024 export const createDatasetAction = authProcedure .createServerAction() .input( diff --git a/apps/web/src/actions/documentLogs/upload.ts b/apps/web/src/actions/documentLogs/upload.ts new file mode 100644 index 000000000..9473dea21 --- /dev/null +++ b/apps/web/src/actions/documentLogs/upload.ts @@ -0,0 +1,50 @@ +'use server' + +import { + DELIMITERS_KEYS, + MAX_SIZE, + MAX_UPLOAD_SIZE_IN_MB, +} from '@latitude-data/core/browser' +import { CommitsRepository } from '@latitude-data/core/repositories' +import { bulkUploadDocumentLogs } from '@latitude-data/core/services/documentLogs/bulkUpload' +import { z } from 'zod' + +import { withDocument } from '../procedures' + +export const uploadDocumentLogsAction = withDocument + .createServerAction() + .input( + z.object({ + csvDelimiter: z.enum(DELIMITERS_KEYS, { + message: 'Choose a valid delimiter option', + }), + logsFile: z + .instanceof(File) + .refine((file) => { + return !file || file.size <= MAX_UPLOAD_SIZE_IN_MB + }, `Your file must be less than ${MAX_SIZE}MB in size`) + .refine( + (file) => file.type === 'text/csv', + 'Your file must be a CSV file', + ), + }), + ) + .handler(async ({ ctx, input }) => { + const commitsScope = new CommitsRepository(ctx.workspace.id) + const commit = await commitsScope + .getCommitByUuid({ + projectId: ctx.project.id, + uuid: ctx.currentCommitUuid, + }) + .then((r) => r.unwrap()) + + await bulkUploadDocumentLogs({ + workspace: ctx.workspace, + document: ctx.document, + commit, + csvDelimiter: input.csvDelimiter, + logsFile: input.logsFile, + }) + + return { success: true } + }) diff --git a/apps/web/src/app/(private)/datasets/new/_components/DelimiterSelector/index.tsx b/apps/web/src/app/(private)/datasets/new/_components/DelimiterSelector/index.tsx index 59dac5e29..56788c781 100644 --- a/apps/web/src/app/(private)/datasets/new/_components/DelimiterSelector/index.tsx +++ b/apps/web/src/app/(private)/datasets/new/_components/DelimiterSelector/index.tsx @@ -13,16 +13,22 @@ export enum DelimiterEnum { } const DELIMITERS = [ - { value: DelimiterEnum.Comma, label: 'Comma (ex.: column1,column2,column3)' }, + { + value: DelimiterEnum.Comma, + label: 'Comma (e.g.: column1,column2,column3)', + }, { value: DelimiterEnum.Semicolon, - label: 'Semicolon (ex.: column1;column2;column3)', + label: 'Semicolon (e.g.: column1;column2;column3)', }, { value: DelimiterEnum.Tab, - label: 'Tab (ex.: column1 \\t column2 \\t column3)', + label: 'Tab (e.g.: column1 \\t column2 \\t column3)', + }, + { + value: DelimiterEnum.Space, + label: 'Space (e.g.: column1 column2 column3)', }, - { value: DelimiterEnum.Space, label: 'Space (ex.: column1 column2 column3)' }, { value: DelimiterEnum.Custom, label: 'Custom' }, ] export const DELIMITER_KEYS = DELIMITERS.map(({ value }) => value) @@ -32,15 +38,15 @@ export default function DelimiterSelector({ customDelimiterInputName, delimiterErrors, customDelimiterErrors, - delimiterValue, + delimiterDefaultValue, customDelimiterValue, }: { delimiterInputName: string - delimiterValue: string | undefined - delimiterErrors: string[] | undefined + delimiterDefaultValue?: string | undefined + delimiterErrors?: string[] | undefined customDelimiterInputName: string - customDelimiterValue: string - customDelimiterErrors: string[] | undefined + customDelimiterValue?: string + customDelimiterErrors?: string[] | undefined }) { const inputRef = useRef(null) const [isCustom, setIsCustom] = useState(false) @@ -60,7 +66,7 @@ export default function DelimiterSelector({ + + + +
+ +
+ + Upload a csv file with your prompt logs.{' '} + + Learn more + + +
+
+ + + + + ) +} diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/upload/page.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/upload/page.tsx new file mode 100644 index 000000000..906855598 --- /dev/null +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/upload/page.tsx @@ -0,0 +1,19 @@ +import UploadLogModal from './UploadLogModal' + +export default function UploadLogModalPage({ + params, +}: { + params: { + documentUuid: string + projectId: string + commitUuid: string + } +}) { + return ( + + ) +} diff --git a/apps/web/src/app/_lib/formatUtils.ts b/apps/web/src/app/_lib/formatUtils.ts index 4d1c473f6..39dc1c665 100644 --- a/apps/web/src/app/_lib/formatUtils.ts +++ b/apps/web/src/app/_lib/formatUtils.ts @@ -15,7 +15,8 @@ export function relativeTime(date: Date | null) { return format(date, 'PPpp') } -export function formatDuration(duration: number) { +export function formatDuration(duration?: number | null) { + if (!duration) return '-' if (duration < MINUTES) return `${(duration / SECONDS).toFixed(3)}s` const hours = Math.floor(duration / HOURS) diff --git a/apps/web/src/services/routes.ts b/apps/web/src/services/routes.ts index 443415e20..ab7addb85 100644 --- a/apps/web/src/services/routes.ts +++ b/apps/web/src/services/routes.ts @@ -134,6 +134,7 @@ export const ROUTES = { }, [DocumentRoutes.logs]: { root: `${root}/${DocumentRoutes.logs}`, + upload: `${root}/${DocumentRoutes.logs}/upload`, }, } }, diff --git a/docs/assets/upload_logs_1.png b/docs/assets/upload_logs_1.png new file mode 100644 index 000000000..64897efcd Binary files /dev/null and b/docs/assets/upload_logs_1.png differ diff --git a/docs/guides/evaluations/running-evaluations.mdx b/docs/guides/evaluations/running-evaluations.mdx index 97a98a58b..9f5461458 100644 --- a/docs/guides/evaluations/running-evaluations.mdx +++ b/docs/guides/evaluations/running-evaluations.mdx @@ -42,4 +42,4 @@ Click the "Run evaluation" button to start the evaluation process. You'll see th Evaluations running in live mode will run on all new logs generated in your project. This is useful if you want to monitor the performance of your prompts in real-time. -We recommend keeping a few key evaluations running in live mode to spot degradations in response quality as soon as they happen. Sometimes new model releases or changes in parameters can lead to a drop in response quality, so this is a good way to catch those issues early. \ No newline at end of file +We recommend keeping a few key evaluations running in live mode to spot degradations in response quality as soon as they happen. Sometimes new model releases or changes in parameters can lead to a drop in response quality, so this is a good way to catch those issues early. diff --git a/docs/guides/logs/upload-logs.mdx b/docs/guides/logs/upload-logs.mdx new file mode 100644 index 000000000..53392b187 --- /dev/null +++ b/docs/guides/logs/upload-logs.mdx @@ -0,0 +1,44 @@ +--- +title: Upload logs +description: 'Learn how to upload logs to Latitude and start evaluating your production systems.' +--- + +## Upload logs to Latitude + +Most users use Latitude's SDK to run their prompts via Latitude's AI Gateway. However, if you're using a different method to run your prompts, you can still upload logs to Latitude and evalute them in order to assess the quality of your outputs. + +### Upload logs from Latitude's Dashboard + +Navigate to your prompt's logs section and click on the **Upload logs** button. The following screen will appear: + +![Upload logs](/images/guides/logs/upload_logs_1.png) + +You can upload a list of logs in a single csv file. The CSV file should be a single column csv where each row is a log, i.e a user-assitant interaction. [Here's an example](https://docs.google.com/spreadsheets/d/1uxmUW2XhcqRB_cK0SBmHzUfa9xMqVzKZ0eT8umO8pr8/edit?usp=sharing) of a valid CSV file: + +```csv +"[{""role"":""system"",""content"":""Tell me a joke about School""},{""role"": ""assistant"", ""content"": ""Why did the math book look sad? Because it had too many problems!""}]" +"[{""role"":""system"",""content"":""Tell me a joke about Travel""}],{""role"": ""assistant"", ""content"": ""Why did the hiker refuse to eat pizza on the way to the top of the mountain? Because he was afraid of heights and the pizza was too big!""}]" +"[{""role"":""system"",""content"":""Tell me a joke about Work""}],{""role"": ""assistant"", ""content"": ""Why did the worker get fired from his job? Because he was not properly training for it!""}]" +"[{""role"":""system"",""content"":""Tell me a joke about Sports""}],{""role"": ""assistant"", ""content"": ""Why did the sports star quit his job? Because he was not properly training for it!""}]" +``` + + + Logs follow OpenAI's [ChatCompletion + format](https://platform.openai.com/docs/guides/text-generation/building-prompts). + If you're using a different method to run your prompts, you'll need to format + your logs accordingly. + + +Do not include headers in your CSV file. + +Once you've uploaded your logs, they will quickly appear in your prompt's logs section. + +### Evaluating uploaded logs + +To evaluate your uploaded logs, make sure to first create and connect an evaluation to your prompt. Also, make sure the evaluation is configured to [evaluate live logs](/guides/evaluations/running-evaluations). + +Once you've correctly configured your evaluation, just upload your logs as described above and they will be automatically evaluated in real time. + +### Upload logs from the SDK + +You can push logs to Latitude with Latitude's SDK. Check out Latitude' SDK [`log` method](/guides/sdk/javascript-typescript-sdk). diff --git a/docs/guides/sdk/javascript-typescript-sdk.mdx b/docs/guides/sdk/javascript-typescript-sdk.mdx index d15812c30..a9db3cc36 100644 --- a/docs/guides/sdk/javascript-typescript-sdk.mdx +++ b/docs/guides/sdk/javascript-typescript-sdk.mdx @@ -125,7 +125,53 @@ onError: (error) => { ``` **IMPORTANT**: If you don't provide `onError` callback `await sdk.run` will throw an error. -## More examples +## Pushing a log to Latitude + +You can push a log to Latitude in order to evaluate it, using the `log` method: + +```typescript +const sdk = new Latitude(process.env.LATITUDE_API_KEY, { + projectId: 1, +}) +const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, +}) + +const messages = [ + { + role: 'user' as MessageRole.user, + content: [ + { + type: 'text' as ContentType.text, + text: 'Please tell me a joke about doctors', + }, + ], + }, +] +const chatCompletion = await openai.chat.completions.create({ + messages, + model: 'gpt-4o-mini', +}) + +// Push the log to the live version of our prompt called 'joker' +sdk.log('joker', messages, { + response: chatCompletion.choices[0].message.content, +}) +``` + + + Logs follow OpenAI's + [format](https://platform.openai.com/docs/guides/text-generation/building-prompts). + If you're using a different method to run your prompts, you'll need to format + your logs accordingly. + + + + If you include the assistant response in the optional `response` parameter, + make sure to not include it in the log so it isn't included twice. + + +## Common examples ### Executing tool calls diff --git a/docs/mint.json b/docs/mint.json index 46c2b518a..6c6892366 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -65,7 +65,10 @@ }, { "group": "Logs", - "pages": ["guides/logs/overview"] + "pages": [ + "guides/logs/overview", + "guides/logs/upload-logs" + ] }, { "group": "Datasets", diff --git a/examples/sdks/typescript/package.json b/examples/sdks/typescript/package.json new file mode 100644 index 000000000..c9e72f6fd --- /dev/null +++ b/examples/sdks/typescript/package.json @@ -0,0 +1,17 @@ +{ + "name": "typescript-examples", + "version": "1.0.0", + "description": "", + "main": "index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@latitude-data/sdk": "workspace:^", + "openai": "^4.68.4" + } +} diff --git a/examples/sdks/typescript/push_logs.ts b/examples/sdks/typescript/push_logs.ts new file mode 100644 index 000000000..6f232149b --- /dev/null +++ b/examples/sdks/typescript/push_logs.ts @@ -0,0 +1,30 @@ +import { ContentType, Latitude, MessageRole } from '@latitude-data/sdk' +import OpenAI from 'openai' + +const sdk = new Latitude(process.env.LATITUDE_API_KEY, { + projectId: 1, +}) +const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, +}) + +const messages = [ + { + role: 'user' as MessageRole.user, + content: [ + { + type: 'text' as ContentType.text, + text: 'Please tell me a joke about doctors', + }, + ], + }, +] +const chatCompletion = await openai.chat.completions.create({ + messages, + model: 'gpt-4o-mini', +}) + +// Push the log to the live version of our prompt called 'joker' +sdk.log('joker', messages, { + response: chatCompletion.choices[0].message.content, +}) diff --git a/examples/sdks/typescript/tsconfig.json b/examples/sdks/typescript/tsconfig.json new file mode 100644 index 000000000..41254be94 --- /dev/null +++ b/examples/sdks/typescript/tsconfig.json @@ -0,0 +1,6 @@ +{ + "compilerOptions": { + "module": "node16", + "moduleResolution": "node16" + } +} diff --git a/packages/constants/src/errors.ts b/packages/constants/src/errors.ts index f5628aea2..e8d6485a9 100644 --- a/packages/constants/src/errors.ts +++ b/packages/constants/src/errors.ts @@ -33,7 +33,7 @@ export type RunErrorDetails = : never export enum ApiErrorCodes { - HTTPExeption = 'http_exception', + HTTPException = 'http_exception', InternalServerError = 'internal_server_error', } diff --git a/packages/core/drizzle/0089_giant_maverick.sql b/packages/core/drizzle/0089_giant_maverick.sql new file mode 100644 index 000000000..ed54411b0 --- /dev/null +++ b/packages/core/drizzle/0089_giant_maverick.sql @@ -0,0 +1,8 @@ +ALTER TYPE "latitude"."log_source" ADD VALUE 'user';--> statement-breakpoint +ALTER TABLE "latitude"."document_logs" ALTER COLUMN "duration" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "provider_id" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "finish_reason" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "config" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "tokens" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "duration" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "generated_at" DROP NOT NULL; \ No newline at end of file diff --git a/packages/core/drizzle/meta/0089_snapshot.json b/packages/core/drizzle/meta/0089_snapshot.json new file mode 100644 index 000000000..39f60eaef --- /dev/null +++ b/packages/core/drizzle/meta/0089_snapshot.json @@ -0,0 +1,2982 @@ +{ + "id": "6a84afd1-2279-479a-955c-49416778519a", + "prevId": "63bdabc9-4171-4c9c-a338-7d97eae28711", + "version": "7", + "dialect": "postgresql", + "tables": { + "latitude.users": { + "name": "users", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "admin": { + "name": "admin", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "latitude.sessions": { + "name": "sessions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.workspaces": { + "name": "workspaces", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "current_subscription_id": { + "name": "current_subscription_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_current_subscription_id_subscriptions_id_fk": { + "name": "workspaces_current_subscription_id_subscriptions_id_fk", + "tableFrom": "workspaces", + "tableTo": "subscriptions", + "schemaTo": "latitude", + "columnsFrom": [ + "current_subscription_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "workspaces_creator_id_users_id_fk": { + "name": "workspaces_creator_id_users_id_fk", + "tableFrom": "workspaces", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.subscriptions": { + "name": "subscriptions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "subscription_plans", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "subscriptions_workspace_id_index": { + "name": "subscriptions_workspace_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "subscriptions_plan_index": { + "name": "subscriptions_plan_index", + "columns": [ + { + "expression": "plan", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.memberships": { + "name": "memberships", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_token": { + "name": "invitation_token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_workspace_id_user_id_index": { + "name": "memberships_workspace_id_user_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_invitation_token_index": { + "name": "memberships_invitation_token_index", + "columns": [ + { + "expression": "invitation_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_workspace_id_workspaces_id_fk": { + "name": "memberships_workspace_id_workspaces_id_fk", + "tableFrom": "memberships", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "memberships_invitation_token_unique": { + "name": "memberships_invitation_token_unique", + "nullsNotDistinct": false, + "columns": [ + "invitation_token" + ] + } + } + }, + "latitude.api_keys": { + "name": "api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_id_idx": { + "name": "workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_workspace_id_workspaces_id_fk": { + "name": "api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_token_unique": { + "name": "api_keys_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.claimed_rewards": { + "name": "claimed_rewards", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reward_type": { + "name": "reward_type", + "type": "reward_types", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "reference": { + "name": "reference", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "is_valid": { + "name": "is_valid", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "claimed_rewards_workspace_id_idx": { + "name": "claimed_rewards_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "claimed_rewards_workspace_id_workspaces_id_fk": { + "name": "claimed_rewards_workspace_id_workspaces_id_fk", + "tableFrom": "claimed_rewards", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "claimed_rewards_creator_id_users_id_fk": { + "name": "claimed_rewards_creator_id_users_id_fk", + "tableFrom": "claimed_rewards", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.projects": { + "name": "projects", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_idx": { + "name": "workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "projects_deleted_at_idx": { + "name": "projects_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "projects_workspace_id_workspaces_id_fk": { + "name": "projects_workspace_id_workspaces_id_fk", + "tableFrom": "projects", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.commits": { + "name": "commits", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_commit_order_idx": { + "name": "project_commit_order_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_commit_version": { + "name": "unique_commit_version", + "columns": [ + { + "expression": "version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "user_idx": { + "name": "user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "merged_at_idx": { + "name": "merged_at_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "project_id_idx": { + "name": "project_id_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "commits_deleted_at_indx": { + "name": "commits_deleted_at_indx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "commits_project_id_projects_id_fk": { + "name": "commits_project_id_projects_id_fk", + "tableFrom": "commits", + "tableTo": "projects", + "schemaTo": "latitude", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "commits_user_id_users_id_fk": { + "name": "commits_user_id_users_id_fk", + "tableFrom": "commits", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "commits_uuid_unique": { + "name": "commits_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.document_versions": { + "name": "document_versions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_versions_unique_document_uuid_commit_id": { + "name": "document_versions_unique_document_uuid_commit_id", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_unique_path_commit_id_deleted_at": { + "name": "document_versions_unique_path_commit_id_deleted_at", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_commit_id_idx": { + "name": "document_versions_commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_deleted_at_idx": { + "name": "document_versions_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_versions_path_idx": { + "name": "document_versions_path_idx", + "columns": [ + { + "expression": "path", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_versions_commit_id_commits_id_fk": { + "name": "document_versions_commit_id_commits_id_fk", + "tableFrom": "document_versions", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.provider_api_keys": { + "name": "provider_api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_apikeys_workspace_id_idx": { + "name": "provider_apikeys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_api_keys_name_workspace_id_deleted_at_index": { + "name": "provider_api_keys_name_workspace_id_deleted_at_index", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_user_id_idx": { + "name": "provider_apikeys_user_id_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_api_keys_token_provider_workspace_id_deleted_at_index": { + "name": "provider_api_keys_token_provider_workspace_id_deleted_at_index", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_api_keys_author_id_users_id_fk": { + "name": "provider_api_keys_author_id_users_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_api_keys_workspace_id_workspaces_id_fk": { + "name": "provider_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_api_keys_name_workspace_id_deleted_at_unique": { + "name": "provider_api_keys_name_workspace_id_deleted_at_unique", + "nullsNotDistinct": true, + "columns": [ + "name", + "workspace_id", + "deleted_at" + ] + }, + "provider_api_keys_token_provider_workspace_id_deleted_at_unique": { + "name": "provider_api_keys_token_provider_workspace_id_deleted_at_unique", + "nullsNotDistinct": true, + "columns": [ + "token", + "provider", + "workspace_id", + "deleted_at" + ] + } + } + }, + "latitude.document_logs": { + "name": "document_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parameters": { + "name": "parameters", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "custom_identifier": { + "name": "custom_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_log_own_uuid_idx": { + "name": "document_log_own_uuid_idx", + "columns": [ + { + "expression": "uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_log_uuid_idx": { + "name": "document_log_uuid_idx", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_logs_commit_id_idx": { + "name": "document_logs_commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_logs_content_hash_idx": { + "name": "document_logs_content_hash_idx", + "columns": [ + { + "expression": "content_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_logs_created_at_idx": { + "name": "document_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_logs_commit_id_commits_id_fk": { + "name": "document_logs_commit_id_commits_id_fk", + "tableFrom": "document_logs", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "document_logs_uuid_unique": { + "name": "document_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.run_errors": { + "name": "run_errors", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "code": { + "name": "code", + "type": "run_error_code_enum", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "errorable_type": { + "name": "errorable_type", + "type": "run_error_entity_enum", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "errorable_uuid": { + "name": "errorable_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "run_errors_errorable_entity_uuid_idx": { + "name": "run_errors_errorable_entity_uuid_idx", + "columns": [ + { + "expression": "errorable_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "errorable_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.provider_logs": { + "name": "provider_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_log_uuid": { + "name": "document_log_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'stop'" + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "response_object": { + "name": "response_object", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_text": { + "name": "response_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "tokens": { + "name": "tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cost_in_millicents": { + "name": "cost_in_millicents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "apiKeyId": { + "name": "apiKeyId", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "generated_at": { + "name": "generated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_idx": { + "name": "provider_idx", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_logs_created_at_idx": { + "name": "provider_logs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_logs_workspace_id_index": { + "name": "provider_logs_workspace_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_logs_provider_id_provider_api_keys_id_fk": { + "name": "provider_logs_provider_id_provider_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "provider_api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_apiKeyId_api_keys_id_fk": { + "name": "provider_logs_apiKeyId_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "apiKeyId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_logs_uuid_unique": { + "name": "provider_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.datasets": { + "name": "datasets", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "csv_delimiter": { + "name": "csv_delimiter", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_key": { + "name": "file_key", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "file_metadata": { + "name": "file_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_workspace_idx": { + "name": "datasets_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_author_idx": { + "name": "datasets_author_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_workspace_id_name_index": { + "name": "datasets_workspace_id_name_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_workspace_id_workspaces_id_fk": { + "name": "datasets_workspace_id_workspaces_id_fk", + "tableFrom": "datasets", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_author_id_users_id_fk": { + "name": "datasets_author_id_users_id_fk", + "tableFrom": "datasets", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations": { + "name": "evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata_id": { + "name": "metadata_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "metadata_type", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_workspace_idx": { + "name": "evaluation_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_metadata_idx": { + "name": "evaluation_metadata_idx", + "columns": [ + { + "expression": "metadata_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "metadata_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluations_deleted_at_idx": { + "name": "evaluations_deleted_at_idx", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluations_workspace_id_workspaces_id_fk": { + "name": "evaluations_workspace_id_workspaces_id_fk", + "tableFrom": "evaluations", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "evaluations_uuid_unique": { + "name": "evaluations_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.llm_as_judge_evaluation_metadatas": { + "name": "llm_as_judge_evaluation_metadatas", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "template_id": { + "name": "template_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "llm_as_judge_evaluation_metadatas_template_id_idx": { + "name": "llm_as_judge_evaluation_metadatas_template_id_idx", + "columns": [ + { + "expression": "template_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk": { + "name": "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk", + "tableFrom": "llm_as_judge_evaluation_metadatas", + "tableTo": "evaluations_templates", + "schemaTo": "latitude", + "columnsFrom": [ + "template_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.connected_evaluations": { + "name": "connected_evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "live": { + "name": "live", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "connected_evaluations_evaluation_idx": { + "name": "connected_evaluations_evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "connected_evaluations_evaluation_id_evaluations_id_fk": { + "name": "connected_evaluations_evaluation_id_evaluations_id_fk", + "tableFrom": "connected_evaluations", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "connected_evaluations_unique_idx": { + "name": "connected_evaluations_unique_idx", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "evaluation_id" + ] + } + } + }, + "latitude.evaluation_results": { + "name": "evaluation_results", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "document_log_id": { + "name": "document_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_log_id": { + "name": "provider_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "resultable_type": { + "name": "resultable_type", + "type": "evaluation_result_types", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "resultable_id": { + "name": "resultable_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_idx": { + "name": "evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_log_idx": { + "name": "document_log_idx", + "columns": [ + { + "expression": "document_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_log_idx": { + "name": "provider_log_idx", + "columns": [ + { + "expression": "provider_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resultable_idx": { + "name": "resultable_idx", + "columns": [ + { + "expression": "resultable_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resultable_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_results_created_at_idx": { + "name": "evaluation_results_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_results_evaluation_id_evaluations_id_fk": { + "name": "evaluation_results_evaluation_id_evaluations_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "evaluation_results_document_log_id_document_logs_id_fk": { + "name": "evaluation_results_document_log_id_document_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "document_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "document_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "evaluation_results_provider_log_id_provider_logs_id_fk": { + "name": "evaluation_results_provider_log_id_provider_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "provider_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "evaluation_results_uuid_unique": { + "name": "evaluation_results_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.evaluations_templates": { + "name": "evaluations_templates", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "evaluations_templates_category_evaluations_template_categories_id_fk": { + "name": "evaluations_templates_category_evaluations_template_categories_id_fk", + "tableFrom": "evaluations_templates", + "tableTo": "evaluations_template_categories", + "schemaTo": "latitude", + "columnsFrom": [ + "category" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_template_categories": { + "name": "evaluations_template_categories", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.magic_link_tokens": { + "name": "magic_link_tokens", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "expired_at": { + "name": "expired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "magic_link_tokens_user_id_users_id_fk": { + "name": "magic_link_tokens_user_id_users_id_fk", + "tableFrom": "magic_link_tokens", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "magic_link_tokens_token_unique": { + "name": "magic_link_tokens_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.events": { + "name": "events", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_workspace_idx": { + "name": "event_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_type_idx": { + "name": "event_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "events_workspace_id_workspaces_id_fk": { + "name": "events_workspace_id_workspaces_id_fk", + "tableFrom": "events", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_numbers": { + "name": "evaluation_resultable_numbers", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_texts": { + "name": "evaluation_resultable_texts", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_booleans": { + "name": "evaluation_resultable_booleans", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "latitude.subscription_plans": { + "name": "subscription_plans", + "schema": "latitude", + "values": [ + "hobby_v1", + "hobby_v2", + "team_v1", + "enterprise_v1" + ] + }, + "latitude.reward_types": { + "name": "reward_types", + "schema": "latitude", + "values": [ + "github_star", + "follow", + "post", + "github_issue", + "referral", + "signup_launch_day" + ] + }, + "latitude.provider": { + "name": "provider", + "schema": "latitude", + "values": [ + "openai", + "anthropic", + "groq", + "mistral", + "azure", + "google", + "custom" + ] + }, + "latitude.run_error_code_enum": { + "name": "run_error_code_enum", + "schema": "latitude", + "values": [ + "unknown_error", + "default_provider_exceeded_quota_error", + "document_config_error", + "missing_provider_error", + "chain_compile_error", + "ai_run_error", + "unsupported_provider_response_type_error", + "ai_provider_config_error", + "ev_run_missing_provider_log_error", + "ev_run_missing_workspace_error", + "ev_run_unsupported_result_type_error", + "ev_run_response_json_format_error", + "default_provider_invalid_model_error" + ] + }, + "latitude.run_error_entity_enum": { + "name": "run_error_entity_enum", + "schema": "latitude", + "values": [ + "document_log", + "evaluation_result" + ] + }, + "latitude.log_source": { + "name": "log_source", + "schema": "latitude", + "values": [ + "playground", + "api", + "evaluation", + "user" + ] + }, + "latitude.metadata_type": { + "name": "metadata_type", + "schema": "latitude", + "values": [ + "llm_as_judge" + ] + }, + "public.evaluation_result_types": { + "name": "evaluation_result_types", + "schema": "public", + "values": [ + "evaluation_resultable_booleans", + "evaluation_resultable_texts", + "evaluation_resultable_numbers" + ] + } + }, + "schemas": { + "latitude": "latitude" + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/core/drizzle/meta/_journal.json b/packages/core/drizzle/meta/_journal.json index c8e60fda6..35515c2d1 100644 --- a/packages/core/drizzle/meta/_journal.json +++ b/packages/core/drizzle/meta/_journal.json @@ -617,6 +617,13 @@ "when": 1730192854255, "tag": "0088_legacy_evaluations_part_3", "breakpoints": true + }, + { + "idx": 89, + "version": "7", + "when": 1730198912637, + "tag": "0089_giant_maverick", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index b8811524d..29bb73ec5 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -1,9 +1,9 @@ -import type { - AssistantMessage, - Message as CompilerMessage, - SystemMessage, - ToolCall, - UserMessage, +import { + type AssistantMessage, + type Message as CompilerMessage, + type SystemMessage, + type ToolCall, + type UserMessage, } from '@latitude-data/compiler' import { CoreTool, @@ -11,6 +11,7 @@ import { ObjectStreamPart, TextStreamPart, } from 'ai' +import { z } from 'zod' import { ProviderLog } from './browser' import { Config } from './services/ai' @@ -78,6 +79,7 @@ export enum LogSources { API = 'api', Playground = 'playground', Evaluation = 'evaluation', + User = 'user', } export enum ErrorableEntity { @@ -264,8 +266,8 @@ export type SerializedProviderLog = { messages: SerializedConversation context: string response: string | null - config: object - duration: number + config: object | null + duration: number | null cost: number } @@ -296,3 +298,88 @@ export type SerializedEvaluationResult = Omit< } export const ULTRA_LARGE_PAGE_SIZE = 1000 +export const DELIMITER_VALUES = { + comma: ',', + semicolon: ';', + tab: '\t', + space: ' ', +} +export const DELIMITERS_KEYS = [ + 'comma', + 'semicolon', + 'tab', + 'space', + 'custom', +] as const +export const MAX_SIZE = 15 +export const MAX_UPLOAD_SIZE_IN_MB = MAX_SIZE * 1024 * 1024 + +const userContentSchema = z.array( + z + .object({ + type: z.literal('text'), + text: z.string(), + }) + .or( + z.object({ + type: z.literal('image'), + image: z + .string() + .or(z.instanceof(Uint8Array)) + .or(z.instanceof(Buffer)) + .or(z.instanceof(ArrayBuffer)) + .or(z.instanceof(URL)), + }), + ), +) + +export const messageSchema = z + .object({ + role: z.literal('system'), + content: z.string(), + }) + .or( + z.object({ + role: z.literal('user'), + name: z.string().optional(), + content: userContentSchema, + }), + ) + .or( + z.object({ + role: z.literal('assistant'), + content: z.string().or( + z.array( + z.object({ + type: z.literal('tool-call'), + toolCallId: z.string(), + toolName: z.string(), + args: z.record(z.any()), + }), + ), + ), + toolCalls: z.array( + z.object({ + id: z.string(), + name: z.string(), + arguments: z.record(z.any()), + }), + ), + }), + ) + .or( + z.object({ + role: z.literal('tool'), + content: z.array( + z.object({ + type: z.literal('tool-result'), + toolCallId: z.string(), + toolName: z.string(), + result: z.string(), + isError: z.boolean().optional(), + }), + ), + }), + ) + +export const messagesSchema = z.array(z.any(messageSchema)) diff --git a/packages/core/src/jobs/constants.ts b/packages/core/src/jobs/constants.ts index 2d4f2e8ef..e2bd656db 100644 --- a/packages/core/src/jobs/constants.ts +++ b/packages/core/src/jobs/constants.ts @@ -19,6 +19,7 @@ export enum Jobs { runEvaluationJob = 'runEvaluationJob', publishToAnalyticsJob = 'publishToAnalyticsJob', runLiveEvaluationJob = 'runLiveEvaluationJob', + uploadDocumentLogsJob = 'uploadDocumentLogsJob', } export const QUEUES = { @@ -31,6 +32,8 @@ export const QUEUES = { 'runDocumentInBatchJob', 'runDocumentJob', 'runEvaluationJob', + 'uploadDocumentLogsJob', + 'createDocumentLogJob', ], }, [Queues.eventsQueue]: { diff --git a/packages/core/src/jobs/job-definitions/documentLogs/createDocumentLogJob.ts b/packages/core/src/jobs/job-definitions/documentLogs/createDocumentLogJob.ts new file mode 100644 index 000000000..1ca954c37 --- /dev/null +++ b/packages/core/src/jobs/job-definitions/documentLogs/createDocumentLogJob.ts @@ -0,0 +1,46 @@ +import { Message } from '@latitude-data/compiler' +import { Job } from 'bullmq' + +import { Commit } from '../../../browser' +import { LogSources } from '../../../constants' +import { generateUUIDIdentifier } from '../../../lib' +import { DocumentVersionsRepository } from '../../../repositories' +import { createDocumentLog } from '../../../services/documentLogs' + +export type CreateDocumentLogJobProps = { + workspaceId: number + documentUuid: string + commit: Commit + source: LogSources + messages: Message[] + responseText: string +} + +export const createDocumentLogJob = async ( + job: Job, +) => { + const { workspaceId, documentUuid, commit, source, messages, responseText } = + job.data + const docsRepo = new DocumentVersionsRepository(workspaceId) + const document = await docsRepo + .getDocumentByUuid({ + documentUuid, + commit, + }) + .then((r) => r.unwrap()) + + await createDocumentLog({ + data: { + uuid: generateUUIDIdentifier(), + documentUuid: document.documentUuid, + resolvedContent: document.resolvedContent || '', + source, + parameters: {}, + providerLog: { + messages, + responseText, + }, + }, + commit, + }).then((r) => r.unwrap()) +} diff --git a/packages/core/src/jobs/job-definitions/documentLogs/uploadDocumentLogsJob.ts b/packages/core/src/jobs/job-definitions/documentLogs/uploadDocumentLogsJob.ts new file mode 100644 index 000000000..2e9906366 --- /dev/null +++ b/packages/core/src/jobs/job-definitions/documentLogs/uploadDocumentLogsJob.ts @@ -0,0 +1,42 @@ +import { Job } from 'bullmq' + +import { setupJobs } from '../..' +import { Commit } from '../../../browser' +import { LogSources, messagesSchema } from '../../../constants' + +export type UploadDocumentLogsJobData = { + workspaceId: number + documentUuid: string + commit: Commit + source: LogSources + csv: { data: { record: string[] }[] } +} + +export const uploadDocumentLogsJob = async ( + job: Job, +) => { + const { workspaceId, source, documentUuid, commit, csv } = job.data + const jobs = await setupJobs() + + csv.data.forEach((row) => { + const messages = JSON.parse(row.record[0]!) + const response = row.record[1] + const result = messagesSchema.safeParse(messages) + if (!result.success) { + // TODO: notify client of invalid log format + return + } + + jobs.defaultQueue.jobs.enqueueCreateDocumentLogJob({ + workspaceId, + documentUuid, + commit, + source, + messages: response ? messages : messages.slice(0, -1), + responseText: + response ?? + (messages[messages.length - 1].content?.text || + messages[messages.length - 1].content), + }) + }) +} diff --git a/packages/core/src/jobs/job-definitions/index.ts b/packages/core/src/jobs/job-definitions/index.ts index 037c67bbc..4dcbc7f1d 100644 --- a/packages/core/src/jobs/job-definitions/index.ts +++ b/packages/core/src/jobs/job-definitions/index.ts @@ -9,3 +9,5 @@ export * from '../../events/handlers/sendMagicLinkHandler' export * from '../../events/handlers/sendInvitationToUser' export * from '../../events/handlers/sendReferralInvitation' export * from '../../events/handlers/createLoopsContact' +export * from './documentLogs/uploadDocumentLogsJob' +export * from './documentLogs/createDocumentLogJob' diff --git a/packages/core/src/jobs/job-definitions/types.d.ts b/packages/core/src/jobs/job-definitions/types.d.ts index ee5a3a8dc..64465e5ba 100644 --- a/packages/core/src/jobs/job-definitions/types.d.ts +++ b/packages/core/src/jobs/job-definitions/types.d.ts @@ -9,6 +9,7 @@ import type { RunBatchEvaluationJobParams } from './batchEvaluations/runBatchEva import type { RunEvaluationJobData } from './batchEvaluations/runEvaluationJob' import type { RunDocumentInBatchJobProps } from './documents/runDocumentInBatchJob' import type { RunDocumentJobData } from './documents/runDocumentJob' +import type { UploadDocumentLogsJobData } from './documents/uploadDocumentLogsJob' import type { RunLiveEvaluationJobData } from './liveEvaluations/runLiveEvaluationJob' export type JobDataMap = { @@ -23,6 +24,7 @@ export type JobDataMap = { [Jobs.runEvaluationJob]: RunEvaluationJobData [Jobs.publishToAnalyticsJob]: LatitudeEvent [Jobs.runLiveEvaluationJob]: RunLiveEvaluationJobData + [Jobs.uploadDocumentLogsJob]: UploadDocumentLogsJobData } type JobData = J extends keyof JobDataMap diff --git a/packages/core/src/lib/readCsv.ts b/packages/core/src/lib/readCsv.ts index bcafbd424..93bc223bf 100644 --- a/packages/core/src/lib/readCsv.ts +++ b/packages/core/src/lib/readCsv.ts @@ -16,6 +16,7 @@ type ParseCsvOptions = { // https://csv.js.org/parse/options/to_line/ toLine?: number fromLine?: number + columns?: boolean } type ParseResult = { record: Record @@ -28,7 +29,7 @@ export type CsvParsedData = { } export async function syncReadCsv( file: File | string, - { delimiter, toLine, fromLine }: ParseCsvOptions, + { delimiter, toLine, fromLine, columns = true }: ParseCsvOptions, ) { try { const data = await getData(file) @@ -37,7 +38,7 @@ export async function syncReadCsv( relax_column_count: true, skip_empty_lines: true, relax_quotes: true, - columns: true, + columns, trim: true, info: true, } @@ -58,7 +59,8 @@ export async function syncReadCsv( return Result.ok({ headers: [], rowCount: 0, data: [] }) const firstRecord = records[0]! - const headers = firstRecord.info.columns.map((column) => column.name) + const headers = + firstRecord.info?.columns?.map?.((column) => column.name) ?? [] return Result.ok({ rowCount: records.length, headers, data: records }) } catch (e) { const error = e as CsvError diff --git a/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts b/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts index 9593dffef..424e87575 100644 --- a/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts +++ b/packages/core/src/repositories/connectedEvaluationsRepository/getConnectedDocumentsWithMetadata.test.ts @@ -231,7 +231,8 @@ describe('getConnectedDocumentsWithMetadata', () => { ) const totalTokens = results.reduce( - (acc, r) => acc + r.providerLogs.reduce((acc2, l) => acc2 + l.tokens, 0), + (acc, r) => + acc + r.providerLogs.reduce((acc2, l) => acc2 + (l?.tokens ?? 0), 0), 0, ) const totalCost = results.reduce( diff --git a/packages/core/src/repositories/providerLogsRepository.ts b/packages/core/src/repositories/providerLogsRepository.ts index 97a8ded9a..2e79eac1e 100644 --- a/packages/core/src/repositories/providerLogsRepository.ts +++ b/packages/core/src/repositories/providerLogsRepository.ts @@ -29,10 +29,6 @@ export class ProviderLogsRepository extends Repository { async findByDocumentUuid(documentUuid: string, opts: QueryOptions = {}) { const query = this.scope - .innerJoin( - documentLogs, - eq(documentLogs.uuid, providerLogs.documentLogUuid), - ) .where(eq(documentLogs.documentUuid, documentUuid)) .orderBy(asc(providerLogs.generatedAt)) diff --git a/packages/core/src/schema/messages.ts b/packages/core/src/schema/messages.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/core/src/schema/models/documentLogs.ts b/packages/core/src/schema/models/documentLogs.ts index e7db96f24..e5f97e7eb 100644 --- a/packages/core/src/schema/models/documentLogs.ts +++ b/packages/core/src/schema/models/documentLogs.ts @@ -28,7 +28,7 @@ export const documentLogs = latitudeSchema.table( contentHash: text('content_hash').notNull(), parameters: jsonb('parameters').$type>().notNull(), customIdentifier: text('custom_identifier'), - duration: bigint('duration', { mode: 'number' }).notNull(), + duration: bigint('duration', { mode: 'number' }), source: logSourcesEnum('source'), ...timestamps(), }, diff --git a/packages/core/src/schema/models/providerLogs.ts b/packages/core/src/schema/models/providerLogs.ts index 6137c87e8..c3f5d407b 100644 --- a/packages/core/src/schema/models/providerLogs.ts +++ b/packages/core/src/schema/models/providerLogs.ts @@ -23,6 +23,7 @@ export const logSourcesEnum = latitudeSchema.enum('log_source', [ LogSources.Playground, LogSources.API, LogSources.Evaluation, + LogSources.User, ]) export const providerLogs = latitudeSchema.table( @@ -32,22 +33,23 @@ export const providerLogs = latitudeSchema.table( workspaceId: bigint('workspace_id', { mode: 'number' }), uuid: uuid('uuid').notNull().unique(), documentLogUuid: uuid('document_log_uuid'), - providerId: bigint('provider_id', { mode: 'number' }) - .notNull() - .references(() => providerApiKeys.id, { + providerId: bigint('provider_id', { mode: 'number' }).references( + () => providerApiKeys.id, + { onDelete: 'restrict', onUpdate: 'cascade', - }), + }, + ), model: varchar('model'), - finishReason: varchar('finish_reason').notNull().default('stop'), - config: json('config').$type().notNull(), + finishReason: varchar('finish_reason').default('stop'), + config: json('config').$type(), messages: json('messages').$type().notNull(), responseObject: jsonb('response_object').$type(), responseText: text('response_text').$type(), toolCalls: json('tool_calls').$type().notNull().default([]), - tokens: bigint('tokens', { mode: 'number' }).notNull(), + tokens: bigint('tokens', { mode: 'number' }), costInMillicents: integer('cost_in_millicents').notNull().default(0), - duration: bigint('duration', { mode: 'number' }).notNull(), // in milliseconds! + duration: bigint('duration', { mode: 'number' }), // in milliseconds! source: logSourcesEnum('source').notNull(), apiKeyId: bigint('apiKeyId', { mode: 'number' }).references( () => apiKeys.id, @@ -56,7 +58,7 @@ export const providerLogs = latitudeSchema.table( onUpdate: 'cascade', }, ), - generatedAt: timestamp('generated_at', { mode: 'date' }).notNull(), + generatedAt: timestamp('generated_at', { mode: 'date' }), ...timestamps(), }, (table) => ({ diff --git a/packages/core/src/services/documentLogs/bulkUpload.ts b/packages/core/src/services/documentLogs/bulkUpload.ts new file mode 100644 index 000000000..380155dc7 --- /dev/null +++ b/packages/core/src/services/documentLogs/bulkUpload.ts @@ -0,0 +1,32 @@ +import { Commit, DocumentVersion, Workspace } from '../../browser' +import { LogSources } from '../../constants' +import { setupJobs } from '../../jobs' +import { syncReadCsv } from '../../lib/readCsv' + +export async function bulkUploadDocumentLogs({ + csvDelimiter, + logsFile, + workspace, + document, + commit, +}: { + csvDelimiter: string + logsFile: File + workspace: Workspace + document: DocumentVersion + commit: Commit +}) { + const csv = await syncReadCsv(logsFile, { + delimiter: csvDelimiter, + columns: false, + }).then((r) => r.unwrap()) + const queues = await setupJobs() + + queues.defaultQueue.jobs.enqueueUploadDocumentLogsJob({ + workspaceId: workspace.id, + documentUuid: document.documentUuid, + commit, + csv, + source: LogSources.User, + }) +} diff --git a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts index f28ca1c4f..8b4522de8 100644 --- a/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts +++ b/packages/core/src/services/documentLogs/computeDocumentLogWithMetadata.test.ts @@ -68,8 +68,14 @@ describe('computeDocumentLogWithMetadata', () => { id: documentLog.id, uuid: documentLog.uuid, documentUuid: documentLog.documentUuid, - tokens: totalProviderLogs.reduce((acc, log) => acc + log.tokens, 0), - duration: totalProviderLogs.reduce((acc, log) => acc + log.duration, 0), + tokens: totalProviderLogs.reduce( + (acc, log) => acc + (log?.tokens ?? 0), + 0, + ), + duration: totalProviderLogs.reduce( + (acc, log) => acc + (log?.duration ?? 0), + 0, + ), costInMillicents: totalProviderLogs.reduce( (acc, log) => acc + log.costInMillicents, 0, diff --git a/packages/core/src/services/documentLogs/create.ts b/packages/core/src/services/documentLogs/create.ts index 67a85d5fc..8bd23abce 100644 --- a/packages/core/src/services/documentLogs/create.ts +++ b/packages/core/src/services/documentLogs/create.ts @@ -1,8 +1,18 @@ +import { Message } from '@latitude-data/compiler' + import { Commit, DocumentLog, LogSources } from '../../browser' import { database } from '../../client' +import { findWorkspaceFromCommit } from '../../data-access' import { publisher } from '../../events/publisher' -import { hashContent, Result, Transaction } from '../../lib' +import { + generateUUIDIdentifier, + hashContent, + NotFoundError, + Result, + Transaction, +} from '../../lib' import { documentLogs } from '../../schema' +import { createProviderLog } from '../providerLogs' export type CreateDocumentLogProps = { commit: Commit @@ -11,10 +21,14 @@ export type CreateDocumentLogProps = { documentUuid: string parameters: Record resolvedContent: string - duration: number + duration?: number source: LogSources customIdentifier?: string createdAt?: Date + providerLog?: { + messages: Message[] + responseText?: string + } } } @@ -29,6 +43,7 @@ export async function createDocumentLog( duration, source, createdAt, + providerLog, }, commit, }: CreateDocumentLogProps, @@ -52,6 +67,22 @@ export async function createDocumentLog( .returning() const documentLog = inserts[0]! + if (providerLog) { + const workspace = await findWorkspaceFromCommit(commit, trx) + if (!workspace) { + throw new NotFoundError('Workspace not found') + } + + await createProviderLog({ + uuid: generateUUIDIdentifier(), + documentLogUuid: documentLog.uuid, + messages: providerLog.messages, + responseText: providerLog.responseText, + generatedAt: new Date(), + source, + workspace, + }).then((r) => r.unwrap()) + } publisher.publishLater({ type: 'documentLogCreated', diff --git a/packages/core/src/services/documentLogs/index.ts b/packages/core/src/services/documentLogs/index.ts index b90681ae4..b3db43969 100644 --- a/packages/core/src/services/documentLogs/index.ts +++ b/packages/core/src/services/documentLogs/index.ts @@ -3,3 +3,4 @@ export * from './addMessages' export * from './computeDocumentLogWithMetadata' export * from './computeDocumentLogsWithMetadata' export * from './fetchDocumentLogWithMetadata' +export * from './bulkUpload' diff --git a/packages/core/src/services/evaluations/run/handleEvaluationResponse.ts b/packages/core/src/services/evaluations/run/handleEvaluationResponse.ts index b20539482..6ba618141 100644 --- a/packages/core/src/services/evaluations/run/handleEvaluationResponse.ts +++ b/packages/core/src/services/evaluations/run/handleEvaluationResponse.ts @@ -82,7 +82,7 @@ export async function handleEvaluationResponse({ if (!response.object) { error = new ChainError({ code: RunErrorCodes.EvaluationRunResponseJsonFormatError, - message: `Provider with model [${providerLog.config.model}] did not return a valid JSON object`, + message: `Provider with model [${providerLog?.config?.model ?? 'unknown'}] did not return a valid JSON object`, }) } else { result = response.object diff --git a/packages/core/src/services/providerLogs/addMessages.ts b/packages/core/src/services/providerLogs/addMessages.ts index 563fe13be..2df31f1ae 100644 --- a/packages/core/src/services/providerLogs/addMessages.ts +++ b/packages/core/src/services/providerLogs/addMessages.ts @@ -38,6 +38,21 @@ export async function addMessages({ messages: Message[] source: LogSources }) { + if (!providerLog.providerId) { + return Result.error( + new NotFoundError( + `Cannot add messages to a conversation that has no associated provider`, + ), + ) + } + if (!providerLog.config) { + return Result.error( + new NotFoundError( + `Cannot add messages to a conversation that has no associated configuration`, + ), + ) + } + const provider = await unsafelyFindProviderApiKey(providerLog.providerId) if (!provider) { return Result.error( @@ -78,7 +93,7 @@ export async function addMessages({ iterate({ workspace, source, - config: providerLog.config, + config: providerLog.config!, provider, controller, documentLogUuid: providerLog.documentLogUuid!, diff --git a/packages/core/src/services/providerLogs/create.ts b/packages/core/src/services/providerLogs/create.ts index 4b13c048d..925c6b1ee 100644 --- a/packages/core/src/services/providerLogs/create.ts +++ b/packages/core/src/services/providerLogs/create.ts @@ -17,13 +17,13 @@ export type CreateProviderLogProps = { workspace: Workspace uuid: string generatedAt: Date - providerId: number - providerType: Providers - model: string - config: PartialConfig + providerId?: number + providerType?: Providers + model?: string + config?: PartialConfig messages: Message[] - usage: LanguageModelUsage - duration: number + usage?: LanguageModelUsage + duration?: number source: LogSources finishReason?: StreamConsumeReturn['finishReason'] apiKeyId?: number @@ -60,10 +60,15 @@ export async function createProviderLog( return await Transaction.call(async (trx) => { const cost = costInMillicents ?? - Math.floor( - estimateCost({ provider: providerType, model, usage }) * - TO_MILLICENTS_FACTOR, - ) + (providerType && model && usage + ? Math.floor( + estimateCost({ + provider: providerType!, + model: model!, + usage: usage!, + }) * TO_MILLICENTS_FACTOR, + ) + : undefined) const inserts = await trx .insert(providerLogs) .values({ @@ -78,7 +83,11 @@ export async function createProviderLog( responseText, responseObject, toolCalls, - tokens: isNaN(usage.totalTokens) ? 0 : (usage.totalTokens ?? 0), + tokens: usage + ? isNaN(usage.totalTokens) + ? 0 + : (usage.totalTokens ?? 0) + : undefined, costInMillicents: cost, duration, source, @@ -88,8 +97,7 @@ export async function createProviderLog( .returning() const log = inserts[0]! as ProviderLog - await touchProviderApiKey(providerId, trx) - + if (providerId) await touchProviderApiKey(providerId, trx) if (apiKeyId) await touchApiKey(apiKeyId, trx) publisher.publishLater({ diff --git a/packages/core/src/tests/factories/documentLogs.ts b/packages/core/src/tests/factories/documentLogs.ts index 00f5e0d89..02a6e9cdc 100644 --- a/packages/core/src/tests/factories/documentLogs.ts +++ b/packages/core/src/tests/factories/documentLogs.ts @@ -120,7 +120,7 @@ export async function createDocumentLog({ const duration = Math.floor(Math.random() * 100) + - providerLogs.reduce((acc, log) => acc + log.duration, 0) + providerLogs.reduce((acc, log) => acc + (log?.duration ?? 0), 0) const documentLog = await ogCreateDocumentLog({ commit, diff --git a/packages/sdks/typescript/package.json b/packages/sdks/typescript/package.json index 299eb8bb9..624e781be 100644 --- a/packages/sdks/typescript/package.json +++ b/packages/sdks/typescript/package.json @@ -19,16 +19,24 @@ "prettier": "prettier --write \"**/*.{ts,tsx,md}\"" }, "type": "module", - "files": [ - "dist" - ], + "files": ["dist"], + "main": "./dist/index.cjs", + "module": "./dist/index.js", "exports": { ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "require": "./dist/index.cjs" + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.cjs" + } } }, + "types": { + "index.d.ts": "./dist/index.d.ts" + }, "dependencies": { "@t3-oss/env-core": "^0.10.1", "eventsource-parser": "^2.0.1", diff --git a/packages/sdks/typescript/src/index.ts b/packages/sdks/typescript/src/index.ts index 47ba6147f..31dfe8c9d 100644 --- a/packages/sdks/typescript/src/index.ts +++ b/packages/sdks/typescript/src/index.ts @@ -7,7 +7,11 @@ import { } from '@latitude-data/core/browser' import env from '$sdk/env' import { GatewayApiConfig, RouteResolver } from '$sdk/utils' -import { ApiErrorJsonResponse, LatitudeApiError } from '$sdk/utils/errors' +import { + ApiErrorCodes, + ApiErrorJsonResponse, + LatitudeApiError, +} from '$sdk/utils/errors' import { handleStream } from '$sdk/utils/handleStream' import { makeRequest } from '$sdk/utils/request' import { streamRun } from '$sdk/utils/streamRun' @@ -111,6 +115,45 @@ class Latitude { return syncRun(path, options) } + async log( + path: string, + messages: Message[], + { + response, + projectId, + versionUuid, + }: { + projectId?: number + versionUuid?: string + response?: string + } = {}, + ) { + projectId = projectId ?? this.options.projectId + if (!projectId) { + throw new Error('Project ID is required') + } + versionUuid = versionUuid ?? this.options.versionUuid + + const httpResponse = await makeRequest({ + method: 'POST', + handler: HandlerType.Log, + params: { projectId, versionUuid }, + body: { path, messages, response }, + options: this.options, + }) + + if (!httpResponse.ok) { + throw new LatitudeApiError({ + status: httpResponse.status, + message: httpResponse.statusText, + serverResponse: await httpResponse.text(), + errorCode: ApiErrorCodes.HTTPException, + }) + } + + return await httpResponse.json() + } + async chat( uuid: string, messages: Message[], diff --git a/packages/sdks/typescript/src/utils/errors.ts b/packages/sdks/typescript/src/utils/errors.ts index 5fc6dc5a8..cb2e9e2b2 100644 --- a/packages/sdks/typescript/src/utils/errors.ts +++ b/packages/sdks/typescript/src/utils/errors.ts @@ -16,7 +16,7 @@ function getErrorMessage({ message: string errorCode: ApiResponseCode }) { - const httpExeception = ApiErrorCodes.HTTPExeption + const httpExeception = ApiErrorCodes.HTTPException const internalServerError = ApiErrorCodes.InternalServerError const isUnexpectedError = errorCode === httpExeception || errorCode === internalServerError diff --git a/packages/sdks/typescript/src/utils/index.ts b/packages/sdks/typescript/src/utils/index.ts index 4e67203ff..6c5b58e9b 100644 --- a/packages/sdks/typescript/src/utils/index.ts +++ b/packages/sdks/typescript/src/utils/index.ts @@ -2,6 +2,7 @@ import { ChatUrlParams, GetDocumentUrlParams, HandlerType, + LogUrlParams, RunUrlParams, UrlParams, } from '$sdk/utils/types' @@ -46,6 +47,8 @@ export class RouteResolver { return this.conversations().chat( (params as ChatUrlParams).conversationUuid, ) + case HandlerType.Log: + return this.documents(params as LogUrlParams).log default: throw new Error(`Unknown handler: ${handler satisfies never}`) } @@ -63,6 +66,7 @@ export class RouteResolver { return { run: `${base}/run`, document: (path: string) => `${base}/${path}`, + log: `${base}/logs`, } } diff --git a/packages/sdks/typescript/src/utils/types.ts b/packages/sdks/typescript/src/utils/types.ts index d04e4ceb7..d0c6015bd 100644 --- a/packages/sdks/typescript/src/utils/types.ts +++ b/packages/sdks/typescript/src/utils/types.ts @@ -27,6 +27,12 @@ type RunDocumentBodyParam = { type ChatBodyParams = { messages: Message[] } +export type LogUrlParams = RunUrlParams +type LogBodyParams = { + path: string + messages: Message[] + response?: string +} export type GetDocumentUrlParams = { projectId: number @@ -38,6 +44,7 @@ export enum HandlerType { Chat = 'chat', GetDocument = 'get-document', RunDocument = 'run-document', + Log = 'log', } export type UrlParams = T extends HandlerType.RunDocument @@ -46,14 +53,18 @@ export type UrlParams = T extends HandlerType.RunDocument ? GetDocumentUrlParams : T extends HandlerType.Chat ? ChatUrlParams - : never + : T extends HandlerType.Log + ? LogUrlParams + : never export type BodyParams = T extends HandlerType.RunDocument ? RunDocumentBodyParam : T extends HandlerType.Chat ? ChatBodyParams - : never + : T extends HandlerType.Log + ? LogBodyParams + : never export type StreamChainResponse = { conversation: Message[] diff --git a/packages/web-ui/src/ds/atoms/DropzoneInput/index.tsx b/packages/web-ui/src/ds/atoms/DropzoneInput/index.tsx index 979171007..eb71a14d3 100644 --- a/packages/web-ui/src/ds/atoms/DropzoneInput/index.tsx +++ b/packages/web-ui/src/ds/atoms/DropzoneInput/index.tsx @@ -56,7 +56,7 @@ export function DropzoneInput({
) { +}) { if (typeof children !== 'string') return children - return ( -

- {children} -

- ) + return {children} } function TooltipMessage({ error }: { error: string | undefined }) { @@ -155,10 +148,7 @@ function FormField({ {children} - {description && ( - {description} - )} - + {description && {description}} {errorStyle === 'inline' ? ( ) : null} diff --git a/packages/web-ui/src/ds/atoms/FormFieldGroup/index.tsx b/packages/web-ui/src/ds/atoms/FormFieldGroup/index.tsx index 9ac4ad069..20ca1d054 100644 --- a/packages/web-ui/src/ds/atoms/FormFieldGroup/index.tsx +++ b/packages/web-ui/src/ds/atoms/FormFieldGroup/index.tsx @@ -34,11 +34,7 @@ export function FormFieldGroup({ > {children}
- {description ? ( - - {description} - - ) : null} + {description ? {description} : null} ) } diff --git a/packages/web-ui/src/ds/atoms/Switch/index.tsx b/packages/web-ui/src/ds/atoms/Switch/index.tsx index 241462c84..ab9fde748 100644 --- a/packages/web-ui/src/ds/atoms/Switch/index.tsx +++ b/packages/web-ui/src/ds/atoms/Switch/index.tsx @@ -96,9 +96,7 @@ function SwitchInput({ ) : null} - {description && ( - {description} - )} + {description && {description}} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44e71b3e9..853339f35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,7 +79,7 @@ importers: version: 0.11.1(typescript@5.6.3)(zod@3.23.8) ai: specifier: ^3.2.42 - version: 3.4.23(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) + version: 3.4.23(openai@4.68.4(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) argon2: specifier: ^0.41.0 version: 0.41.1 @@ -250,9 +250,6 @@ importers: hono: specifier: ^4.5.3 version: 4.6.7 - jet-paths: - specifier: ^1.0.6 - version: 1.0.9 lodash-es: specifier: ^4.17.21 version: 4.17.21 @@ -353,7 +350,7 @@ importers: version: 0.10.1(typescript@5.6.3)(zod@3.23.8) ai: specifier: ^3.2.42 - version: 3.4.23(react@19.0.0-rc-5d19e1c8-20240923)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) + version: 3.4.23(openai@4.68.4(encoding@0.1.13)(zod@3.23.8))(react@19.0.0-rc-5d19e1c8-20240923)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) bullmq: specifier: ^5.8.5 version: 5.21.2 @@ -736,7 +733,7 @@ importers: version: 10.0.0 ai: specifier: ^3.4.7 - version: 3.4.23(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) + version: 3.4.23(openai@4.68.4(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8) argon2: specifier: ^0.41.0 version: 0.41.1 @@ -5220,6 +5217,10 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -5255,6 +5256,10 @@ packages: resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} engines: {node: '>= 14'} + agentkeepalive@4.5.0: + resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} + engines: {node: '>= 8.0.0'} + aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -6867,6 +6872,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventemitter3@4.0.7: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} @@ -7047,6 +7056,9 @@ packages: resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data-encoder@2.1.4: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} @@ -7059,6 +7071,10 @@ packages: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -7475,6 +7491,9 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -7825,9 +7844,6 @@ packages: resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} engines: {node: '>= 10.13.0'} - jet-paths@1.0.9: - resolution: {integrity: sha512-XxYKHYMwwhFr2azKPGXAV80RIdFwueJOilBTXH2ICwXSxD5vo6PUZsfufL+DVk+nZKf6Z2y+19kpwh8lJ81Gng==} - jiti@1.21.6: resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==} hasBin: true @@ -8955,6 +8971,15 @@ packages: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} + openai@4.68.4: + resolution: {integrity: sha512-LRinV8iU9VQplkr25oZlyrsYGPGasIwYN8KFMAAFTHHLHjHhejtJ5BALuLFrkGzY4wfbKhOhuT+7lcHZ+F3iEA==} + hasBin: true + peerDependencies: + zod: ^3.23.8 + peerDependenciesMeta: + zod: + optional: true + openapi-types@12.1.3: resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} @@ -10956,6 +10981,10 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + web-vitals@4.2.4: resolution: {integrity: sha512-r4DIlprAGwJ7YM11VZp4R884m0Vmgr6EAKe3P+kO0PPj3Unqyvv59rczf6UiGcb9Z8QxZVcqKNwv/g0WNdWwsw==} @@ -16501,6 +16530,11 @@ snapshots: abbrev@2.0.0: {} + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + optional: true + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -16534,6 +16568,11 @@ snapshots: transitivePeerDependencies: - supports-color + agentkeepalive@4.5.0: + dependencies: + humanize-ms: 1.2.1 + optional: true + aggregate-error@3.1.0: dependencies: clean-stack: 2.2.0 @@ -16544,7 +16583,7 @@ snapshots: clean-stack: 4.2.0 indent-string: 5.0.0 - ai@3.4.23(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8): + ai@3.4.23(openai@4.68.4(encoding@0.1.13)(zod@3.23.8))(react@18.3.1)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8): dependencies: '@ai-sdk/provider': 0.0.26 '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) @@ -16560,6 +16599,7 @@ snapshots: secure-json-parse: 2.7.0 zod-to-json-schema: 3.23.5(zod@3.23.8) optionalDependencies: + openai: 4.68.4(encoding@0.1.13)(zod@3.23.8) react: 18.3.1 sswr: 2.1.0(svelte@4.2.19) svelte: 4.2.19 @@ -16568,7 +16608,7 @@ snapshots: - solid-js - vue - ai@3.4.23(react@19.0.0-rc-5d19e1c8-20240923)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8): + ai@3.4.23(openai@4.68.4(encoding@0.1.13)(zod@3.23.8))(react@19.0.0-rc-5d19e1c8-20240923)(sswr@2.1.0(svelte@4.2.19))(svelte@4.2.19)(vue@3.5.12(typescript@5.6.3))(zod@3.23.8): dependencies: '@ai-sdk/provider': 0.0.26 '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) @@ -16584,6 +16624,7 @@ snapshots: secure-json-parse: 2.7.0 zod-to-json-schema: 3.23.5(zod@3.23.8) optionalDependencies: + openai: 4.68.4(encoding@0.1.13)(zod@3.23.8) react: 19.0.0-rc-5d19e1c8-20240923 sswr: 2.1.0(svelte@4.2.19) svelte: 4.2.19 @@ -18253,6 +18294,9 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: + optional: true + eventemitter3@4.0.7: {} events@1.1.1: {} @@ -18474,6 +18518,9 @@ snapshots: cross-spawn: 7.0.3 signal-exit: 4.1.0 + form-data-encoder@1.7.2: + optional: true + form-data-encoder@2.1.4: {} form-data@4.0.1: @@ -18484,6 +18531,12 @@ snapshots: format@0.2.2: {} + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + optional: true + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -19110,6 +19163,11 @@ snapshots: human-signals@5.0.0: {} + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + optional: true + iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2 @@ -19426,8 +19484,6 @@ snapshots: merge-stream: 2.0.0 supports-color: 8.1.1 - jet-paths@1.0.9: {} - jiti@1.21.6: {} jju@1.4.0: {} @@ -21191,6 +21247,21 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + openai@4.68.4(encoding@0.1.13)(zod@3.23.8): + dependencies: + '@types/node': 18.19.59 + '@types/node-fetch': 2.6.11 + abort-controller: 3.0.0 + agentkeepalive: 4.5.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0(encoding@0.1.13) + optionalDependencies: + zod: 3.23.8 + transitivePeerDependencies: + - encoding + optional: true + openapi-types@12.1.3: {} optionator@0.9.4: @@ -23886,6 +23957,9 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: + optional: true + web-vitals@4.2.4: {} webidl-conversions@3.0.1: {}