diff --git a/apps/web/src/actions/providerLogs/fetch.ts b/apps/web/src/actions/providerLogs/fetch.ts index fa81a1beb..e8645f71a 100644 --- a/apps/web/src/actions/providerLogs/fetch.ts +++ b/apps/web/src/actions/providerLogs/fetch.ts @@ -1,6 +1,7 @@ 'use server' import { ProviderLogsRepository } from '@latitude-data/core/repositories' +import providerLogPresenter from '$/presenters/providerLogPresenter' import { z } from 'zod' import { authProcedure } from '../procedures' @@ -30,7 +31,7 @@ export const getProviderLogsAction = authProcedure result = await scope.findAll({ limit: 1000 }).then((r) => r.unwrap()) } - return result + return result.map(providerLogPresenter) }) export const getProviderLogAction = authProcedure @@ -39,5 +40,8 @@ export const getProviderLogAction = authProcedure .handler(async ({ input, ctx }) => { const { providerLogId } = input const scope = new ProviderLogsRepository(ctx.workspace.id) - return await scope.find(providerLogId).then((r) => r.unwrap()) + return await scope + .find(providerLogId) + .then((r) => r.unwrap()) + .then(providerLogPresenter) }) diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx index fb3a1bf15..0a88fd8ab 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx @@ -50,7 +50,7 @@ export default function Playground({ setInputs({ messages: JSON.stringify(formatConversation(providerLog)), context: JSON.stringify(formatContext(providerLog)), - response: providerLog.responseText, + response: providerLog.response, prompt: '', parameters: '', config: '', diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx index 10e03fbe1..95634cf28 100644 --- a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx +++ b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx @@ -4,7 +4,7 @@ import { useState } from 'react' import { capitalize } from 'lodash-es' import { MessageContent, TextContent } from '@latitude-data/compiler' -import { HEAD_COMMIT } from '@latitude-data/core/browser' +import { HEAD_COMMIT, ProviderLogDto } from '@latitude-data/core/browser' import { Badge, Button, @@ -193,7 +193,9 @@ const ProviderLogMessages = ({ providerLogUuid?: string }) => { const { data } = useProviderLogs({ documentUuid }) - const providerLog = data?.find((log) => log.uuid === providerLogUuid) + const providerLog = data?.find( + (log) => log.uuid === providerLogUuid, + ) as ProviderLogDto if (!providerLog) { return (
@@ -225,7 +227,7 @@ const ProviderLogMessages = ({ Assistant
- {printMessageContent(providerLog.responseText)} + {printMessageContent(providerLog.response)}
diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat.tsx index 34c0fc149..cd3ec4c7f 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat.tsx @@ -1,6 +1,7 @@ import { useCallback, useContext, useEffect, useRef, useState } from 'react' import { + AssistantMessage, ContentType, Conversation, Message as ConversationMessage, @@ -95,8 +96,8 @@ export default function Chat({ const { event, data } = serverEvent const hasMessages = 'messages' in data if (hasMessages) { - data.messages.forEach(addMessageToConversation) - messagesCount += data.messages.length + data.messages!.forEach(addMessageToConversation) + messagesCount += data.messages!.length } switch (event) { @@ -109,6 +110,7 @@ export default function Chat({ } else if (data.type === ChainEventTypes.Error) { setError(new Error(data.error.message)) } + break } @@ -161,17 +163,17 @@ export default function Chat({ const { event, data } = serverEvent - const hasMessages = 'messages' in data - - if (hasMessages) { - data.messages.forEach(addMessageToConversation) - } - switch (event) { case StreamEventTypes.Latitude: { if (data.type === ChainEventTypes.Error) { setError(new Error(data.error.message)) + } else if (data.type === ChainEventTypes.Complete) { + addMessageToConversation({ + role: MessageRole.assistant, + content: data.response.text, + } as AssistantMessage) } + break } diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx index cd28fe705..3d76def6c 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx @@ -1,20 +1,20 @@ import { useMemo } from 'react' import { AssistantMessage, Message, MessageRole } from '@latitude-data/compiler' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { MessageList } from '@latitude-data/web-ui' export function EvaluationResultMessages({ providerLog, }: { - providerLog?: ProviderLog + providerLog?: ProviderLogDto }) { const messages = useMemo(() => { if (!providerLog) return [] as Message[] const responseMessage = { role: MessageRole.assistant, - content: providerLog.responseText, + content: providerLog.response, toolCalls: providerLog.toolCalls, } as AssistantMessage diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Metadata.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Metadata.tsx index a27034b27..0b8f29bec 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Metadata.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Metadata.tsx @@ -1,6 +1,6 @@ import { ReactNode } from 'react' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories' import { ClickToCopy, @@ -48,7 +48,7 @@ export function EvaluationResultMetadata({ providerLog, }: { evaluationResult: EvaluationResultWithMetadata - providerLog?: ProviderLog + providerLog?: ProviderLogDto }) { const { data: providers, isLoading: providersLoading } = useProviderApiKeys() diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/index.tsx index 3f9d98873..0dfa5b007 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/index.tsx @@ -2,7 +2,7 @@ import { useState } from 'react' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories' import { TabSelector } from '@latitude-data/web-ui' @@ -14,7 +14,7 @@ export function EvaluationResultInfo({ providerLog, }: { evaluationResult: EvaluationResultWithMetadata - providerLog?: ProviderLog + providerLog?: ProviderLogDto }) { const [selectedTab, setSelectedTab] = useState('metadata') return ( diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Messages.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Messages.tsx index 899844a56..02ea269fb 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Messages.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Messages.tsx @@ -1,13 +1,13 @@ import { useMemo } from 'react' import { AssistantMessage, Message, MessageRole } from '@latitude-data/compiler' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { MessageList } from '@latitude-data/web-ui' export function DocumentLogMessages({ providerLogs, }: { - providerLogs?: ProviderLog[] + providerLogs?: ProviderLogDto[] }) { const messages = useMemo(() => { const lastLog = providerLogs?.[providerLogs.length - 1] @@ -15,7 +15,7 @@ export function DocumentLogMessages({ const responseMessage = { role: MessageRole.assistant, - content: lastLog.responseText, + content: lastLog.response, toolCalls: lastLog.toolCalls, } as AssistantMessage diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx index 070786779..f439d6c50 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx @@ -1,6 +1,6 @@ import { ReactNode, useMemo } from 'react' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { DocumentLogWithMetadata } from '@latitude-data/core/repositories' import { ClickToCopy, @@ -48,7 +48,7 @@ export function DocumentLogMetadata({ providerLogs, }: { documentLog: DocumentLogWithMetadata - providerLogs?: ProviderLog[] + providerLogs?: ProviderLogDto[] }) { const { data: providers, isLoading: providersLoading } = useProviderApiKeys() const lastProviderLog = useMemo( diff --git a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/index.tsx b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/index.tsx index 5e8380a5f..8a5eb676b 100644 --- a/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/index.tsx +++ b/apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/index.tsx @@ -2,7 +2,7 @@ import { useState } from 'react' -import { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { DocumentLogWithMetadata } from '@latitude-data/core/repositories' import { TabSelector } from '@latitude-data/web-ui' @@ -14,7 +14,7 @@ export function DocumentLogInfo({ providerLogs, }: { documentLog: DocumentLogWithMetadata - providerLogs?: ProviderLog[] + providerLogs?: ProviderLogDto[] }) { const [selectedTab, setSelectedTab] = useState('metadata') return ( diff --git a/apps/web/src/presenters/providerLogPresenter.ts b/apps/web/src/presenters/providerLogPresenter.ts new file mode 100644 index 000000000..1e7ebaaf4 --- /dev/null +++ b/apps/web/src/presenters/providerLogPresenter.ts @@ -0,0 +1,16 @@ +import { omit } from 'lodash-es' + +import type { ProviderLog, ProviderLogDto } from '@latitude-data/core/browser' + +export default function providerLogPresenter( + providerLog: ProviderLog, +): ProviderLogDto { + return { + ...omit(providerLog, 'responseText', 'responseObject'), + response: + providerLog.responseText || + (providerLog.responseObject + ? JSON.stringify(providerLog.responseObject) + : ''), + } +} diff --git a/apps/web/src/stores/providerLogs.ts b/apps/web/src/stores/providerLogs.ts index dda476c97..d835a2277 100644 --- a/apps/web/src/stores/providerLogs.ts +++ b/apps/web/src/stores/providerLogs.ts @@ -2,7 +2,7 @@ import { compact } from 'lodash-es' -import type { ProviderLog } from '@latitude-data/core/browser' +import { ProviderLogDto } from '@latitude-data/core/browser' import { useToast } from '@latitude-data/web-ui' import { getProviderLogAction, @@ -19,10 +19,10 @@ export default function useProviderLogs( ) { const { toast } = useToast() const { - data = undefined, + data = [], isLoading, error: swrError, - } = useSWR( + } = useSWR( compact(['providerLogs', documentUuid, documentLogUuid]), async () => { const [data, error] = await getProviderLogsAction({ @@ -38,10 +38,11 @@ export default function useProviderLogs( description: error.formErrors?.[0] || error.message, variant: 'destructive', }) - return undefined + + return [] } - return data as ProviderLog[] + return data }, opts, ) @@ -62,7 +63,7 @@ export function useProviderLog( data = undefined, isLoading, error: swrError, - } = useSWR( + } = useSWR( compact(['providerLog', providerLogId]), async () => { if (!providerLogId) return undefined @@ -82,7 +83,7 @@ export function useProviderLog( return undefined } - return data as ProviderLog + return data }, opts, ) diff --git a/packages/core/drizzle/0052_famous_daimon_hellstrom.sql b/packages/core/drizzle/0052_famous_daimon_hellstrom.sql new file mode 100644 index 000000000..891e4c380 --- /dev/null +++ b/packages/core/drizzle/0052_famous_daimon_hellstrom.sql @@ -0,0 +1,3 @@ +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP DEFAULT;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP NOT NULL;--> statement-breakpoint +ALTER TABLE "latitude"."provider_logs" ADD COLUMN "response_object" jsonb; \ No newline at end of file diff --git a/packages/core/drizzle/meta/0052_snapshot.json b/packages/core/drizzle/meta/0052_snapshot.json new file mode 100644 index 000000000..7760ae01f --- /dev/null +++ b/packages/core/drizzle/meta/0052_snapshot.json @@ -0,0 +1,2194 @@ +{ + "id": "9b5a6c7c-2af0-4e63-b502-4f7ae011066d", + "prevId": "eb7ce322-fe62-4bd3-b181-5386c6db6c01", + "version": "7", + "dialect": "postgresql", + "tables": { + "latitude.users": { + "name": "users", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "encrypted_password": { + "name": "encrypted_password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + } + }, + "latitude.sessions": { + "name": "sessions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.workspaces": { + "name": "workspaces", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "workspaces_creator_id_users_id_fk": { + "name": "workspaces_creator_id_users_id_fk", + "tableFrom": "workspaces", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "creator_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.memberships": { + "name": "memberships", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "invitation_token": { + "name": "invitation_token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "confirmed_at": { + "name": "confirmed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_workspace_id_user_id_index": { + "name": "memberships_workspace_id_user_id_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_invitation_token_index": { + "name": "memberships_invitation_token_index", + "columns": [ + { + "expression": "invitation_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_workspace_id_workspaces_id_fk": { + "name": "memberships_workspace_id_workspaces_id_fk", + "tableFrom": "memberships", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_user_id_users_id_fk": { + "name": "memberships_user_id_users_id_fk", + "tableFrom": "memberships", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "memberships_invitation_token_unique": { + "name": "memberships_invitation_token_unique", + "nullsNotDistinct": false, + "columns": [ + "invitation_token" + ] + } + } + }, + "latitude.api_keys": { + "name": "api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_id_idx": { + "name": "workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_keys_workspace_id_workspaces_id_fk": { + "name": "api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_keys_token_unique": { + "name": "api_keys_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.projects": { + "name": "projects", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspace_idx": { + "name": "workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "projects_workspace_id_workspaces_id_fk": { + "name": "projects_workspace_id_workspaces_id_fk", + "tableFrom": "projects", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.commits": { + "name": "commits", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "title": { + "name": "title", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "project_commit_order_idx": { + "name": "project_commit_order_idx", + "columns": [ + { + "expression": "merged_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "commits_project_id_projects_id_fk": { + "name": "commits_project_id_projects_id_fk", + "tableFrom": "commits", + "tableTo": "projects", + "schemaTo": "latitude", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "commits_user_id_users_id_fk": { + "name": "commits_user_id_users_id_fk", + "tableFrom": "commits", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "commits_uuid_unique": { + "name": "commits_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + }, + "unique_commit_version": { + "name": "unique_commit_version", + "nullsNotDistinct": false, + "columns": [ + "version", + "project_id" + ] + } + } + }, + "latitude.document_versions": { + "name": "document_versions", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "document_versions_commit_id_commits_id_fk": { + "name": "document_versions_commit_id_commits_id_fk", + "tableFrom": "document_versions", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "unique_document_uuid_commit_id": { + "name": "unique_document_uuid_commit_id", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "commit_id" + ] + }, + "unique_path_commit_id_deleted_at": { + "name": "unique_path_commit_id_deleted_at", + "nullsNotDistinct": false, + "columns": [ + "path", + "commit_id", + "deleted_at" + ] + } + } + }, + "latitude.provider_api_keys": { + "name": "provider_api_keys", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "provider", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "provider_apikeys_workspace_id_idx": { + "name": "provider_apikeys_workspace_id_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_name_idx": { + "name": "provider_apikeys_name_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_apikeys_user_id_idx": { + "name": "provider_apikeys_user_id_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_api_keys_author_id_users_id_fk": { + "name": "provider_api_keys_author_id_users_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_api_keys_workspace_id_workspaces_id_fk": { + "name": "provider_api_keys_workspace_id_workspaces_id_fk", + "tableFrom": "provider_api_keys", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_apikeys_token_provider_unique": { + "name": "provider_apikeys_token_provider_unique", + "nullsNotDistinct": false, + "columns": [ + "token", + "provider", + "workspace_id" + ] + } + } + }, + "latitude.document_logs": { + "name": "document_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "commit_id": { + "name": "commit_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resolved_content": { + "name": "resolved_content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "parameters": { + "name": "parameters", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "custom_identifier": { + "name": "custom_identifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "document_log_uuid_idx": { + "name": "document_log_uuid_idx", + "columns": [ + { + "expression": "document_uuid", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "commit_id_idx": { + "name": "commit_id_idx", + "columns": [ + { + "expression": "commit_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "document_logs_commit_id_commits_id_fk": { + "name": "document_logs_commit_id_commits_id_fk", + "tableFrom": "document_logs", + "tableTo": "commits", + "schemaTo": "latitude", + "columnsFrom": [ + "commit_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "document_logs_uuid_unique": { + "name": "document_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.provider_logs": { + "name": "provider_logs", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_log_uuid": { + "name": "document_log_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "messages": { + "name": "messages", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "response_object": { + "name": "response_object", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_text": { + "name": "response_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "json", + "primaryKey": false, + "notNull": true, + "default": "'[]'::json" + }, + "tokens": { + "name": "tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "cost_in_millicents": { + "name": "cost_in_millicents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "duration": { + "name": "duration", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "log_source", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "apiKeyId": { + "name": "apiKeyId", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "generated_at": { + "name": "generated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_logs_provider_id_provider_api_keys_id_fk": { + "name": "provider_logs_provider_id_provider_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "provider_api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "provider_logs_apiKeyId_api_keys_id_fk": { + "name": "provider_logs_apiKeyId_api_keys_id_fk", + "tableFrom": "provider_logs", + "tableTo": "api_keys", + "schemaTo": "latitude", + "columnsFrom": [ + "apiKeyId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_logs_uuid_unique": { + "name": "provider_logs_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.datasets": { + "name": "datasets", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "csv_delimiter": { + "name": "csv_delimiter", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "file_key": { + "name": "file_key", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "file_metadata": { + "name": "file_metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "datasets_workspace_idx": { + "name": "datasets_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_author_idx": { + "name": "datasets_author_idx", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "datasets_workspace_id_name_index": { + "name": "datasets_workspace_id_name_index", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "datasets_workspace_id_workspaces_id_fk": { + "name": "datasets_workspace_id_workspaces_id_fk", + "tableFrom": "datasets", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "datasets_author_id_users_id_fk": { + "name": "datasets_author_id_users_id_fk", + "tableFrom": "datasets", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations": { + "name": "evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "uuid": { + "name": "uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata_id": { + "name": "metadata_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "metadata_type", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "workspace_id": { + "name": "workspace_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_workspace_idx": { + "name": "evaluation_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "evaluation_metadata_idx": { + "name": "evaluation_metadata_idx", + "columns": [ + { + "expression": "metadata_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "metadata_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluations_workspace_id_workspaces_id_fk": { + "name": "evaluations_workspace_id_workspaces_id_fk", + "tableFrom": "evaluations", + "tableTo": "workspaces", + "schemaTo": "latitude", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "evaluations_uuid_unique": { + "name": "evaluations_uuid_unique", + "nullsNotDistinct": false, + "columns": [ + "uuid" + ] + } + } + }, + "latitude.llm_as_judge_evaluation_metadatas": { + "name": "llm_as_judge_evaluation_metadatas", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "metadata_type": { + "name": "metadata_type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true, + "default": "'llm_as_judge'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template_id": { + "name": "template_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "llm_as_judge_evaluation_metadatas_template_id_idx": { + "name": "llm_as_judge_evaluation_metadatas_template_id_idx", + "columns": [ + { + "expression": "template_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk": { + "name": "llm_as_judge_evaluation_metadatas_template_id_evaluations_templates_id_fk", + "tableFrom": "llm_as_judge_evaluation_metadatas", + "tableTo": "evaluations_templates", + "schemaTo": "latitude", + "columnsFrom": [ + "template_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.connected_evaluations": { + "name": "connected_evaluations", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "document_uuid": { + "name": "document_uuid", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "evaluation_mode": { + "name": "evaluation_mode", + "type": "evaluation_mode_enum", + "typeSchema": "latitude", + "primaryKey": false, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "connected_evaluations_evaluation_idx": { + "name": "connected_evaluations_evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "connected_evaluations_evaluation_id_evaluations_id_fk": { + "name": "connected_evaluations_evaluation_id_evaluations_id_fk", + "tableFrom": "connected_evaluations", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "connected_evaluations_unique_idx": { + "name": "connected_evaluations_unique_idx", + "nullsNotDistinct": false, + "columns": [ + "document_uuid", + "evaluation_id" + ] + } + } + }, + "latitude.evaluation_results": { + "name": "evaluation_results", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "evaluation_id": { + "name": "evaluation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "document_log_id": { + "name": "document_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_log_id": { + "name": "provider_log_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "resultable_type": { + "name": "resultable_type", + "type": "evaluation_result_types", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "resultable_id": { + "name": "resultable_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "evaluation_idx": { + "name": "evaluation_idx", + "columns": [ + { + "expression": "evaluation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "document_log_idx": { + "name": "document_log_idx", + "columns": [ + { + "expression": "document_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_log_idx": { + "name": "provider_log_idx", + "columns": [ + { + "expression": "provider_log_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "resultable_idx": { + "name": "resultable_idx", + "columns": [ + { + "expression": "resultable_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "resultable_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "evaluation_results_evaluation_id_evaluations_id_fk": { + "name": "evaluation_results_evaluation_id_evaluations_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "evaluations", + "schemaTo": "latitude", + "columnsFrom": [ + "evaluation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "evaluation_results_document_log_id_document_logs_id_fk": { + "name": "evaluation_results_document_log_id_document_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "document_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "document_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "evaluation_results_provider_log_id_provider_logs_id_fk": { + "name": "evaluation_results_provider_log_id_provider_logs_id_fk", + "tableFrom": "evaluation_results", + "tableTo": "provider_logs", + "schemaTo": "latitude", + "columnsFrom": [ + "provider_log_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_templates": { + "name": "evaluations_templates", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "configuration": { + "name": "configuration", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "evaluations_templates_category_evaluations_template_categories_id_fk": { + "name": "evaluations_templates_category_evaluations_template_categories_id_fk", + "tableFrom": "evaluations_templates", + "tableTo": "evaluations_template_categories", + "schemaTo": "latitude", + "columnsFrom": [ + "category" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluations_template_categories": { + "name": "evaluations_template_categories", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.magic_link_tokens": { + "name": "magic_link_tokens", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "expired_at": { + "name": "expired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "magic_link_tokens_user_id_users_id_fk": { + "name": "magic_link_tokens_user_id_users_id_fk", + "tableFrom": "magic_link_tokens", + "tableTo": "users", + "schemaTo": "latitude", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "magic_link_tokens_token_unique": { + "name": "magic_link_tokens_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + } + }, + "latitude.events": { + "name": "events", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "event_type_idx": { + "name": "event_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_numbers": { + "name": "evaluation_resultable_numbers", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_texts": { + "name": "evaluation_resultable_texts", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "latitude.evaluation_resultable_booleans": { + "name": "evaluation_resultable_booleans", + "schema": "latitude", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "result": { + "name": "result", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": { + "latitude.provider": { + "name": "provider", + "schema": "latitude", + "values": [ + "openai", + "anthropic", + "groq", + "mistral", + "azure", + "google" + ] + }, + "latitude.log_source": { + "name": "log_source", + "schema": "latitude", + "values": [ + "playground", + "api", + "evaluation" + ] + }, + "latitude.metadata_type": { + "name": "metadata_type", + "schema": "latitude", + "values": [ + "llm_as_judge" + ] + }, + "public.evaluation_result_types": { + "name": "evaluation_result_types", + "schema": "public", + "values": [ + "evaluation_resultable_booleans", + "evaluation_resultable_texts", + "evaluation_resultable_numbers" + ] + } + }, + "schemas": { + "latitude": "latitude" + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/core/drizzle/meta/_journal.json b/packages/core/drizzle/meta/_journal.json index 2b594cf46..d25411418 100644 --- a/packages/core/drizzle/meta/_journal.json +++ b/packages/core/drizzle/meta/_journal.json @@ -365,6 +365,13 @@ "when": 1726148717688, "tag": "0051_colorful_lightspeed", "breakpoints": true + }, + { + "idx": 52, + "version": "7", + "when": 1726227047167, + "tag": "0052_famous_daimon_hellstrom", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/core/package.json b/packages/core/package.json index 9101c9ee1..25b6c2e5e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -26,7 +26,7 @@ "db:migrate:test": "NODE_ENV=test pnpm run db:migrate", "db:drop": "drizzle-kit drop", "db:check": "drizzle-kit check", - "db:studio": "drizzle-kit studio --port 3002", + "db:studio": "drizzle-kit studio --port 3003", "lint": "eslint src", "tc": "tsc --noEmit", "test": "pnpm run db:migrate:test && vitest run --pool=forks", @@ -48,6 +48,7 @@ "@latitude-data/typescript-config": "workspace:*", "@sindresorhus/slugify": "^2.2.1", "@t3-oss/env-core": "^0.11.1", + "@types/json-schema": "^7.0.15", "@types/lodash-es": "^4.17.12", "@types/node": "^22.5.0", "@types/pg": "^8.11.6", @@ -61,6 +62,7 @@ "eslint-plugin-drizzle": "^0.2.3", "flydrive": "^1.1.0", "jose": "^5.8.0", + "json-schema": "^0.4.0", "lodash-es": "^4.17.21", "pg": "^8.12.0", "pg-transactional-tests": "^1.0.9", @@ -88,12 +90,13 @@ "@latitude-data/mailers": "workspace:^", "@sindresorhus/slugify": "^2.2.1", "@t3-oss/env-core": "^0.11.1", - "ai": "^3.2.42", + "ai": "^3.3.3", "argon2": "^0.41.0", "csv-parse": "^5.5.6", "drizzle-orm": "^0.33.0", "flydrive": "^1.1.0", "jose": "^5.8.0", + "json-schema": "^0.4.0", "lodash-es": "^4.17.21", "pg": "^8.12.0", "react": "^18.3.1", diff --git a/packages/core/src/browser.ts b/packages/core/src/browser.ts index d89cd8aa5..4c4f5accb 100644 --- a/packages/core/src/browser.ts +++ b/packages/core/src/browser.ts @@ -1,3 +1,4 @@ export * from './constants' export * from './schema/types' export * from './websockets/constants' +export * from './helpers' diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 911d063f3..d2d02f29b 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -2,7 +2,12 @@ import type { Message as CompilerMessage, ToolCall, } from '@latitude-data/compiler' -import { CompletionTokenUsage, CoreTool, TextStreamPart } from 'ai' +import { + CompletionTokenUsage, + CoreTool, + ObjectStreamPart, + TextStreamPart, +} from 'ai' import { Config } from './services/ai' @@ -34,14 +39,24 @@ export const HELP_CENTER = { commitVersions: `${LATITUDE_DOCS_URL}/not-found`, } -export type ChainStepCallResponse = { +export type ChainStepTextResponse = { text: string usage: CompletionTokenUsage + toolCalls: ToolCall[] +} +export type ChainStepObjectResponse = { + object: any + text: string + usage: CompletionTokenUsage +} + +export type ChainTextResponse = ChainStepTextResponse & { + documentLogUuid: string } -export type ChainCallResponse = ChainStepCallResponse & { +export type ChainObjectResponse = ChainStepObjectResponse & { documentLogUuid: string - toolCalls: ToolCall[] } +export type ChainCallResponse = ChainTextResponse | ChainObjectResponse export enum Providers { OpenAI = 'openai', @@ -69,7 +84,9 @@ export enum ChainEventTypes { StepComplete = 'chain-step-complete', } -type ProviderData = TextStreamPart> +export type ProviderData = + | TextStreamPart> + | ObjectStreamPart> export type ProviderDataType = ProviderData['type'] type LatitudeEventData = @@ -81,12 +98,13 @@ type LatitudeEventData = } | { type: ChainEventTypes.StepComplete - response: ChainStepCallResponse + response: ChainCallResponse } | { type: ChainEventTypes.Complete config: Config - messages: Message[] + messages?: Message[] + object?: any response: ChainCallResponse } | { diff --git a/packages/core/src/events/handlers/createEvaluationResultJob.ts b/packages/core/src/events/handlers/createEvaluationResultJob.ts index 09ffa92cc..7126e4ccc 100644 --- a/packages/core/src/events/handlers/createEvaluationResultJob.ts +++ b/packages/core/src/events/handlers/createEvaluationResultJob.ts @@ -1,4 +1,5 @@ import { EvaluationRunEvent } from '.' +import { ChainObjectResponse } from '../../constants' import { unsafelyFindDocumentLogByUuid, unsafelyFindEvaluation, @@ -28,6 +29,6 @@ export const createEvaluationResultJob = async ({ evaluation, documentLog, providerLog, - result: response.text, + result: (response as ChainObjectResponse).object, }).then((r) => r.unwrap()) } diff --git a/packages/core/src/events/handlers/createProviderLogJob.ts b/packages/core/src/events/handlers/createProviderLogJob.ts index 37dce0bbc..de923aca5 100644 --- a/packages/core/src/events/handlers/createProviderLogJob.ts +++ b/packages/core/src/events/handlers/createProviderLogJob.ts @@ -1,10 +1,15 @@ -import { AIProviderCallCompleted } from '.' -import { createProviderLog } from '../../services/providerLogs' +import { + createProviderLog, + CreateProviderLogProps, +} from '../../services/providerLogs' export const createProviderLogJob = async ({ - data: event, + data, }: { - data: AIProviderCallCompleted + data: Omit & { generatedAt: string } }) => { - await createProviderLog(event.data) + return await createProviderLog({ + ...data, + generatedAt: new Date(data.generatedAt), + }).then((r) => r.unwrap()) } diff --git a/packages/core/src/events/handlers/index.ts b/packages/core/src/events/handlers/index.ts index ca7d86c70..8bcab9afe 100644 --- a/packages/core/src/events/handlers/index.ts +++ b/packages/core/src/events/handlers/index.ts @@ -1,18 +1,11 @@ -import { ToolCall } from '@latitude-data/compiler' -import { CompletionTokenUsage } from 'ai' - import { ChainCallResponse, - LogSources, MagicLinkToken, Membership, - Message, - Providers, + ProviderLog, User, } from '../../browser' -import { PartialConfig } from '../../services/ai' import { createEvaluationResultJob } from './createEvaluationResultJob' -import { createProviderLogJob } from './createProviderLogJob' import { createDocumentLogJob } from './documentLogs/createJob' import { sendInvitationToUserJob } from './sendInvitationToUser' import { sendMagicLinkJob } from './sendMagicLinkHandler' @@ -31,24 +24,6 @@ export type EventHandler = ({ data: E }) => void -export type AIProviderCallCompleted = LatitudeEventGeneric< - 'aiProviderCallCompleted', - { - uuid: string - source: LogSources - generatedAt: Date - documentLogUuid?: string - providerId: number - providerType: Providers - model: string - config: PartialConfig - messages: Message[] - responseText: string - toolCalls?: ToolCall[] - usage: CompletionTokenUsage - duration: number - } -> export type MagicLinkTokenCreated = LatitudeEventGeneric< 'magicLinkTokenCreated', MagicLinkToken @@ -83,28 +58,33 @@ export type DocumentRunEvent = LatitudeEventGeneric< } > +export type ProviderLogCreatedEvent = LatitudeEventGeneric< + 'providerLogCreated', + ProviderLog +> + export type LatitudeEvent = | MembershipCreatedEvent | UserCreatedEvent | MagicLinkTokenCreated - | AIProviderCallCompleted | EvaluationRunEvent | DocumentRunEvent + | ProviderLogCreatedEvent export interface IEventsHandlers { - aiProviderCallCompleted: EventHandler[] magicLinkTokenCreated: EventHandler[] membershipCreated: EventHandler[] userCreated: EventHandler[] evaluationRun: EventHandler[] documentRun: EventHandler[] + providerLogCreated: EventHandler[] } export const EventHandlers: IEventsHandlers = { magicLinkTokenCreated: [sendMagicLinkJob], membershipCreated: [sendInvitationToUserJob], userCreated: [], - aiProviderCallCompleted: [createProviderLogJob], evaluationRun: [createEvaluationResultJob], documentRun: [createDocumentLogJob], + providerLogCreated: [], } as const diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts new file mode 100644 index 000000000..f679c2309 --- /dev/null +++ b/packages/core/src/helpers.ts @@ -0,0 +1,7 @@ +export function objectToString(object: any) { + try { + return JSON.stringify(object) + } catch (error) { + return 'Error: Provider returned an object that could not be stringified' + } +} diff --git a/packages/core/src/schema/models/providerLogs.ts b/packages/core/src/schema/models/providerLogs.ts index 0118367b1..097348cb8 100644 --- a/packages/core/src/schema/models/providerLogs.ts +++ b/packages/core/src/schema/models/providerLogs.ts @@ -4,11 +4,13 @@ import { bigserial, integer, json, + jsonb, text, timestamp, uuid, varchar, } from 'drizzle-orm/pg-core' +import { JSONSchema7 } from 'json-schema' import { LogSources } from '../../constants' import { PartialConfig } from '../../services/ai' @@ -36,7 +38,8 @@ export const providerLogs = latitudeSchema.table('provider_logs', { model: varchar('model'), config: json('config').$type().notNull(), messages: json('messages').$type().notNull(), - responseText: text('response_text').$type().notNull().default(''), + responseObject: jsonb('response_object').$type(), + responseText: text('response_text').$type(), toolCalls: json('tool_calls').$type().notNull().default([]), tokens: bigint('tokens', { mode: 'number' }).notNull(), costInMillicents: integer('cost_in_millicents').notNull().default(0), diff --git a/packages/core/src/schema/types.ts b/packages/core/src/schema/types.ts index a3009e2e1..5f51862e7 100644 --- a/packages/core/src/schema/types.ts +++ b/packages/core/src/schema/types.ts @@ -1,4 +1,3 @@ -import { ToolCall } from '@latitude-data/compiler' import { type InferSelectModel } from 'drizzle-orm' import { EvaluationResultableType } from '../constants' @@ -38,12 +37,7 @@ export type ApiKey = InferSelectModel export type Commit = InferSelectModel export type DocumentVersion = InferSelectModel export type Project = InferSelectModel -export type ProviderLog = InferSelectModel & { - // Typescript thinks these 2 are optional because they are in the schema - // but we add a default empty string and empty array to them - responseText: string - toolCalls: ToolCall[] -} +export type ProviderLog = InferSelectModel export type DocumentLog = InferSelectModel export type Evaluation = InferSelectModel export type ConnectedEvaluation = InferSelectModel @@ -80,3 +74,8 @@ export type EvaluationResultConfiguration = { export type EvaluationTemplateWithCategory = EvaluationTemplate & { category: string } + +export type ProviderLogDto = Omit< + ProviderLog, + 'responseText' | 'responseObject' +> & { response: string } diff --git a/packages/core/src/services/ai/helpers.ts b/packages/core/src/services/ai/helpers.ts new file mode 100644 index 000000000..fd5ffebc0 --- /dev/null +++ b/packages/core/src/services/ai/helpers.ts @@ -0,0 +1,96 @@ +import { createAnthropic } from '@ai-sdk/anthropic' +import { createAzure } from '@ai-sdk/azure' +import { createGoogleGenerativeAI } from '@ai-sdk/google' +import { createMistral } from '@ai-sdk/mistral' +import { createOpenAI } from '@ai-sdk/openai' +import { z } from 'zod' + +import { Providers } from '../../constants' + +export type Config = { + [key: string]: any + provider: string + model: string + azure?: { resourceName: string } +} + +export type PartialConfig = Omit + +const GROQ_API_URL = 'https://api.groq.com/openai/v1' + +export function createProvider({ + provider, + apiKey, + config, +}: { + provider: Providers + apiKey: string + config?: PartialConfig +}) { + switch (provider) { + case Providers.OpenAI: + return createOpenAI({ + apiKey, + // Needed for OpenAI to return token usage + compatibility: 'strict', + }) + case Providers.Groq: + return createOpenAI({ + apiKey, + compatibility: 'compatible', + baseURL: GROQ_API_URL, + }) + case Providers.Anthropic: + return createAnthropic({ + apiKey, + }) + case Providers.Mistral: + return createMistral({ + apiKey, + }) + case Providers.Azure: + return createAzure({ + apiKey, + ...(config?.azure ?? {}), + }) + case Providers.Google: + return createGoogleGenerativeAI({ + apiKey, + ...(config?.google ?? {}), + }) + default: + throw new Error(`Provider ${provider} not supported`) + } +} + +export function validateConfig(config: Record): Config { + const configSchema = z + .object({ + model: z.string(), + provider: z.string(), + google: z + .object({ + structuredOutputs: z.boolean().optional(), + cachedContent: z.string().optional(), + safetySettings: z + .array( + z + .object({ + category: z.string().optional(), // TODO: can be an enum + threshold: z.string().optional(), // TODO: can be an enum + }) + .optional(), + ) + .optional(), + }) + .optional(), + azure: z + .object({ + resourceName: z.string(), + }) + .optional(), + }) + .catchall(z.unknown()) + + return configSchema.parse(config) +} diff --git a/packages/core/src/services/ai/index.ts b/packages/core/src/services/ai/index.ts index ed2627ddb..96904c102 100644 --- a/packages/core/src/services/ai/index.ts +++ b/packages/core/src/services/ai/index.ts @@ -1,22 +1,23 @@ -import { createAnthropic } from '@ai-sdk/anthropic' -import { createAzure } from '@ai-sdk/azure' -import { createGoogleGenerativeAI } from '@ai-sdk/google' -import { createMistral } from '@ai-sdk/mistral' -import { createOpenAI } from '@ai-sdk/openai' import { Message } from '@latitude-data/compiler' +import { setupJobs } from '@latitude-data/jobs' import { CallWarning, CompletionTokenUsage, CoreMessage, FinishReason, + jsonSchema, + streamObject, streamText, } from 'ai' +import { JSONSchema7 } from 'json-schema' import { v4 } from 'uuid' -import { z } from 'zod' -import { LogSources, ProviderApiKey, Providers } from '../../browser' -import { publisher } from '../../events/publisher' -import { CreateProviderLogProps } from '../providerLogs/create' +import { LogSources, ProviderApiKey } from '../../browser' +import { + createProviderLog, + CreateProviderLogProps, +} from '../providerLogs/create' +import { createProvider, PartialConfig } from './helpers' export type FinishCallbackEvent = { finishReason: FinishReason @@ -38,85 +39,30 @@ export type FinishCallbackEvent = { } export type FinishCallback = (event: FinishCallbackEvent) => void -export type Config = { - [key: string]: any - provider: string - model: string - azure?: { resourceName: string } -} - -export type PartialConfig = Omit - -const GROQ_API_URL = 'https://api.groq.com/openai/v1' - -function createProvider({ - provider, - apiKey, +export type AILog = Omit +export async function ai({ + provider: apiProvider, + prompt, + messages, config, + documentLogUuid, + source, + schema = config.schema, + output = config.schema?.type || 'no-schema', + transactionalLogs = false, + onFinish, }: { - provider: Providers - apiKey: string - config?: PartialConfig + provider: ProviderApiKey + config: PartialConfig + messages: Message[] + documentLogUuid?: string + prompt?: string + source: LogSources + schema?: JSONSchema7 + output?: 'object' | 'array' | 'no-schema' + transactionalLogs?: boolean + onFinish?: FinishCallback }) { - switch (provider) { - case Providers.OpenAI: - return createOpenAI({ - apiKey, - // Needed for OpenAI to return token usage - compatibility: 'strict', - }) - case Providers.Groq: - return createOpenAI({ - apiKey, - compatibility: 'compatible', - baseURL: GROQ_API_URL, - }) - case Providers.Anthropic: - return createAnthropic({ - apiKey, - }) - case Providers.Mistral: - return createMistral({ - apiKey, - }) - case Providers.Azure: - return createAzure({ - apiKey, - ...(config?.azure ?? {}), - }) - case Providers.Google: - return createGoogleGenerativeAI({ - apiKey, - ...(config?.google ?? {}), - }) - default: - throw new Error(`Provider ${provider} not supported`) - } -} - -export type AILog = Omit -export async function ai( - { - provider: apiProvider, - prompt, - messages, - config, - documentLogUuid, - source, - }: { - provider: ProviderApiKey - config: PartialConfig - messages: Message[] - documentLogUuid?: string - prompt?: string - source: LogSources - }, - { - onFinish, - }: { - onFinish?: FinishCallback - } = {}, -) { const startTime = Date.now() const { provider, @@ -127,76 +73,87 @@ export async function ai( const model = config.model const m = createProvider({ provider, apiKey, config })(model) - const result = await streamText({ + const commonOptions = { model: m, prompt, messages: messages as CoreMessage[], - onFinish: (event) => { - publisher.publish({ - type: 'aiProviderCallCompleted', - data: { - uuid: v4(), - source, - generatedAt: new Date(), - documentLogUuid, - providerId, - providerType, - model, - config, - messages, - responseText: event.text, - toolCalls: event.toolCalls?.map((t) => ({ - id: t.toolCallId, - name: t.toolName, - arguments: t.args, - })), - usage: event.usage, - duration: Date.now() - startTime, - }, - }) + } + + const createFinishHandler = (isStructured: boolean) => async (event: any) => { + const commonData = { + uuid: v4(), + source, + generatedAt: new Date(), + documentLogUuid, + providerId, + providerType, + model, + config, + messages, + toolCalls: event.toolCalls?.map((t: any) => ({ + id: t.toolCallId, + name: t.toolName, + arguments: t.args, + })), + usage: event.usage, + duration: Date.now() - startTime, + } + + const payload = { + type: 'aiProviderCallCompleted', + data: { + ...commonData, + responseText: event.text, + responseObject: isStructured ? event.object : undefined, + }, + } - onFinish?.(event) - }, - }) + let providerLogUuid + if (transactionalLogs) { + const providerLog = await createProviderLog(payload.data).then((r) => + r.unwrap(), + ) + providerLogUuid = providerLog.uuid + } else { + await setupJobs().defaultQueue.jobs.enqueueCreateProviderLogJob( + payload.data, + ) + } - return { - fullStream: result.fullStream, - text: result.text, - usage: result.usage, - toolCalls: result.toolCalls, + onFinish?.({ ...event, providerLogUuid }) } -} -export function validateConfig(config: Record): Config { - const configSchema = z - .object({ - model: z.string(), - provider: z.string(), - google: z - .object({ - structuredOutputs: z.boolean().optional(), - cachedContent: z.string().optional(), - safetySettings: z - .array( - z - .object({ - category: z.string().optional(), // TODO: can be an enum - threshold: z.string().optional(), // TODO: can be an enum - }) - .optional(), - ) - .optional(), - }) - .optional(), - azure: z - .object({ - resourceName: z.string(), - }) - .optional(), + if (schema && output) { + const result = await streamObject({ + ...commonOptions, + schema: jsonSchema(schema), + // @ts-expect-error - output is vale but depending on the type of schema + // there might be a mismatch (e.g you pass an object schema but the + // output is "array"). Not really an issue we need to defend atm + output, + onFinish: createFinishHandler(true), + }) + + return { + fullStream: result.fullStream, + object: result.object, + usage: result.usage, + } + } else { + const result = await streamText({ + ...commonOptions, + onFinish: createFinishHandler(false), }) - .catchall(z.unknown()) - return configSchema.parse(config) + return { + fullStream: result.fullStream, + text: result.text, + usage: result.usage, + toolCalls: result.toolCalls, + } + } } export { estimateCost } from './estimateCost' +export { validateConfig } from './helpers' +export type { PartialConfig, Config } from './helpers' diff --git a/packages/core/src/services/chains/run.test.ts b/packages/core/src/services/chains/run.test.ts new file mode 100644 index 000000000..cfc1a0e07 --- /dev/null +++ b/packages/core/src/services/chains/run.test.ts @@ -0,0 +1,293 @@ +import { Chain, ContentType, MessageRole } from '@latitude-data/compiler' +import { v4 as uuid } from 'uuid' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { LogSources, Providers } from '../../constants' +import * as factories from '../../tests/factories' +import * as aiModule from '../ai' +import { runChain } from './run' + +// Mock other dependencies +vi.mock('@latitude-data/compiler') +vi.mock('uuid') + +describe('runChain', () => { + const mockChain: Partial = { + step: vi.fn(), + rawText: 'Test raw text', + } + + const mockUUID = '12345678-1234-1234-1234-123456789012' + let apikeys: Map + + function createMockAiResponse(text: string, totalTokens: number) { + return { + text: Promise.resolve(text), + usage: Promise.resolve({ totalTokens }), + toolCalls: Promise.resolve([]), + fullStream: new ReadableStream({ + start(controller) { + controller.enqueue({ type: 'text', text }) + controller.close() + }, + }), + } + } + + beforeEach(async () => { + vi.resetAllMocks() + vi.mocked(uuid).mockReturnValue(mockUUID) + + const { providers } = await factories.createProject({ + providers: [{ name: 'openai', type: Providers.OpenAI }], + }) + apikeys = new Map(providers.map((p) => [p.name, p])) + }) + + it('runs a chain without schema override', async () => { + const mockAiResponse = createMockAiResponse('AI response', 10) + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Test message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response', + usage: { totalTokens: 10 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + schema: undefined, + output: undefined, + }), + ) + }) + + it('runs a chain with schema override', async () => { + const mockSchema = { + type: 'object', + properties: { + name: { type: 'string' }, + age: { type: 'number' }, + }, + } as const + + const mockAiResponse = { + object: Promise.resolve({ name: 'John', age: 30 }), + usage: Promise.resolve({ totalTokens: 15 }), + fullStream: new ReadableStream({ + start(controller) { + controller.enqueue({ + type: 'object', + object: { name: 'John', age: 30 }, + }) + controller.close() + }, + }), + } + + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Test message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + configOverrides: { + schema: mockSchema, + output: 'object', + }, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + object: { name: 'John', age: 30 }, + text: '{"name":"John","age":30}', + usage: { totalTokens: 15 }, + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + schema: mockSchema, + output: 'object', + }), + ) + }) + + it('handles errors during chain execution', async () => { + vi.mocked(mockChain.step!).mockRejectedValue( + new Error('Chain execution failed'), + ) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + await expect(result.value.response).rejects.toThrow( + 'Chain execution failed', + ) + await expect(result.value.duration).rejects.toThrow( + 'Chain execution failed', + ) + }) + + it('handles multiple steps in a chain', async () => { + const mockAiResponse1 = createMockAiResponse('AI response 1', 10) + const mockAiResponse2 = createMockAiResponse('AI response 2', 15) + + vi.spyOn(aiModule, 'ai') + .mockResolvedValueOnce(mockAiResponse1 as any) + .mockResolvedValueOnce(mockAiResponse2 as any) + + vi.mocked(mockChain.step!) + .mockResolvedValueOnce({ + completed: false, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 1' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + .mockResolvedValueOnce({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 1' }], + }, + { + role: MessageRole.assistant, + content: 'AI response 1', + toolCalls: [], + }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'Step 2' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response 2', + usage: { totalTokens: 15 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledTimes(2) + }) + + it('handles system messages correctly', async () => { + const mockAiResponse = createMockAiResponse('AI response', 10) + vi.spyOn(aiModule, 'ai').mockResolvedValue(mockAiResponse as any) + + vi.mocked(mockChain.step!).mockResolvedValue({ + completed: true, + conversation: { + messages: [ + { + role: MessageRole.system, + content: 'System instruction', + }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'User message' }], + }, + ], + config: { provider: 'openai', model: 'gpt-3.5-turbo' }, + }, + }) + + const result = await runChain({ + chain: mockChain as Chain, + apikeys, + source: LogSources.API, + }) + + expect(result.ok).toBe(true) + if (!result.ok) return + + const response = await result.value.response + expect(response).toEqual({ + documentLogUuid: expect.any(String), + text: 'AI response', + usage: { totalTokens: 10 }, + toolCalls: [], + }) + + expect(aiModule.ai).toHaveBeenCalledWith( + expect.objectContaining({ + messages: [ + { role: MessageRole.system, content: 'System instruction' }, + { + role: MessageRole.user, + content: [{ type: ContentType.text, text: 'User message' }], + }, + ], + }), + ) + }) +}) diff --git a/packages/core/src/services/chains/run.ts b/packages/core/src/services/chains/run.ts index d5e8d67da..276117d19 100644 --- a/packages/core/src/services/chains/run.ts +++ b/packages/core/src/services/chains/run.ts @@ -1,13 +1,18 @@ import { Chain, MessageRole } from '@latitude-data/compiler' +import { CoreTool, ObjectStreamPart, TextStreamPart } from 'ai' +import { JSONSchema7 } from 'json-schema' import { v4 } from 'uuid' import { ZodError } from 'zod' -import { ProviderApiKey } from '../../browser' +import { objectToString, ProviderApiKey } from '../../browser' import { ChainCallResponse, ChainEvent, ChainEventTypes, + ChainObjectResponse, + ChainTextResponse, LogSources, + ProviderData, StreamEventTypes, } from '../../constants' import { NotFoundError, Result, UnprocessableEntityError } from '../../lib' @@ -21,11 +26,16 @@ export async function runChain({ apikeys, source, generateUUID = v4, + configOverrides, }: { chain: Chain generateUUID?: () => string source: LogSources apikeys: CachedApiKeys + configOverrides?: { + schema: JSONSchema7 + output: 'object' | 'array' | 'no-schema' + } }) { const documentLogUuid = generateUUID() @@ -46,6 +56,7 @@ export async function runChain({ apikeys, controller, documentLogUuid, + configOverrides, }) .then(responseResolve) .catch(responseReject) @@ -70,6 +81,7 @@ async function iterate({ previousCount = 0, previousResponse, documentLogUuid, + configOverrides, }: { source: LogSources chain: Chain @@ -78,20 +90,14 @@ async function iterate({ previousCount?: number previousApiKey?: ProviderApiKey documentLogUuid: string - previousResponse?: { - text: string - usage: Record + previousResponse?: ChainTextResponse + configOverrides?: { + schema: JSONSchema7 + output: 'object' | 'array' | 'no-schema' } }) { try { - const { - newMessagesInStep, - conversation, - completed, - config, - apiKey, - sentCount, - } = await doChainStep({ + const stepResult = await computeStepData({ chain, previousResponse, apikeys, @@ -99,35 +105,91 @@ async function iterate({ sentCount: previousCount, }) - enqueueChainEvent(controller, { - data: { - type: ChainEventTypes.Step, - isLastStep: completed, - config: conversation.config as Config, - messages: newMessagesInStep, - }, - event: StreamEventTypes.Latitude, - }) + publishStepStartEvent(controller, stepResult) - const result = await ai({ + const aiResult = await ai({ source, documentLogUuid, - messages: conversation.messages, - config: config, - provider: apiKey, + messages: stepResult.conversation.messages, + config: stepResult.config, + provider: stepResult.apiKey, + schema: configOverrides?.schema, + output: configOverrides?.output, + transactionalLogs: stepResult.completed, }) - for await (const value of streamToGenerator(result.fullStream)) { - enqueueChainEvent(controller, { - event: StreamEventTypes.Provider, - data: value, + await streamAIResult(controller, aiResult) + + const response = await createChainResponse(aiResult, documentLogUuid) + + if (stepResult.completed) { + await handleCompletedChain(controller, stepResult, response) + return response + } else { + publishStepCompleteEvent(controller, response) + + return iterate({ + source, + chain, + documentLogUuid, + apikeys, + controller, + previousApiKey: stepResult.apiKey, + previousCount: stepResult.sentCount, + previousResponse: response as ChainTextResponse, + configOverrides, }) } + } catch (error) { + handleIterationError(controller, error) + throw error + } +} + +// Helper functions - // TODO: The type on `ai` package return a different definition for `toolCalls` - // than `@latitude-data/compiler` package. We have to unify the naming. - // For now no toolCalls - const response: ChainCallResponse = { +function publishStepStartEvent( + controller: ReadableStreamDefaultController, + stepResult: Awaited>, +) { + enqueueChainEvent(controller, { + data: { + type: ChainEventTypes.Step, + isLastStep: stepResult.completed, + config: stepResult.conversation.config as Config, + messages: stepResult.newMessagesInStep, + }, + event: StreamEventTypes.Latitude, + }) +} + +async function streamAIResult( + controller: ReadableStreamDefaultController, + result: Awaited>, +) { + for await (const value of streamToGenerator< + TextStreamPart> | ObjectStreamPart + >(result.fullStream)) { + enqueueChainEvent(controller, { + event: StreamEventTypes.Provider, + data: value as unknown as ProviderData, + }) + } +} + +async function createChainResponse( + result: Awaited>, + documentLogUuid: string, +): Promise { + if (result.object) { + return { + text: objectToString(await result.object), + object: await result.object, + usage: await result.usage, + documentLogUuid, + } + } else { + return { documentLogUuid, text: await result.text, usage: await result.usage, @@ -137,80 +199,90 @@ async function iterate({ arguments: t.args, })), } + } +} - if (completed) { - const completedResponse = { - ...response, - documentLogUuid, - } - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.Complete, - config: conversation.config as Config, - messages: [ - { - role: MessageRole.assistant, - toolCalls: response.toolCalls, - content: response.text, - }, - ], - response: completedResponse, +async function handleCompletedChain( + controller: ReadableStreamDefaultController, + stepResult: Awaited>, + response: ChainCallResponse, +) { + const eventData = { + type: ChainEventTypes.Complete, + config: stepResult.conversation.config as Config, + response, + } as const + + if ('text' in response) { + Object.assign(eventData, { + messages: [ + { + role: MessageRole.assistant, + toolCalls: (response as ChainTextResponse).toolCalls || [], + content: response.text || '', }, - }) + ], + }) + } else if ('object' in response) { + Object.assign(eventData, { + object: (response as ChainObjectResponse).object, + text: objectToString((response as ChainObjectResponse).object), + }) + } - controller.close() + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: eventData, + }) - return completedResponse - } else { - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.StepComplete, - response, - }, - }) - return iterate({ - source, - chain, - documentLogUuid, - apikeys, - controller, - previousApiKey: apiKey, - previousCount: sentCount, - previousResponse: response, - }) - } - } catch (e) { - const error = e as Error - enqueueChainEvent(controller, { - event: StreamEventTypes.Latitude, - data: { - type: ChainEventTypes.Error, - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, + controller.close() +} + +function publishStepCompleteEvent( + controller: ReadableStreamDefaultController, + response: ChainCallResponse, +) { + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: { + type: ChainEventTypes.StepComplete, + response: response, + }, + }) +} + +function handleIterationError( + controller: ReadableStreamDefaultController, + error: unknown, +) { + const chainError = + error instanceof Error ? error : new Error('An unknown error occurred') + + enqueueChainEvent(controller, { + event: StreamEventTypes.Latitude, + data: { + type: ChainEventTypes.Error, + error: { + name: chainError.name, + message: chainError.message, + stack: chainError.stack, }, - }) - controller.close() + }, + }) + controller.close() - if (error instanceof ZodError) { - throw new UnprocessableEntityError( - 'Error validating document configuration', - error.formErrors.fieldErrors, - ) - } else { - throw error - } + if (error instanceof ZodError) { + throw new UnprocessableEntityError( + 'Error validating document configuration', + error.formErrors.fieldErrors, + ) } } /** * Performs some common operations needed for processing an iteration step **/ -async function doChainStep({ +async function computeStepData({ chain, apikeys, previousResponse, @@ -219,7 +291,7 @@ async function doChainStep({ }: { chain: Chain apikeys: CachedApiKeys - previousResponse?: { text: string; usage: Record } + previousResponse?: ChainTextResponse apiKey?: ProviderApiKey sentCount: number }) { diff --git a/packages/core/src/services/documentLogs/addMessages/index.ts b/packages/core/src/services/documentLogs/addMessages/index.ts index 8409ce12b..53c01e9be 100644 --- a/packages/core/src/services/documentLogs/addMessages/index.ts +++ b/packages/core/src/services/documentLogs/addMessages/index.ts @@ -1,4 +1,5 @@ import { Message, MessageRole } from '@latitude-data/compiler' +import { CoreTool, ObjectStreamPart, TextStreamPart } from 'ai' import { ChainCallResponse, @@ -6,7 +7,9 @@ import { ChainEventTypes, DocumentLog, LogSources, + objectToString, ProviderApiKey, + ProviderData, StreamEventTypes, Workspace, } from '../../../browser' @@ -64,7 +67,9 @@ export async function addMessages({ ...providerLog.messages, { role: MessageRole.assistant, - content: providerLog.responseText, + content: + providerLog.responseText || + objectToString(providerLog.responseObject), toolCalls: providerLog.toolCalls, }, ...messages, @@ -75,10 +80,6 @@ export async function addMessages({ }, }) - // Dummy handling of the response - // This is helpful for not throwing the error - // when no one is listening to the promise - response.then(() => {}).catch(() => {}) return Result.ok({ stream, response }) } @@ -106,22 +107,27 @@ async function streamMessageResponse({ source, }) - for await (const value of streamToGenerator(result.fullStream)) { + for await (const value of streamToGenerator< + TextStreamPart> | ObjectStreamPart + >(result.fullStream)) { enqueueChainEvent(controller, { event: StreamEventTypes.Provider, - data: value, + data: value as unknown as ProviderData, }) } const response = { documentLogUuid, + object: await result.object, text: await result.text, usage: await result.usage, - toolCalls: (await result.toolCalls).map((t) => ({ - id: t.toolCallId, - name: t.toolName, - arguments: t.args, - })), + toolCalls: result.toolCalls + ? (await result.toolCalls).map((t) => ({ + id: t.toolCallId, + name: t.toolName, + arguments: t.args, + })) + : [], } enqueueChainEvent(controller, { event: StreamEventTypes.Latitude, @@ -132,14 +138,22 @@ async function streamMessageResponse({ { role: MessageRole.assistant, toolCalls: response.toolCalls, - content: response.text, + content: response.text || objectToString(response.object), }, ], - response, + response: { + ...response, + text: response.text || objectToString(response.object), + }, }, }) + controller.close() - return response + + return { + ...response, + text: response.text || objectToString(response.object), + } } catch (e) { const error = e as Error enqueueChainEvent(controller, { diff --git a/packages/core/src/services/evaluationResults/create.test.ts b/packages/core/src/services/evaluationResults/create.test.ts index 5b683494d..89cd74667 100644 --- a/packages/core/src/services/evaluationResults/create.test.ts +++ b/packages/core/src/services/evaluationResults/create.test.ts @@ -59,7 +59,10 @@ describe('createEvaluationResult', () => { evaluation, documentLog, providerLog, - result: true, + result: { + result: true, + reason: 'This is a boolean result', + }, }) expect(result.ok).toBe(true) @@ -70,7 +73,6 @@ describe('createEvaluationResult', () => { expect(result.value.providerLogId).toBe(providerLog.id) expect(result.value.resultableType).toBe(EvaluationResultableType.Boolean) expect(result.value.result).toBe(true) - // Verify in database const dbResult = await database.query.evaluationResults.findFirst({ where: eq(evaluationResults.id, result.value.id), @@ -95,7 +97,10 @@ describe('createEvaluationResult', () => { evaluation, documentLog, providerLog, - result: 75, + result: { + result: 75, + reason: 'This is a number result', + }, }) expect(result.ok).toBe(true) @@ -106,7 +111,6 @@ describe('createEvaluationResult', () => { expect(result.value.providerLogId).toBe(providerLog.id) expect(result.value.resultableType).toBe(EvaluationResultableType.Number) expect(result.value.result).toBe(75) - // Verify in database const dbResult = await database.query.evaluationResults.findFirst({ where: eq(evaluationResults.id, result.value.id), @@ -131,7 +135,10 @@ describe('createEvaluationResult', () => { evaluation, documentLog, providerLog, - result: 'This is a text result', + result: { + result: 'This is a text result', + reason: 'Explanation for the text result', + }, }) expect(result.ok).toBe(true) @@ -142,7 +149,6 @@ describe('createEvaluationResult', () => { expect(result.value.providerLogId).toBe(providerLog.id) expect(result.value.resultableType).toBe(EvaluationResultableType.Text) expect(result.value.result).toBe('This is a text result') - // Verify in database const dbResult = await database.query.evaluationResults.findFirst({ where: eq(evaluationResults.id, result.value.id), @@ -168,7 +174,10 @@ describe('createEvaluationResult', () => { evaluation, documentLog, providerLog, - result: 'Some result', + result: { + result: 'Some result', + reason: 'Some reason', + }, }) expect(result.ok).toBe(false) diff --git a/packages/core/src/services/evaluationResults/create.ts b/packages/core/src/services/evaluationResults/create.ts index bcc4d871d..566978ed3 100644 --- a/packages/core/src/services/evaluationResults/create.ts +++ b/packages/core/src/services/evaluationResults/create.ts @@ -17,7 +17,7 @@ export type CreateEvaluationResultProps = { evaluation: Evaluation | EvaluationDto documentLog: DocumentLog providerLog: ProviderLog - result: number | string | boolean + result: { result: number | string | boolean; reason: string } } export async function createEvaluationResult( @@ -44,7 +44,11 @@ export async function createEvaluationResult( ) } - const metadata = await trx.insert(table).values({ result }).returning() + // TODO: store the reason + const metadata = await trx + .insert(table) + .values({ result: result.result }) + .returning() const inserts = await trx .insert(evaluationResults) .values({ diff --git a/packages/core/src/services/evaluations/run.ts b/packages/core/src/services/evaluations/run.ts index fb80db647..7c351e258 100644 --- a/packages/core/src/services/evaluations/run.ts +++ b/packages/core/src/services/evaluations/run.ts @@ -1,6 +1,12 @@ import { createChain, readMetadata } from '@latitude-data/compiler' +import { JSONSchema7 } from 'json-schema' -import { DocumentLog, EvaluationDto, LogSources } from '../../browser' +import { + DocumentLog, + EvaluationDto, + EvaluationResultableType, + LogSources, +} from '../../browser' import { database } from '../../client' import { findLastProviderLogFromDocumentLogUuid } from '../../data-access' import { publisher } from '../../events/publisher' @@ -10,6 +16,20 @@ import { computeDocumentLogWithMetadata } from '../documentLogs' import { buildProviderApikeysMap } from '../providerApiKeys/buildMap' import { formatContext, formatConversation } from '../providerLogs' +// Helper function to get the result schema based on evaluation type +const getResultSchema = (type: EvaluationResultableType): JSONSchema7 => { + switch (type) { + case EvaluationResultableType.Boolean: + return { type: 'boolean' } + case EvaluationResultableType.Number: + return { type: 'number' } + case EvaluationResultableType.Text: + return { type: 'string' } + default: + throw new Error(`Unsupported evaluation type: ${type}`) + } +} + export const runEvaluation = async ( { documentLog, @@ -32,12 +52,11 @@ export const runEvaluation = async ( ) } - const rezult = await computeDocumentLogWithMetadata(documentLog) - if (rezult.error) return rezult - const documentLogWithMetadata = rezult.value + const documentLogWithMetadataResult = + await computeDocumentLogWithMetadata(documentLog) + if (documentLogWithMetadataResult.error) return documentLogWithMetadataResult + const documentLogWithMetadata = documentLogWithMetadataResult.value - // TODO: This will need to become polymorphic in the future given different - // types of evaluations const metadata = await readMetadata({ prompt: documentLog.resolvedContent }) const chain = createChain({ prompt: evaluation.metadata.prompt, @@ -50,21 +69,37 @@ export const runEvaluation = async ( config: metadata.config, duration: documentLogWithMetadata.duration, cost: documentLogWithMetadata.costInMillicents - ? documentLogWithMetadata.costInMillicents * 1000 + ? documentLogWithMetadata.costInMillicents / 1000 : 0, }, }) - const result = await runChain({ + // Use the helper function to get the result schema + const resultSchema = getResultSchema(evaluation.configuration.type) + + const schema: JSONSchema7 = { + type: 'object', + properties: { + result: resultSchema, + reason: { type: 'string' }, + }, + required: ['result', 'reason'], + } + + const chainResult = await runChain({ chain, source: LogSources.Evaluation, apikeys: await buildProviderApikeysMap({ workspaceId: evaluation.workspaceId, }), + configOverrides: { + schema, + output: 'object', + }, }) - if (result.error) return result + if (chainResult.error) return chainResult - result.value.response.then((response) => { + chainResult.value.response.then((response) => { publisher.publish({ type: 'evaluationRun', data: { @@ -76,5 +111,5 @@ export const runEvaluation = async ( }) }) - return result + return chainResult } diff --git a/packages/core/src/services/providerLogs/create.ts b/packages/core/src/services/providerLogs/create.ts index 73a4cb057..a6856a236 100644 --- a/packages/core/src/services/providerLogs/create.ts +++ b/packages/core/src/services/providerLogs/create.ts @@ -1,8 +1,10 @@ import { Message, ToolCall } from '@latitude-data/compiler' import { CompletionTokenUsage } from 'ai' +import { JSONSchema7 } from 'json-schema' import { LogSources, ProviderLog, Providers } from '../../browser' import { database } from '../../client' +import { publisher } from '../../events/publisher' import { Result, Transaction } from '../../lib' import { providerLogs } from '../../schema' import { estimateCost, PartialConfig } from '../ai' @@ -19,7 +21,8 @@ export type CreateProviderLogProps = { model: string config: PartialConfig messages: Message[] - responseText: string + responseText?: string + responseObject?: JSONSchema7 toolCalls?: ToolCall[] usage: CompletionTokenUsage duration: number @@ -38,6 +41,7 @@ export async function createProviderLog( config, messages, responseText, + responseObject, toolCalls, usage, duration, @@ -67,6 +71,7 @@ export async function createProviderLog( config, messages, responseText, + responseObject, toolCalls, tokens: isNaN(usage.totalTokens) ? 0 : (usage.totalTokens ?? 0), costInMillicents: cost, @@ -80,6 +85,11 @@ export async function createProviderLog( await touchProviderApiKey(providerId, trx) if (apiKeyId) await touchApiKey(apiKeyId, trx) + publisher.publishLater({ + type: 'providerLogCreated', + data: log, + }) + return Result.ok(log) }, db) } diff --git a/packages/core/src/services/providerLogs/formatForEvaluation.ts b/packages/core/src/services/providerLogs/formatForEvaluation.ts index 7eb76b0a9..a1defb37a 100644 --- a/packages/core/src/services/providerLogs/formatForEvaluation.ts +++ b/packages/core/src/services/providerLogs/formatForEvaluation.ts @@ -1,23 +1,39 @@ import { MessageRole } from '@latitude-data/compiler' -import { Message, ProviderLog } from '../../browser' - -export function formatConversation(providerLog: ProviderLog) { +import { + Message, + objectToString, + ProviderLog, + ProviderLogDto, +} from '../../browser' + +export function formatConversation(providerLog: ProviderLogDto | ProviderLog) { const messages: Message[] = [...(providerLog.messages || [])] - // Add the response as an assistant message if it exists - if (providerLog.responseText) { + if ((providerLog as ProviderLogDto).response) { + messages.push({ + role: MessageRole.assistant, + content: (providerLog as ProviderLogDto).response, + toolCalls: providerLog.toolCalls, + }) + } else if ((providerLog as ProviderLog).responseText) { messages.push({ role: MessageRole.assistant, - content: providerLog.responseText, + content: (providerLog as ProviderLog).responseText!, toolCalls: providerLog.toolCalls, }) + } else if ((providerLog as ProviderLog).responseObject) { + messages.push({ + role: MessageRole.assistant, + content: objectToString((providerLog as ProviderLog).responseObject), + toolCalls: [], + }) } return formatMessages(messages) } -export function formatContext(providerLog: ProviderLog) { +export function formatContext(providerLog: ProviderLog | ProviderLogDto) { return formatMessages(providerLog.messages) } diff --git a/packages/core/src/tests/factories/evaluationResults.ts b/packages/core/src/tests/factories/evaluationResults.ts index b0555b468..8f7a9fc8e 100644 --- a/packages/core/src/tests/factories/evaluationResults.ts +++ b/packages/core/src/tests/factories/evaluationResults.ts @@ -88,7 +88,10 @@ export async function createEvaluationResult({ evaluation, documentLog, providerLog: providerLogs[providerLogs.length - 1]!, - result: mockedResponse, + result: { + result: mockedResponse, + reason: 'I do not even know to be honest.', + }, }) return { diff --git a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts index 745f37b81..019033330 100644 --- a/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts +++ b/packages/jobs/src/job-definitions/batchEvaluations/runEvaluationJob.ts @@ -40,13 +40,11 @@ export const runEvaluationJob = async (job: Job) => { .find(evaluationId) .then((r) => r.unwrap()) - const { response } = await runEvaluation({ + await runEvaluation({ documentLog, evaluation, }).then((r) => r.unwrap()) - await response - await progressTracker.incrementCompleted() } catch (error) { await progressTracker.incrementErrors() diff --git a/packages/jobs/src/queues/index.ts b/packages/jobs/src/queues/index.ts index ce1f902df..3b9f85c06 100644 --- a/packages/jobs/src/queues/index.ts +++ b/packages/jobs/src/queues/index.ts @@ -1,3 +1,4 @@ +import { createProviderLogJob } from '@latitude-data/core/events/handlers/createProviderLogJob' import { EventHandlers } from '@latitude-data/core/events/handlers/index' import { Job, JobsOptions, Queue, QueueEvents } from 'bullmq' @@ -66,7 +67,12 @@ function setupQueue({ export const QUEUES = { [Queues.defaultQueue]: { name: Queues.defaultQueue, - jobs: [runBatchEvaluationJob, runDocumentJob, runEvaluationJob], + jobs: [ + runBatchEvaluationJob, + runDocumentJob, + runEvaluationJob, + createProviderLogJob, + ], }, [Queues.eventsQueue]: { name: Queues.eventsQueue, diff --git a/packages/sdks/typescript/src/index.ts b/packages/sdks/typescript/src/index.ts index 1f957edae..74f0f0c9c 100644 --- a/packages/sdks/typescript/src/index.ts +++ b/packages/sdks/typescript/src/index.ts @@ -1,5 +1,5 @@ import type { Message } from '@latitude-data/compiler' -import { LogSources } from '@latitude-data/core/browser' +import { ChainCallResponse, LogSources } from '@latitude-data/core/browser' import env from '$sdk/env' import { GatewayApiConfig, RouteResolver } from '$sdk/utils' import { @@ -9,15 +9,6 @@ import { UrlParams, } from '$sdk/utils/types' -type ChainCallResponse = { - documentLogUuid: string - text: string - usage: { - promptTokens: number - completionTokens: number - totalTokens: number - } -} export type StreamChainResponse = { conversation: Message[] response: ChainCallResponse @@ -115,7 +106,7 @@ export class LatitudeSdk { const reader = body.getReader() const conversation: Message[] = [] - let chainResponse: ChainCallResponse | null = null + let chainResponse: ChainCallResponse while (true) { const { done, value } = await reader.read() @@ -131,7 +122,7 @@ export class LatitudeSdk { // in a new package. Now they are in `core` but I don't want to depend on core // just for this. We need these enums in production not only as a dev dependency. if (chunk.event === 'latitude-event') { - const messages = 'messages' in chunk.data ? chunk.data.messages : [] + const messages = 'messages' in chunk.data ? chunk.data.messages! : [] if (messages.length > 0) { // At the moment all message.content should be a string // but in the future this could be an image or other type diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a957174d5..8d4afa1b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -615,6 +615,9 @@ importers: '@t3-oss/env-core': specifier: ^0.11.1 version: 0.11.1(typescript@5.6.2)(zod@3.23.8) + '@types/json-schema': + specifier: ^7.0.15 + version: 7.0.15 '@types/lodash-es': specifier: ^4.17.12 version: 4.17.12 @@ -654,6 +657,9 @@ importers: jose: specifier: ^5.8.0 version: 5.9.2 + json-schema: + specifier: ^0.4.0 + version: 0.4.0 lodash-es: specifier: ^4.17.21 version: 4.17.21