feature: structured outputs

Adds support for structured outputs responses from providers and
implements it in evaluations and across our UI

apps/web/src/actions/providerLogs/fetch.ts

@@ -1,6 +1,7 @@
 'use server'
 
 import { ProviderLogsRepository } from '@latitude-data/core/repositories'
+import providerLogPresenter from '$/presenters/providerLogPresenter'
 import { z } from 'zod'
 
 import { authProcedure } from '../procedures'
@@ -30,7 +31,7 @@ export const getProviderLogsAction = authProcedure
       result = await scope.findAll({ limit: 1000 }).then((r) => r.unwrap())
     }
 
-    return result
+    return result.map(providerLogPresenter)
   })
 
 export const getProviderLogAction = authProcedure
@@ -39,5 +40,8 @@ export const getProviderLogAction = authProcedure
   .handler(async ({ input, ctx }) => {
     const { providerLogId } = input
     const scope = new ProviderLogsRepository(ctx.workspace.id)
-    return await scope.find(providerLogId).then((r) => r.unwrap())
+    return await scope
+      .find(providerLogId)
+      .then((r) => r.unwrap())
+      .then(providerLogPresenter)
   })

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/index.tsx

@@ -50,7 +50,7 @@ export default function Playground({
       setInputs({
         messages: JSON.stringify(formatConversation(providerLog)),
         context: JSON.stringify(formatContext(providerLog)),
-        response: providerLog.responseText,
+        response: providerLog.response,
         prompt: '',
         parameters: '',
         config: '',

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/import-logs/page.tsx

@@ -4,7 +4,7 @@ import { useState } from 'react'
 import { capitalize } from 'lodash-es'
 
 import { MessageContent, TextContent } from '@latitude-data/compiler'
-import { HEAD_COMMIT } from '@latitude-data/core/browser'
+import { HEAD_COMMIT, ProviderLogDto } from '@latitude-data/core/browser'
 import {
   Badge,
   Button,
@@ -193,7 +193,9 @@ const ProviderLogMessages = ({
   providerLogUuid?: string
 }) => {
   const { data } = useProviderLogs({ documentUuid })
-  const providerLog = data?.find((log) => log.uuid === providerLogUuid)
+  const providerLog = data?.find(
+    (log) => log.uuid === providerLogUuid,
+  ) as ProviderLogDto
   if (!providerLog) {
     return (
       <div className='flex flex-col items-center justify-center rounded-lg border border-2 bg-secondary p-4 h-[480px]'>
@@ -225,7 +227,7 @@ const ProviderLogMessages = ({
             <Badge variant={roleVariant('assistant')}>Assistant</Badge>
           </div>
           <div className='pl-4'>
-            <Text.H6M>{printMessageContent(providerLog.responseText)}</Text.H6M>
+            <Text.H6M>{printMessageContent(providerLog.response)}</Text.H6M>
           </div>
         </div>
       </div>

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Chat.tsx

@@ -1,6 +1,7 @@
 import { useCallback, useContext, useEffect, useRef, useState } from 'react'
 
 import {
+  AssistantMessage,
   ContentType,
   Conversation,
   Message as ConversationMessage,
@@ -95,8 +96,8 @@ export default function Chat({
       const { event, data } = serverEvent
       const hasMessages = 'messages' in data
       if (hasMessages) {
-        data.messages.forEach(addMessageToConversation)
-        messagesCount += data.messages.length
+        data.messages!.forEach(addMessageToConversation)
+        messagesCount += data.messages!.length
       }
 
       switch (event) {
@@ -109,6 +110,7 @@ export default function Chat({
           } else if (data.type === ChainEventTypes.Error) {
             setError(new Error(data.error.message))
           }
+
           break
         }
 
@@ -161,17 +163,17 @@ export default function Chat({
 
         const { event, data } = serverEvent
 
-        const hasMessages = 'messages' in data
-
-        if (hasMessages) {
-          data.messages.forEach(addMessageToConversation)
-        }
-
         switch (event) {
           case StreamEventTypes.Latitude: {
             if (data.type === ChainEventTypes.Error) {
               setError(new Error(data.error.message))
+            } else if (data.type === ChainEventTypes.Complete) {
+              addMessageToConversation({
+                role: MessageRole.assistant,
+                content: data.response.text,
+              } as AssistantMessage)
             }
+
             break
           }
 

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Messages.tsx

@@ -1,20 +1,20 @@
 import { useMemo } from 'react'
 
 import { AssistantMessage, Message, MessageRole } from '@latitude-data/compiler'
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { MessageList } from '@latitude-data/web-ui'
 
 export function EvaluationResultMessages({
   providerLog,
 }: {
-  providerLog?: ProviderLog
+  providerLog?: ProviderLogDto
 }) {
   const messages = useMemo<Message[]>(() => {
     if (!providerLog) return [] as Message[]
 
     const responseMessage = {
       role: MessageRole.assistant,
-      content: providerLog.responseText,
+      content: providerLog.response,
       toolCalls: providerLog.toolCalls,
     } as AssistantMessage
 

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/Metadata.tsx

@@ -1,6 +1,6 @@
 import { ReactNode } from 'react'
 
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'
 import {
   ClickToCopy,
@@ -48,7 +48,7 @@ export function EvaluationResultMetadata({
   providerLog,
 }: {
   evaluationResult: EvaluationResultWithMetadata
-  providerLog?: ProviderLog
+  providerLog?: ProviderLogDto
 }) {
   const { data: providers, isLoading: providersLoading } = useProviderApiKeys()
 

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/EvaluationResults/EvaluationResultInfo/index.tsx

@@ -2,7 +2,7 @@
 
 import { useState } from 'react'
 
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { EvaluationResultWithMetadata } from '@latitude-data/core/repositories'
 import { TabSelector } from '@latitude-data/web-ui'
 
@@ -14,7 +14,7 @@ export function EvaluationResultInfo({
   providerLog,
 }: {
   evaluationResult: EvaluationResultWithMetadata
-  providerLog?: ProviderLog
+  providerLog?: ProviderLogDto
 }) {
   const [selectedTab, setSelectedTab] = useState<string>('metadata')
   return (

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Messages.tsx

@@ -1,21 +1,21 @@
 import { useMemo } from 'react'
 
 import { AssistantMessage, Message, MessageRole } from '@latitude-data/compiler'
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { MessageList } from '@latitude-data/web-ui'
 
 export function DocumentLogMessages({
   providerLogs,
 }: {
-  providerLogs?: ProviderLog[]
+  providerLogs?: ProviderLogDto[]
 }) {
   const messages = useMemo<Message[]>(() => {
     const lastLog = providerLogs?.[providerLogs.length - 1]
     if (!lastLog) return [] as Message[]
 
     const responseMessage = {
       role: MessageRole.assistant,
-      content: lastLog.responseText,
+      content: lastLog.response,
       toolCalls: lastLog.toolCalls,
     } as AssistantMessage
 

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/Metadata.tsx

@@ -1,6 +1,6 @@
 import { ReactNode, useMemo } from 'react'
 
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { DocumentLogWithMetadata } from '@latitude-data/core/repositories'
 import {
   ClickToCopy,
@@ -48,7 +48,7 @@ export function DocumentLogMetadata({
   providerLogs,
 }: {
   documentLog: DocumentLogWithMetadata
-  providerLogs?: ProviderLog[]
+  providerLogs?: ProviderLogDto[]
 }) {
   const { data: providers, isLoading: providersLoading } = useProviderApiKeys()
   const lastProviderLog = useMemo(

apps/web/src/app/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/logs/_components/DocumentLogs/DocumentLogInfo/index.tsx

@@ -2,7 +2,7 @@
 
 import { useState } from 'react'
 
-import { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { DocumentLogWithMetadata } from '@latitude-data/core/repositories'
 import { TabSelector } from '@latitude-data/web-ui'
 
@@ -14,7 +14,7 @@ export function DocumentLogInfo({
   providerLogs,
 }: {
   documentLog: DocumentLogWithMetadata
-  providerLogs?: ProviderLog[]
+  providerLogs?: ProviderLogDto[]
 }) {
   const [selectedTab, setSelectedTab] = useState<string>('metadata')
   return (

apps/web/src/presenters/providerLogPresenter.ts

@@ -0,0 +1,16 @@
+import { omit } from 'lodash-es'
+
+import type { ProviderLog, ProviderLogDto } from '@latitude-data/core/browser'
+
+export default function providerLogPresenter(
+  providerLog: ProviderLog,
+): ProviderLogDto {
+  return {
+    ...omit(providerLog, 'responseText', 'responseObject'),
+    response:
+      providerLog.responseText ||
+      (providerLog.responseObject
+        ? JSON.stringify(providerLog.responseObject)
+        : ''),
+  }
+}

apps/web/src/stores/providerLogs.ts

@@ -2,7 +2,7 @@
 
 import { compact } from 'lodash-es'
 
-import type { ProviderLog } from '@latitude-data/core/browser'
+import { ProviderLogDto } from '@latitude-data/core/browser'
 import { useToast } from '@latitude-data/web-ui'
 import {
   getProviderLogAction,
@@ -19,10 +19,10 @@ export default function useProviderLogs(
 ) {
   const { toast } = useToast()
   const {
-    data = undefined,
+    data = [],
     isLoading,
     error: swrError,
-  } = useSWR<ProviderLog[] | undefined>(
+  } = useSWR<ProviderLogDto[]>(
     compact(['providerLogs', documentUuid, documentLogUuid]),
     async () => {
       const [data, error] = await getProviderLogsAction({
@@ -38,10 +38,11 @@ export default function useProviderLogs(
           description: error.formErrors?.[0] || error.message,
           variant: 'destructive',
         })
-        return undefined
+
+        return []
       }
 
-      return data as ProviderLog[]
+      return data
     },
     opts,
   )
@@ -62,7 +63,7 @@ export function useProviderLog(
     data = undefined,
     isLoading,
     error: swrError,
-  } = useSWR<ProviderLog | undefined>(
+  } = useSWR<ProviderLogDto | undefined>(
     compact(['providerLog', providerLogId]),
     async () => {
       if (!providerLogId) return undefined
@@ -82,7 +83,7 @@ export function useProviderLog(
         return undefined
       }
 
-      return data as ProviderLog
+      return data
     },
     opts,
   )

packages/core/drizzle/0052_famous_daimon_hellstrom.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP DEFAULT;--> statement-breakpoint
+ALTER TABLE "latitude"."provider_logs" ALTER COLUMN "response_text" DROP NOT NULL;--> statement-breakpoint
+ALTER TABLE "latitude"."provider_logs" ADD COLUMN "response_object" jsonb;