feature: default provider banner and some other impros (#324)

- Redirect to the evaluation editor from evaluation page - Preload providers in server for better editor experience - Add default provider banner in evaluation page
latitude-dev · Oct 2, 2024 · 6adabbc · 6adabbc
1 parent 19f98bb
commit 6adabbc
Show file tree

Hide file tree

Showing 10 changed files with 111 additions and 27 deletions.
diff --git a/apps/web/src/app/(private)/_data-access/index.ts b/apps/web/src/app/(private)/_data-access/index.ts
@@ -10,6 +10,7 @@ import {
   DocumentVersionsRepository,
   EvaluationsRepository,
   ProjectsRepository,
+  ProviderApiKeysRepository,
   ProviderLogsRepository,
 } from '@latitude-data/core/repositories/index'
 import { getCurrentUser } from '$/services/auth/getCurrentUser'
@@ -209,3 +210,17 @@ export const getApiKeysCached = cache(async () => {
   const result = await scope.findAll()
   return result.unwrap()
 })
+
+export const getProviderApiKeyCached = cache(async (name: string) => {
+  const { workspace } = await getCurrentUser()
+  const scope = new ProviderApiKeysRepository(workspace.id)
+  const result = await scope.findByName(name)
+  return result.unwrap()
+})
+
+export const getProviderApiKeysCached = cache(async () => {
+  const { workspace } = await getCurrentUser()
+  const scope = new ProviderApiKeysRepository(workspace.id)
+  const result = await scope.findAll()
+  return result.unwrap()
+})
diff --git a/...ations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx b/...ations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/index.tsx
@@ -2,7 +2,7 @@
 
 import { Suspense, useCallback, useMemo, useState } from 'react'
 
-import { promptConfigSchema } from '@latitude-data/core/browser'
+import { promptConfigSchema, ProviderApiKey } from '@latitude-data/core/browser'
 import {
   Button,
   DocumentTextEditor,
@@ -19,16 +19,20 @@ import { EVALUATION_PARAMETERS } from './Playground/Chat'
 export default function EvaluationEditor({
   evaluationUuid,
   defaultPrompt,
+  providerApiKeys,
 }: {
   evaluationUuid: string
   defaultPrompt: string
+  providerApiKeys?: ProviderApiKey[]
 }) {
   const { data, isLoading, update, isUpdating } = useEvaluations()
   const evaluation = useMemo(
     () => data.find((e) => e.uuid === evaluationUuid),
     [evaluationUuid, data],
   )
-  const { data: providers } = useProviderApiKeys()
+  const { data: providers } = useProviderApiKeys({
+    fallbackData: providerApiKeys,
+  })
   const [value, setValue] = useState(defaultPrompt)
   const configSchema = useMemo(
     () => promptConfigSchema({ providers: providers ?? [] }),
@@ -66,6 +70,7 @@ export default function EvaluationEditor({
     <div className='flex flex-row w-full h-full gap-8'>
       <div className='flex flex-col flex-1 flex-grow flex-shrink gap-2 min-w-0'>
         <EditorHeader
+          providers={providers}
           title='Evaluation editor'
           metadata={metadata}
           prompt={value}

diff --git a/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/layout.tsx b/apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/layout.tsx
@@ -1,7 +1,10 @@
 import { ReactNode } from 'react'
 
 import { Text } from '@latitude-data/web-ui'
-import { getEvaluationByUuidCached } from '$/app/(private)/_data-access'
+import {
+  getEvaluationByUuidCached,
+  getProviderApiKeysCached,
+} from '$/app/(private)/_data-access'
 import { NAV_LINKS } from '$/app/(private)/_lib/constants'
 import BreadcrumbLink from '$/components/BreadcrumbLink'
 import { AppLayout } from '$/components/layouts'
@@ -21,6 +24,7 @@ export default async function DocumentPage({
   const evaluationUuid = params.evaluationUuid
   const evaluation = await getEvaluationByUuidCached(evaluationUuid)
   const session = await getCurrentUser()
+  const providerApiKeys = await getProviderApiKeysCached()
 
   return (
     <AppLayout
@@ -45,6 +49,7 @@ export default async function DocumentPage({
         <EvaluationTabSelector evaluation={evaluation} />
         <div className='flex-grow'>
           <EvaluationEditor
+            providerApiKeys={providerApiKeys}
             evaluationUuid={evaluationUuid}
             defaultPrompt={evaluation.metadata.prompt}
           />

diff --git a/...ersions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx b/...ersions/[commitUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/index.tsx
@@ -7,6 +7,7 @@ import { Document as RefDocument } from '@latitude-data/compiler'
 import {
   DocumentVersion,
   promptConfigSchema,
+  ProviderApiKey,
 } from '@latitude-data/core/browser'
 import {
   DocumentTextEditor,
@@ -37,16 +38,19 @@ export default function DocumentEditor({
   addMessagesAction,
   document,
   documents,
+  providerApiKeys,
 }: {
   runDocumentAction: Function
   addMessagesAction: Function
   document: DocumentVersion
   documents: DocumentVersion[]
+  providerApiKeys?: ProviderApiKey[]
 }) {
   const { commit } = useCurrentCommit()
   const { project } = useCurrentProject()
-  const { data: providers } = useProviderApiKeys()
-
+  const { data: providers } = useProviderApiKeys({
+    fallbackData: providerApiKeys,
+  })
   const { data: _documents, updateContent } = useDocumentVersions(
     {
       commitUuid: commit.uuid,
@@ -140,6 +144,7 @@ export default function DocumentEditor({
       <div className='flex flex-row w-full h-full gap-8 p-6'>
         <div className='flex flex-col flex-1 flex-grow flex-shrink gap-2 min-w-0'>
           <EditorHeader
+            providers={providers}
             disabledMetadataSelectors={isMerged}
             title='Prompt editor'
             metadata={metadata}

diff --git a/...itUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/index.tsx b/...itUuid]/documents/[documentUuid]/evaluations/[evaluationId]/_components/Actions/index.tsx
@@ -4,6 +4,7 @@ import { EvaluationDto } from '@latitude-data/core/browser'
 import { TableWithHeader, useCurrentDocument } from '@latitude-data/web-ui'
 import { useToggleModal } from '$/hooks/useToogleModal'
 
+import DefaultProviderBanner from '../DefaulProviderBanner'
 import CreateBatchEvaluationModal from './CreateBatchEvaluationModal'
 import LiveEvaluationToggle from './LiveEvaluationToggle'
 
@@ -12,20 +13,25 @@ export function Actions({
   projectId,
   commitUuid,
   documentUuid,
+  isUsingDefaultProvider,
 }: {
   evaluation: EvaluationDto
   projectId: string
   commitUuid: string
   documentUuid: string
+  isUsingDefaultProvider?: boolean
 }) {
   const document = useCurrentDocument()
   const { open, onClose, onOpen } = useToggleModal()
   return (
     <div className='flex flex-row items-center gap-4'>
-      <LiveEvaluationToggle
-        documentUuid={documentUuid}
-        evaluation={evaluation}
-      />
+      <div className='flex flex-row items-center gap-4'>
+        {isUsingDefaultProvider && <DefaultProviderBanner />}
+        <LiveEvaluationToggle
+          documentUuid={documentUuid}
+          evaluation={evaluation}
+        />
+      </div>
       <TableWithHeader.Button onClick={onOpen}>
         Run batch evaluation
       </TableWithHeader.Button>

diff --git a/...ents/[documentUuid]/evaluations/[evaluationId]/_components/DefaulProviderBanner/index.tsx b/...ents/[documentUuid]/evaluations/[evaluationId]/_components/DefaulProviderBanner/index.tsx
@@ -0,0 +1,17 @@
+import { Icon, Tooltip } from '@latitude-data/web-ui'
+
+export default function DefaultProviderBanner() {
+  return (
+    <Tooltip
+      trigger={
+        <div>
+          <Icon name='alert' color='warningMutedForeground' />
+        </div>
+      }
+    >
+      This evaluation is using the default Latitude provider. This provider has
+      a limited number of free runs and is not recommended for running
+      evaluations. Please use a different provider for production workloads.
+    </Tooltip>
+  )
+}
diff --git a/...tId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx b/...tId]/versions/[commitUuid]/documents/[documentUuid]/evaluations/[evaluationId]/layout.tsx
@@ -1,5 +1,6 @@
 import { ReactNode } from 'react'
 
+import { readMetadata } from '@latitude-data/compiler'
 import {
   Commit,
   Evaluation,
@@ -13,8 +14,12 @@ import {
   getEvaluationModalValueQuery,
   getEvaluationTotalsQuery,
 } from '@latitude-data/core/services/evaluationResults/index'
+import { env } from '@latitude-data/env'
 import { Icon, TableWithHeader, Text, Tooltip } from '@latitude-data/web-ui'
-import { findCommitCached } from '$/app/(private)/_data-access'
+import {
+  findCommitCached,
+  getProviderApiKeyCached,
+} from '$/app/(private)/_data-access'
 import BreadcrumbLink from '$/components/BreadcrumbLink'
 import { Breadcrumb } from '$/components/layouts/AppLayout/Header'
 import { getCurrentUser } from '$/services/auth/getCurrentUser'
@@ -106,6 +111,20 @@ export default async function ConnectedEvaluationLayout({
     isNumeric,
     commit,
   })
+
+  let provider
+  if (evaluation.metadata.prompt) {
+    const metadata = await readMetadata({
+      prompt: evaluation.metadata.prompt,
+    })
+
+    if (
+      metadata.config.provider &&
+      typeof metadata.config.provider === 'string'
+    ) {
+      provider = await getProviderApiKeyCached(metadata.config.provider)
+    }
+  }
   return (
     <div className='flex flex-col w-full h-full gap-6 p-6'>
       <TableWithHeader
@@ -140,7 +159,7 @@ export default async function ConnectedEvaluationLayout({
                         <Link
                           href={
                             ROUTES.evaluations.detail({ uuid: evaluation.uuid })
-                              .root
+                              .editor.root
                           }
                         >
                           <Icon name='externalLink' />
@@ -157,6 +176,9 @@ export default async function ConnectedEvaluationLayout({
         }
         actions={
           <Actions
+            isUsingDefaultProvider={
+              provider && provider.token === env.DEFAULT_PROVIDER_API_KEY
+            }
             evaluation={evaluation}
             projectId={params.projectId}
             commitUuid={params.commitUuid}

diff --git a/...pp/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/page.tsx b/...pp/(private)/projects/[projectId]/versions/[commitUuid]/documents/[documentUuid]/page.tsx
@@ -4,6 +4,7 @@ import {
   findCommitCached,
   getDocumentByUuidCached,
   getDocumentsAtCommitCached,
+  getProviderApiKeysCached,
 } from '$/app/(private)/_data-access'
 
 import DocumentEditor from './_components/DocumentEditor/Editor'
@@ -22,13 +23,15 @@ export default async function DocumentPage({
     commitUuid,
   })
   const documents = await getDocumentsAtCommitCached({ commit })
+  const providerApiKeys = await getProviderApiKeysCached()
 
   return (
     <DocumentEditor
       runDocumentAction={runDocumentAction}
       addMessagesAction={addMessagesAction}
       documents={documents}
       document={document}
+      providerApiKeys={providerApiKeys}
     />
   )
 }
diff --git a/apps/web/src/components/EditorHeader/index.tsx b/apps/web/src/components/EditorHeader/index.tsx
@@ -18,6 +18,7 @@ import {
 import useProviderApiKeys from '$/stores/providerApiKeys'
 
 const CUSTOM_MODEL = 'custom-model'
+
 function useModelsOptions({ provider }: { provider: Providers | undefined }) {
   return useMemo(() => {
     const models = provider ? PROVIDER_MODELS[provider] : null
@@ -53,13 +54,15 @@ export default function EditorHeader({
   onChangePrompt,
   rightActions,
   disabledMetadataSelectors = false,
+  providers,
 }: {
   title: string
   metadata: ConversationMetadata | undefined
   prompt: string
   onChangePrompt: (prompt: string) => void
   rightActions?: ReactNode
   disabledMetadataSelectors?: boolean
+  providers?: ProviderApiKey[]
 }) {
   const promptMetadata = useMemo<PromptMeta>(() => {
     const config = metadata?.config
@@ -68,7 +71,9 @@ export default function EditorHeader({
     return { providerName, model }
   }, [metadata?.config])
 
-  const { data: providerApiKeys, isLoading } = useProviderApiKeys()
+  const { data: providerApiKeys, isLoading } = useProviderApiKeys({
+    fallbackData: providers,
+  })
 
   const { value: showLineNumbers, setValue: setShowLineNumbers } =
     useLocalStorage({
@@ -114,23 +119,14 @@ export default function EditorHeader({
   })
 
   useEffect(() => {
-    if (isLoading) return
-
     const foundProvider = providersByName[promptMetadata?.providerName]
     if (foundProvider?.name === selectedProvider) return
 
     setProvider(foundProvider?.name)
     setModel(undefined)
-  }, [
-    isLoading,
-    selectedProvider,
-    providersByName,
-    promptMetadata?.providerName,
-  ])
+  }, [selectedProvider, providersByName, promptMetadata?.providerName])
 
   useEffect(() => {
-    if (isLoading) return
-
     const model = selectModel({
       promptMetadata,
       providersByName,
@@ -140,7 +136,6 @@ export default function EditorHeader({
 
     setModel(model)
   }, [
-    isLoading,
     providersByName,
     selectedModel,
     promptMetadata?.providerName,
@@ -225,7 +220,12 @@ export default function EditorHeader({
           onChange={onSelectProvider}
         />
         <Select
-          disabled={disabledMetadataSelectors || !selectedProvider || !metadata}
+          disabled={
+            isLoading ||
+            disabledMetadataSelectors ||
+            !selectedProvider ||
+            !metadata
+          }
           name='model'
           label='Model'
           placeholder='Select a model'

diff --git a/packages/web-ui/src/ds/atoms/Alert/index.tsx b/packages/web-ui/src/ds/atoms/Alert/index.tsx
@@ -13,13 +13,19 @@ type Props = {
   variant?: AlertProps['variant']
   title?: string
   description?: string
+  cta?: ReactNode
 }
-export function Alert({ title, description, variant = 'default' }: Props) {
+export function Alert({ title, description, cta, variant = 'default' }: Props) {
   return (
     <AlertRoot variant={variant}>
       <Icon name='alert' />
-      {title && <AlertTitle>{title}</AlertTitle>}
-      {description && <AlertDescription>{description}</AlertDescription>}
+      <div className='flex flex-row items-center gap-4 justify-between'>
+        <div className='flex flex-col gap-2'>
+          <AlertTitle>{title}</AlertTitle>
+          <AlertDescription>{description}</AlertDescription>
+        </div>
+        {cta}
+      </div>
     </AlertRoot>
   )
 }