feature: better evaluation editor

Instead of a random list of inputs we display a collapsible box with the
contents of the provider log.

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx

@@ -59,7 +59,6 @@ export default function Preview({
       ref={containerRef}
       className='flex flex-col gap-3 h-full overflow-y-auto'
     >
-      <Text.H6M>Preview</Text.H6M>
       {(conversation?.messages ?? [])
         .filter((message) => message.role === 'assistant')
         .map((message, index) => (

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/CopyButton.tsx

@@ -0,0 +1,16 @@
+import { Button, Icon, toast } from '@latitude-data/web-ui'
+
+export const CopyButton = ({ text }: { text: string }) => (
+  <Button
+    variant='nope'
+    onClick={() => {
+      navigator.clipboard.writeText(text)
+      toast({
+        title: 'Copied to clipboard',
+        description: 'The snippet has been copied to your clipboard',
+      })
+    }}
+  >
+    <Icon name='clipboard' color='foregroundMuted' />
+  </Button>
+)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/InputSection.tsx

@@ -0,0 +1,31 @@
+import { Badge } from '@latitude-data/web-ui'
+
+import { TooltipInfo } from './TooltipInfo'
+
+export const InputSection = ({
+  title,
+  content,
+  tooltip,
+  type = 'text',
+}: {
+  title: string
+  content: string
+  tooltip: string
+  type?: string
+}) => (
+  <div className='flex flex-col gap-2'>
+    <div className='flex flex-row gap-2 items-center'>
+      <Badge variant='accent'>{title}</Badge>
+      <TooltipInfo text={tooltip} />
+    </div>
+    <div>
+      <input
+        type={type}
+        className='p-2 rounded-lg border bg-secondary resize-y text-xs'
+        value={content}
+        disabled
+        readOnly
+      />
+    </div>
+  </div>
+)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/MessagesList.tsx

@@ -0,0 +1,14 @@
+import { Text } from '@latitude-data/web-ui'
+
+import { CopyButton } from './CopyButton'
+
+export const MessagesList = ({ items }: { items: string[] }) => (
+  <ul className='list-none space-y-2 max-w-1/3'>
+    {items.map((item, index) => (
+      <li key={index} className='flex flex-shrink items-center gap-2'>
+        <CopyButton text={item} />
+        <Text.H6M>{item}</Text.H6M>
+      </li>
+    ))}
+  </ul>
+)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/PinnedDocumentation.tsx

@@ -0,0 +1,51 @@
+import { cn, CodeBlock, CopyButton, Text } from '@latitude-data/web-ui'
+
+import { MessagesList } from './MessagesList'
+
+export const PinnedDocumentation = ({ isPinned }: { isPinned: boolean }) => {
+  const messageVariables = [
+    '{{messages.all}}',
+    '{{messages.first}}',
+    '{{messages.last}}',
+    '{{messages.user.all}}',
+    '{{messages.user.first}}',
+    '{{messages.user.last}}',
+    '{{messages.system.all}}',
+    '{{messages.system.first}}',
+    '{{messages.system.last}}',
+    '{{messages.assistant.all}}',
+    '{{messages.assistant.first}}',
+    '{{messages.assistant.last}}',
+  ]
+
+  return (
+    <div className={cn('flex flex-col gap-2', { 'py-4': !isPinned })}>
+      <Text.H6>
+        The <code>messages</code> variable contains all messages in the
+        conversation, with the following properties:
+      </Text.H6>
+      <div className='bg-backgroundCode p-4 rounded-lg'>
+        <div className='flex gap-4'>
+          <MessagesList items={messageVariables.slice(0, 6)} />
+          <MessagesList items={messageVariables.slice(6)} />
+        </div>
+      </div>
+      <Text.H6>
+        You can access these properties in your prompt template using JavaScript
+        object accessor syntax. E.g:
+      </Text.H6>
+      <div className='bg-backgroundCode p-4 rounded-lg relative overflow-x-auto'>
+        <Text.H6 noWrap>
+          {`{{messages.user.first}}`}{' '}
+          <Text.H6 noWrap color='foregroundMuted'>
+            /* This will print the first message from the user in the
+            conversation */
+          </Text.H6>
+        </Text.H6>
+        <div className='absolute top-4 right-2 bg-backgroundCode'>
+          <CopyButton content={`{{messages.user.first}}`} />
+        </div>
+      </div>
+    </div>
+  )
+}

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/TooltipInfo.tsx

@@ -0,0 +1,13 @@
+import { Icon, Tooltip } from '@latitude-data/web-ui'
+
+export const TooltipInfo = ({ text }: { text: string }) => (
+  <Tooltip
+    trigger={
+      <div>
+        <Icon name='info' color='primary' />
+      </div>
+    }
+  >
+    {text}
+  </Tooltip>
+)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/components/VariableSection.tsx

@@ -0,0 +1,72 @@
+import { ReactNode } from 'react'
+
+import { Badge, Button, cn, Icon, Popover, Text } from '@latitude-data/web-ui'
+
+import { PinnedDocumentation } from './PinnedDocumentation'
+import { TooltipInfo } from './TooltipInfo'
+
+export const VariableSection = ({
+  title,
+  content,
+  tooltip,
+  height = '36',
+  isPopoverOpen,
+  setIsPopoverOpen,
+  popoverContent,
+  isPinned,
+  onUnpin,
+}: {
+  title: string
+  content: string
+  tooltip?: string
+  height?: string
+  isPopoverOpen?: boolean
+  setIsPopoverOpen?: (isPopoverOpen: boolean) => void
+  popoverContent?: ReactNode
+  isPinned?: boolean
+  onUnpin?: () => void
+}) => (
+  <div className='flex flex-col gap-2'>
+    <div className='flex flex-row gap-2 items-center justify-between'>
+      <div className='flex flex-row gap-2 items-center'>
+        <Badge variant='accent'>{title}</Badge>
+        {tooltip && <TooltipInfo text={tooltip} />}
+      </div>
+      {popoverContent &&
+        (isPinned ? (
+          <Button variant='nope' onClick={onUnpin}>
+            <Icon name='pinOff' color='foregroundMuted' />
+            <Text.H6M color='foregroundMuted'>Unpin documentation</Text.H6M>
+          </Button>
+        ) : (
+          <Popover.Root open={isPopoverOpen} onOpenChange={setIsPopoverOpen}>
+            <Popover.Trigger asChild>
+              <Button variant='nope'>
+                <Icon name='circleHelp' color='foregroundMuted' />
+                <Text.H6M color='foregroundMuted'>View documentation</Text.H6M>
+              </Button>
+            </Popover.Trigger>
+            <Popover.Content
+              align='end'
+              className='bg-white shadow-lg rounded-lg p-4 max-w-xl'
+            >
+              {popoverContent}
+            </Popover.Content>
+          </Popover.Root>
+        ))}
+    </div>
+    {isPinned && popoverContent && <PinnedDocumentation isPinned />}
+    <textarea
+      className={cn(
+        `w-full p-2 rounded-lg border bg-secondary resize-y text-xs`,
+        {
+          'h-36': height === '36',
+          'h-24': height === '24',
+        },
+      )}
+      value={content}
+      disabled
+      readOnly
+    />
+  </div>
+)

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/hooks/useVariablesData.tsx

@@ -0,0 +1,125 @@
+import { useEffect, useState } from 'react'
+
+import { Config, readMetadata } from '@latitude-data/compiler'
+import { ProviderLogDto } from '@latitude-data/core/browser'
+import {
+  formatContext,
+  formatConversation,
+} from '@latitude-data/core/services/providerLogs/formatForEvaluation'
+import { Button, Icon, Tooltip } from '@latitude-data/web-ui'
+import useDocumentLogWithMetadata from '$/stores/documentLogWithMetadata'
+
+import { PinnedDocumentation } from '../components/PinnedDocumentation'
+
+export const useVariablesData = (providerLog: ProviderLogDto) => {
+  const [config, setConfig] = useState<Config>()
+  const { data: documentLogWithMetadata } = useDocumentLogWithMetadata(
+    providerLog.documentLogUuid,
+  )
+  const [isMessagesPinned, setIsMessagesPinned] = useState(false)
+  const [isPopoverOpen, setIsPopoverOpen] = useState(false)
+
+  useEffect(() => {
+    const fn = async () => {
+      if (!documentLogWithMetadata) return
+
+      const metadata = await readMetadata({
+        prompt: documentLogWithMetadata!.resolvedContent,
+      })
+      setConfig(metadata.config)
+    }
+
+    fn()
+  }, [documentLogWithMetadata])
+
+  const variableSections = [
+    {
+      title: 'messages',
+      content: JSON.stringify(formatConversation(providerLog), null, 2),
+      height: '36',
+      tooltip: 'The full conversation including the last assistant response',
+      popover: (
+        <div className='flex flex-col gap-4 relative pt-8'>
+          <div className='absolute top-2 right-2'>
+            <Button
+              variant='nope'
+              onClick={() => {
+                setIsMessagesPinned(true)
+                setIsPopoverOpen(false)
+              }}
+            >
+              <Tooltip
+                trigger={
+                  <div>
+                    <Icon
+                      name='pin'
+                      color={isMessagesPinned ? 'primary' : 'foregroundMuted'}
+                    />
+                  </div>
+                }
+              >
+                Pin documentation
+              </Tooltip>
+            </Button>
+          </div>
+          <PinnedDocumentation isPinned={false} />
+        </div>
+      ),
+    },
+    {
+      title: 'context',
+      content: formatContext(providerLog),
+      tooltip: 'The full conversation excluding the last assistant response',
+    },
+    {
+      title: 'response',
+      content: providerLog.response,
+      tooltip: 'The last assistant response',
+    },
+    {
+      title: 'prompt',
+      content: documentLogWithMetadata?.resolvedContent || '',
+      tooltip: 'The original prompt',
+    },
+    {
+      title: 'config',
+      content: config ? JSON.stringify(config, null, 2) : '',
+      tooltip: 'The prompt configuration used to generate the prompt',
+      height: '24',
+    },
+    {
+      title: 'parameters',
+      content: documentLogWithMetadata?.parameters
+        ? JSON.stringify(documentLogWithMetadata.parameters, null, 2)
+        : '',
+      tooltip: 'The parameters that were used to build the prompt for this log',
+      height: '24',
+    },
+  ]
+
+  const inputSections = [
+    {
+      title: 'duration',
+      content: documentLogWithMetadata?.duration?.toString() || '0',
+      tooltip: 'The time it took to run this prompt in milliseconds',
+      type: 'number',
+    },
+    {
+      title: 'cost',
+      content: documentLogWithMetadata?.costInMillicents
+        ? (documentLogWithMetadata?.costInMillicents / 1000).toString()
+        : '0',
+      tooltip: 'The cost of running this prompt in cents',
+      type: 'number',
+    },
+  ]
+
+  return {
+    variableSections,
+    inputSections,
+    isMessagesPinned,
+    setIsMessagesPinned,
+    isPopoverOpen,
+    setIsPopoverOpen,
+  }
+}

apps/web/src/app/(private)/evaluations/(evaluation)/[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Variables/index.tsx

@@ -0,0 +1,60 @@
+'use client'
+
+import { ProviderLogDto } from '@latitude-data/core/browser'
+import { Badge, CollapsibleBox } from '@latitude-data/web-ui'
+
+import { InputSection } from './components/InputSection'
+import { VariableSection } from './components/VariableSection'
+import { useVariablesData } from './hooks/useVariablesData'
+import { PARAMETERS } from './utils/constants'
+
+export const Variables = ({ providerLog }: { providerLog: ProviderLogDto }) => {
+  const {
+    variableSections,
+    inputSections,
+    isMessagesPinned,
+    setIsMessagesPinned,
+    isPopoverOpen,
+    setIsPopoverOpen,
+  } = useVariablesData(providerLog)
+
+  if (!providerLog) return null
+
+  const collapsedContent = (
+    <div className='flex flex-wrap gap-2'>
+      {PARAMETERS.map((param) => (
+        <Badge key={param} variant='accent'>
+          {param}
+        </Badge>
+      ))}
+    </div>
+  )
+
+  const expandedContent = (
+    <div className='flex flex-col gap-4'>
+      {variableSections.map((section) => (
+        <VariableSection
+          key={section.title}
+          {...section}
+          isPopoverOpen={isPopoverOpen}
+          setIsPopoverOpen={setIsPopoverOpen}
+          isPinned={isMessagesPinned}
+          onUnpin={() => setIsMessagesPinned(false)}
+          popoverContent={section.popover}
+        />
+      ))}
+      {inputSections.map((section) => (
+        <InputSection key={section.title} {...section} />
+      ))}
+    </div>
+  )
+
+  return (
+    <CollapsibleBox
+      title='Variables'
+      collapsedContent={collapsedContent}
+      expandedContent={expandedContent}
+      expandedHeight='720px'
+    />
+  )
+}