Allow caching system, user and assistant messages for Anthropic (#524)

* Allow caching system, user and assistant messages for Anthropic We want to allow with our syntaxt for users to tag messages with Anthropic cache control tag. Also as part of this PR we extract first system messages in an anthropic prompt as part of the configuration. This way we can remove the warning And last change in this PR is that we allow system messages to have more than one message. This is something that's not valid in OpenAI but is valid in Anthropic so we allow this at compiler level but we restric for OpenAI at Rules level * chore: updated docs * chore: update docs 2 * chore: updated docs 3 * chore: updated docs 4 --------- Co-authored-by: Gerard Clos <[email protected]>
latitude-dev · Nov 5, 2024 · 3540dca · 3540dca
1 parent 899329c
commit 3540dca
Show file tree

Hide file tree

Showing 37 changed files with 1,643 additions and 521 deletions.
diff --git a/apps/gateway/package.json b/apps/gateway/package.json
@@ -7,7 +7,7 @@
   "scripts": {
     "dev": "tsx watch src/server",
     "build": "tsup --config tsup.config.ts && pnpm run sentry:sourcemaps",
-    "dev:debug": "tsx watch --inspect-brk src/server",
+    "dev:debug": "tsx watch --inspect src/server",
     "lint": "eslint src/",
     "prettier": "prettier --write \"**/*.{ts,tsx,md}\"",
     "tc": "tsc --noEmit",
@@ -38,7 +38,7 @@
     "@types/node": "^22.5.1",
     "@types/uuid": "^10.0.0",
     "tsup": "^8.2.4",
-    "tsx": "^4.16.2",
+    "tsx": "^4.19.2",
     "vitest": "^2.0.4"
   }
 }
diff --git a/.../[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx b/.../[evaluationUuid]/editor/_components/EvaluationEditor/Editor/Playground/Preview/index.tsx
@@ -77,6 +77,7 @@ export default function Preview({
         {error || (metadata?.errors.length ?? 0) > 0 ? (
           <Tooltip
             side='bottom'
+            asChild
             trigger={
               <Button fancy disabled>
                 Run prompt

diff --git a/...itUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Preview.tsx b/...itUuid]/documents/[documentUuid]/_components/DocumentEditor/Editor/Playground/Preview.tsx
@@ -10,6 +10,7 @@ import {
   AppliedRules,
   applyCustomRules,
   LATITUDE_DOCS_URL,
+  ProviderRules,
 } from '@latitude-data/core/browser'
 import {
   Alert,
@@ -27,6 +28,34 @@ import { ROUTES } from '$/services/routes'
 import useProviderApiKeys from '$/stores/providerApiKeys'
 import Link from 'next/link'
 
+function WarningLink({ providerRule }: { providerRule: ProviderRules }) {
+  return (
+    <Link
+      target='_blank'
+      href={`${LATITUDE_DOCS_URL}/guides/prompt-manager/custom-rules/${providerRule}`}
+      className='flex-nowrap'
+    >
+      <Text.H5B underline noWrap color='warningMutedForeground'>
+        Learn more
+      </Text.H5B>
+    </Link>
+  )
+}
+
+function Warnings({ warnings }: { warnings: AppliedRules }) {
+  const rules = warnings.rules
+  if (!rules.length) return null
+
+  return rules.map((rule, index) => (
+    <Alert
+      key={index}
+      variant='warning'
+      description={rule.ruleMessage}
+      cta={<WarningLink providerRule={rule.rule} />}
+    />
+  ))
+}
+
 export default function Preview({
   metadata,
   parameters,
@@ -91,6 +120,7 @@ export default function Preview({
     const rule = applyCustomRules({
       providerType: provider.provider,
       messages: conversation.messages,
+      config: conversation.config,
     })
 
     setFixedMessages(rule?.messages ?? conversation.messages)
@@ -99,7 +129,7 @@ export default function Preview({
 
   return (
     <div className='flex flex-col flex-1 gap-2 h-full overflow-hidden'>
-      <WarningMessage rule={warningRule} />
+      {warningRule ? <Warnings warnings={warningRule} /> : null}
       <div
         ref={containerRef}
         className='flex flex-col gap-3 flex-grow flex-shrink min-h-0 custom-scrollbar'
@@ -128,6 +158,7 @@ export default function Preview({
         {error || (metadata?.errors.length ?? 0) > 0 ? (
           <Tooltip
             side='bottom'
+            asChild
             trigger={
               <Button fancy disabled>
                 Run prompt
@@ -158,32 +189,3 @@ export default function Preview({
     </div>
   )
 }
-
-function WarningMessage({ rule }: { rule: AppliedRules | undefined }) {
-  if (!rule) return null
-
-  switch (rule.rule) {
-    case 'AnthropicMultipleSystemMessagesUnsupported':
-      return (
-        <Alert
-          variant='warning'
-          description={rule.ruleMessage}
-          cta={
-            <Link
-              target='_blank'
-              href={`${LATITUDE_DOCS_URL}/guides/prompt-manager/custom-rules`}
-              className='flex-nowrap'
-            >
-              <Text.H5B underline noWrap color='warningMutedForeground'>
-                Learn more
-              </Text.H5B>
-            </Link>
-          }
-        />
-      )
-    case 'GoogleSingleStartingSystemMessageSupported':
-      return <Alert variant='warning' description={rule.ruleMessage} />
-    default:
-      return null
-  }
-}
diff --git a/docs/assets/provider_rules_1.png b/docs/assets/provider_rules_1.png
diff --git a/docs/guides/getting-started/providers.mdx b/docs/guides/getting-started/providers.mdx
@@ -1,11 +1,11 @@
 ---
-title: Providers
+title: Setup Providers
 description: Learn how to add providers into Latitude to use it in your prompts.
 ---
 
 ## Overview
 
-Providers in Latitude are the foundation for connecting to various AI models and services. 
+Providers in Latitude are the foundation for connecting to various AI models and services.
 
 The name you introduce for each provider will be the one you will have to use in your prompts.
 

diff --git a/docs/guides/getting-started/providers/anthropic.mdx b/docs/guides/getting-started/providers/anthropic.mdx
@@ -0,0 +1,43 @@
+---
+title: Anthropic
+description: Common questions about Anthropic provider
+---
+
+## How do I use the Anthropic cache?
+
+[Anthropic cache](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) allows you to cache parts of a prompt. As explained in their documentation, you need to opt-in within the prompt to start caching parts of it.
+
+To do this, add `cacheControl: true` to the front matter of your prompt.
+
+```markdown
+---
+provider: name-of-your-antropic-provider-in-latitude
+model: claude-3-5-sonnet-20241022
+cacheControl: true
+---
+```
+
+Once this is set up, you can start caching specific parts of the prompt:
+
+```
+<system>This part of the text is not cached</system>
+<system>
+  Read this large book and answer users' questions.
+  <text cache_control={{ { type: 'ephemeral' } }}>
+    ...BIG_BOOK_CONTENT...
+  </text>
+</system>
+````
+
+If you want an entire message to be cached, add the cache directive to the `user`, `assistant`, or `system` tags:
+
+```
+<user cache_control={{ { type: 'ephemeral' } }}>
+  This text will be cached.
+  <text>
+    This text will also be cached.
+  </text>
+  And this text as well.
+</user>
+```
+
diff --git a/docs/guides/prompt-manager/custom-rules.mdx b/docs/guides/prompt-manager/custom-rules.mdx
@@ -7,27 +7,6 @@ description: Some providers have specific rules that you need to follow when usi
 
 Some providers have specific rules that you need to follow when using them in your prompts. These rules are called "custom rules" and are enforced by the Latitude engine.
 
-## Anthropic
-
-Anthropic only supports system messages at the beginning of the conversation. All other system messages get converted to user messages.
-
-![](/assets/provider_rules_1.png)
-
-You can add the system message via a property in the front matter of your prompt:
-
-```
----
-provider: Anthropic
-model: claude-3-5-sonnet-latest
-system: |
-  This is a multi-line system prompt
-  that spans multiple lines
----
-
-...
-
-```
-
-## Google
-
-Google only supports system messages at the beginning of the conversation. All other system messages are converted to user messages.
+## Provider rules
+- [Anthropic](/guides/prompt-manager/provider-rules/anthropic)
+- [Google](/guides/prompt-manager/provider-rules/google)
diff --git a/docs/guides/prompt-manager/provider-rules/anthropic.mdx b/docs/guides/prompt-manager/provider-rules/anthropic.mdx
@@ -0,0 +1,75 @@
+---
+title: Anthropic provider rules
+description: Learn about the rules you need to follow when using the Anthropic provider in Latitude.
+---
+
+1. [System messages must be followed by at least by another message](#rule-1-not-only-system-messages)
+2. [No system messages are allowed after an assistant or user message](#rule-2-no-system-messages-after-assistant-or-user-messages)
+
+### System messages must be followed by at least another message
+
+Anthropic does not consider system messages as part of the list of general messages and thus, if you don't add at least a user or assistant message, the list of messages sent to anthropic would be empty, which would result in an error.
+
+```json
+{
+  "system": [
+    {
+      "type": "text",
+      "text": "You are an AI assistant tasked with analyzing literary works. Your goal is to provide insightful commentary on themes, characters, and writing style.\n"
+    },
+    {
+      "type": "text",
+      "text": "<the entire contents of Pride and Prejudice>",
+      "cache_control": { "type": "ephemeral" }
+    }
+  ],
+  "messages": [] // this is invalid
+}
+```
+
+So, the following prompt in Latitude is invalid for an Anthropic provider:
+
+```
+---
+provider: Anthropic
+model: claude-3-5-sonnet-latest
+---
+
+This is a system message
+```
+
+This would generate the following warning:
+
+![](/assets/provider_rules_1.png)
+
+Instead, add at least another message wrapped in a `<user>` or `<assistant>` tag:
+
+```
+---
+provider: Anthropic
+model: claude-3-5-sonnet-latest
+---
+
+This is a system message
+
+<user>This is a user message</user>
+```
+
+### No system messages are allowed after an assistant or user message
+
+Any system message added after an assistant or user message will be automatically converted into a user message:
+
+```
+---
+provider: Anthropic
+model: claude-3-5-sonnet-latest
+---
+
+This is a system message
+
+<user>This is a user message</user>
+
+<system>This is another system message</system> /* This message will be converted to a user message */
+```
+
+This is because Anthropic does not consider system messages as part of the list of general messages and thus they can't be concatenated with other messages.
diff --git a/docs/guides/prompt-manager/provider-rules/google.mdx b/docs/guides/prompt-manager/provider-rules/google.mdx
@@ -0,0 +1,23 @@
+---
+title: Google provider rules
+description: Learn about the rules you need to follow when using the Google provider in Latitude.
+---
+
+1. [System messages must be at the beginning of the conversation](#rule-1-system-messages-must-be-at-the-beginning-of-the-conversation)
+
+### System messages must be at the beginning of the conversation
+
+Google only supports system messages at the beginning of the conversation. All other system messages are converted to user messages.
+
+```
+---
+provider: Google
+model: gemini-1.5-flash
+---
+
+This is a system message
+
+<user>This is a user message</user>
+
+This is another system message /* This message will be converted to a user message */
+```
diff --git a/docs/mint.json b/docs/mint.json
@@ -33,7 +33,13 @@
         "guides/getting-started/introduction",
         "guides/getting-started/concepts",
         "guides/getting-started/quick-start",
-        "guides/getting-started/providers",
+        {
+          "group": "Providers",
+          "pages": [
+            "guides/getting-started/providers",
+            "guides/getting-started/providers/anthropic"
+          ]
+        },
         "guides/getting-started/invite-your-team"
       ]
     },
@@ -50,7 +56,14 @@
         "guides/prompt-manager/version-control",
         "guides/prompt-manager/json-output",
         "guides/prompt-manager/tools",
-        "guides/prompt-manager/custom-rules",
+        {
+          "group": "Custom Rules",
+          "pages": [
+            "guides/prompt-manager/custom-rules",
+            "guides/prompt-manager/provider-rules/anthropic",
+            "guides/prompt-manager/provider-rules/google"
+          ]
+        },
         "guides/prompt-manager/cache"
       ]
     },
@@ -66,10 +79,7 @@
     },
     {
       "group": "Logs",
-      "pages": [
-        "guides/logs/overview",
-        "guides/logs/upload-logs"
-      ]
+      "pages": ["guides/logs/overview", "guides/logs/upload-logs"]
     },
     {
       "group": "Datasets",

diff --git a/packages/compiler/src/compiler/base/nodes/tags/content.ts b/packages/compiler/src/compiler/base/nodes/tags/content.ts
@@ -1,3 +1,4 @@
+import { removeCommonIndent } from '$compiler/compiler/utils'
 import errors from '$compiler/error/errors'
 import { ContentTag } from '$compiler/parser/interfaces'
 import { ContentType } from '$compiler/types'
@@ -15,7 +16,7 @@ export async function compile(
     popStrayText,
     addContent,
   }: CompileNodeContext<ContentTag>,
-  _: Record<string, unknown>,
+  attributes: Record<string, unknown>,
 ) {
   if (isInsideContentTag) {
     baseNodeError(errors.contentTagInsideContent, node)
@@ -29,17 +30,19 @@ export async function compile(
       isInsideContentTag: true,
     })
   }
-  const textContent = popStrayText()
+  const textContent = removeCommonIndent(popStrayText())
 
   // TODO: This if else is probably not required but the types enforce it.
   // Improve types.
   if (node.name === 'text') {
     addContent({
+      ...attributes,
       type: ContentType.text,
       text: textContent,
     })
   } else {
     addContent({
+      ...attributes,
       type: ContentType.image,
       image: textContent,
     })