Updated branch with main (#739)

* Tokens in Evaluator Tests (#730) * Estimate price in Evaluator Tests (#731) * Enum description in tools (#732) * support enum descriptions * added example --------- Co-authored-by: José Carlos Montañez <[email protected]> Co-authored-by: Raúl Raja Martínez <[email protected]> * Update README with instruction to build locally (#725) * Add README instructions for building Xef * Include reasons why build may fail if you don't have docker * fixed error in enum description (#733) Co-authored-by: José Carlos Montañez <[email protected]> --------- Co-authored-by: Javier Pérez Pacheco <[email protected]> Co-authored-by: José Carlos Montañez <[email protected]> Co-authored-by: José Carlos Montañez <[email protected]> Co-authored-by: Raúl Raja Martínez <[email protected]>
xebia-functional · May 14, 2024 · d9dfd74 · d9dfd74
1 parent 2b8f99b
commit d9dfd74
Show file tree

Hide file tree

Showing 15 changed files with 281 additions and 216 deletions.
diff --git a/README.md b/README.md
@@ -76,3 +76,17 @@ In [this](https://xef.ai/learn/quickstart/) small introduction we look at the ma
 ## 🚀 Examples
 
 You can also have a look at the [examples](https://github.com/xebia-functional/xef/tree/main/examples/src/main/kotlin/com/xebia/functional/xef/conversation) to have a feeling of how using the library looks like.
+
+## 🚧 Local Development
+
+To build the project locally, you can use the following commands:
+
+```shell
+./gradlew downloadOpenAIAPI
+./gradlew openaiClientGenerate
+./gradlew build
+```
+
+The server and postgres tests may fail if you don't have [Docker](https://www.docker.com/) installed. 
+The server and postgres related tests depend on [Testcontainers](https://testcontainers.com/), which in turn depends on Docker.
+
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt
@@ -2,11 +2,17 @@ package com.xebia.functional.xef.llm
 
 import com.xebia.functional.openai.generated.api.Chat
 import com.xebia.functional.openai.generated.model.CreateChatCompletionRequest
+import com.xebia.functional.openai.generated.model.CreateChatCompletionResponse
+import com.xebia.functional.openai.generated.model.CreateChatCompletionResponseChoicesInner
 import com.xebia.functional.xef.AIError
 import com.xebia.functional.xef.conversation.AiDsl
 import com.xebia.functional.xef.conversation.Conversation
+import com.xebia.functional.xef.llm.models.MessageWithUsage
+import com.xebia.functional.xef.llm.models.MessagesUsage
+import com.xebia.functional.xef.llm.models.MessagesWithUsage
 import com.xebia.functional.xef.prompt.Prompt
 import com.xebia.functional.xef.prompt.PromptBuilder
+import com.xebia.functional.xef.store.Memory
 import kotlinx.coroutines.flow.*
 
 @AiDsl
@@ -54,9 +60,34 @@ suspend fun Chat.promptMessage(prompt: Prompt, scope: Conversation = Conversatio
 suspend fun Chat.promptMessages(
   prompt: Prompt,
   scope: Conversation = Conversation()
-): List<String> =
+): List<String> = promptResponse(prompt, scope) { it.message.content }.first
+
+@AiDsl
+suspend fun Chat.promptMessageAndUsage(
+  prompt: Prompt,
+  scope: Conversation = Conversation()
+): MessageWithUsage {
+  val response = promptMessagesAndUsage(prompt, scope)
+  val message = response.messages.firstOrNull() ?: throw AIError.NoResponse()
+  return MessageWithUsage(message, response.usage)
+}
+
+@AiDsl
+suspend fun Chat.promptMessagesAndUsage(
+  prompt: Prompt,
+  scope: Conversation = Conversation()
+): MessagesWithUsage {
+  val response = promptResponse(prompt, scope) { it.message.content }
+  return MessagesWithUsage(response.first, response.second.usage?.let { MessagesUsage(it) })
+}
+
+private suspend fun <T> Chat.promptResponse(
+  prompt: Prompt,
+  scope: Conversation = Conversation(),
+  block: suspend Chat.(CreateChatCompletionResponseChoicesInner) -> T?
+): Pair<List<T>, CreateChatCompletionResponse> =
   scope.metric.promptSpan(prompt) {
-    val promptMemories = prompt.messages.toMemory(scope)
+    val promptMemories: List<Memory> = prompt.messages.toMemory(scope)
     val adaptedPrompt = PromptCalculator.adaptPromptToConversationAndModel(prompt, scope)
 
     adaptedPrompt.addMetrics(scope)
@@ -72,13 +103,17 @@ suspend fun Chat.promptMessages(
         seed = adaptedPrompt.configuration.seed,
       )
 
-    createChatCompletion(request)
-      .addMetrics(scope)
-      .choices
-      .addChoiceToMemory(
-        scope,
-        promptMemories,
-        prompt.configuration.messagePolicy.addMessagesToConversation
-      )
-      .mapNotNull { it.message.content }
+    val createResponse: CreateChatCompletionResponse = createChatCompletion(request)
+    Pair(
+      createResponse
+        .addMetrics(scope)
+        .choices
+        .addChoiceToMemory(
+          scope,
+          promptMemories,
+          prompt.configuration.messagePolicy.addMessagesToConversation
+        )
+        .mapNotNull { block(it) },
+      createResponse
+    )
   }
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/Messages.kt
@@ -0,0 +1,18 @@
+package com.xebia.functional.xef.llm.models
+
+import com.xebia.functional.openai.generated.model.CompletionUsage
+
+data class MessagesWithUsage(val messages: List<String>, val usage: MessagesUsage?)
+
+data class MessageWithUsage(val message: String, val usage: MessagesUsage?)
+
+data class MessagesUsage(val completionTokens: Int, val promptTokens: Int, val totalTokens: Int) {
+  companion object {
+    operator fun invoke(usage: CompletionUsage) =
+      MessagesUsage(
+        completionTokens = usage.completionTokens,
+        promptTokens = usage.promptTokens,
+        totalTokens = usage.totalTokens
+      )
+  }
+}
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/functions/JsonSchema.kt
@@ -307,6 +307,7 @@ private fun SerialDescriptor.createJsonSchema(
   }
 }
 
+@OptIn(ExperimentalSerializationApi::class)
 private fun JsonObjectBuilder.applyJsonSchemaDefaults(
   descriptor: SerialDescriptor,
   annotations: List<Annotation>,
@@ -322,25 +323,31 @@ private fun JsonObjectBuilder.applyJsonSchemaDefaults(
     }
   }
 
-  if (descriptor.kind == SerialKind.ENUM) {
-    this["enum"] = descriptor.elementNames
-  }
-
-  if (annotations.isNotEmpty()) {
-    val multiplatformDescription = annotations.filterIsInstance<Description>()
-    val description =
-      if (multiplatformDescription.isEmpty()) {
-        try {
-          val jvmDescription = annotations.filterIsInstance<Description>()
-          jvmDescription.firstOrNull()?.value
-        } catch (e: Throwable) {
-          null
+  val additionalEnumDescription: String? =
+    if (descriptor.kind == SerialKind.ENUM) {
+      this["enum"] = descriptor.elementNames
+      descriptor.elementNames
+        .mapIndexedNotNull { index, name ->
+          val enumDescription =
+            descriptor.getElementAnnotations(index).lastOfInstance<Description>()?.value
+          if (enumDescription != null) {
+            "$name ($enumDescription)"
+          } else {
+            null
+          }
         }
-      } else {
-        multiplatformDescription.firstOrNull()?.value
-      }
+        .joinToString("\n - ")
+    } else null
 
-    this["description"] = description
+  if (annotations.isNotEmpty()) {
+    val description = annotations.filterIsInstance<Description>().firstOrNull()?.value
+    if (!additionalEnumDescription.isNullOrEmpty()) {
+      this["description"] = "$description\n - $additionalEnumDescription"
+    } else {
+      this["description"] = description
+    }
+  } else if (additionalEnumDescription != null) {
+    this["description"] = " - $additionalEnumDescription"
   }
 }
 

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt
@@ -44,6 +44,7 @@ data class SuiteSpec(
               output.description.value,
               item.context,
               output.value,
+              output.tokens,
               classification,
               success.contains(classification)
             )
@@ -67,10 +68,9 @@ data class SuiteSpec(
     E : Enum<E> =
       Html.get(Json.encodeToString(SuiteResults.serializer(serializer<E>()), result), suiteName)
 
-    inline fun <reified E> toMarkdown(
-      result: SuiteResults<E>,
-      suiteName: String,
-    ): Markdown where E : AI.PromptClassifier, E : Enum<E> = Markdown.get(result, suiteName)
+    inline fun <reified E> toMarkdown(result: SuiteResults<E>, suiteName: String): Markdown where
+    E : AI.PromptClassifier,
+    E : Enum<E> = Markdown.get(result, suiteName)
   }
 }
 

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt
@@ -56,10 +56,14 @@ value class Html(val value: String) {
                     const outputDiv = document.createElement('pre');
                     outputDiv.classList.add('output');
                     outputDiv.innerText = 'Output: ' + test.output;
-                    outputDiv.addEventListener('click', function() {
-                        this.classList.toggle('expanded');
-                    });
                     blockDiv.appendChild(outputDiv);
+                    
+                    if (test.usage != undefined) {
+                        const usageDiv = document.createElement('pre');
+                        usageDiv.classList.add('output');
+                        usageDiv.innerText = 'Usage: \n Prompt Tokens: ' + test.usage?.promptTokens + ' (~' + test.usage?.estimatePricePerToken + ' ' + test.usage?.currency + ')\n Completion Tokens: ' + test.usage?.completionTokens + ' (~' + test.usage?.estimatePriceCompletionToken + ' ' + test.usage?.currency + ')\n Total Tokens: ' + test.usage?.totalTokens + '\n Total Price: ~' + test.usage?.estimatePriceTotalToken + ' ' + test.usage?.currency;
+                        blockDiv.appendChild(usageDiv);
+                    }
         
                     const result = document.createElement('div');
                     result.classList.add('score', test.success ? 'score-passed' : 'score-failed');
@@ -99,6 +103,10 @@ value class Html(val value: String) {
               border-bottom: 1px solid #eee;
               padding-bottom: 20px;
           }
+          
+          .test-block pre {
+              margin-bottom: 20px;
+          }
 
           .test-title {
               font-size: 1.2em;
@@ -123,16 +131,11 @@ value class Html(val value: String) {
 
           .output {
               color: #666;
-              cursor: pointer;
-              white-space: nowrap;
+              white-space: normal;
               overflow: hidden;
               text-overflow: ellipsis;
           }
 
-          .output.expanded {
-              white-space: normal;
-          }
-
           .score {
               font-weight: bold;
           }

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt
@@ -21,6 +21,7 @@ data class OutputResult<E>(
   val description: String,
   val contextDescription: String,
   val output: String,
+  val usage: OutputTokens?,
   val result: E,
   val success: Boolean
 ) where E : AI.PromptClassifier, E : Enum<E>
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt
@@ -28,6 +28,17 @@ value class Markdown(val value: String) {
                 |<blockquote>
                 |${outputResult.output}
                 |</blockquote>
+                |- Usage:
+                |<blockquote>
+                |${outputResult.usage?.let { usage ->
+                  """
+                  |Prompt Tokens: ${usage.promptTokens} ${usage.estimatePricePerToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" }
+                  |Completion Tokens: ${usage.completionTokens} ${usage.estimatePriceCompletionToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" }
+                  |Total Tokens: ${usage.totalTokens}
+                  |Total Price: ${usage.estimatePriceTotalToken?.let { "${it.to2DecimalsString()} ${usage.currency ?: ""}" } ?: "Unknown"}
+                  """.trimMargin()
+                } ?: "No usage information available"}
+                |</blockquote>
                 |
                 |Result: ${if (outputResult.success) "✅ Success" else "❌ Failure"} (${outputResult.result})
               """.trimMargin()
@@ -40,5 +51,7 @@ value class Markdown(val value: String) {
           .trimMargin()
       return Markdown(content)
     }
+
+    private fun Double.to2DecimalsString() = String.format("%.6f", this)
   }
 }
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt
@@ -0,0 +1,44 @@
+package com.xebia.functional.xef.evaluator.models
+
+data class ModelsPricing(
+  val modelName: String,
+  val currency: String,
+  val input: ModelsPricingItem,
+  val output: ModelsPricingItem
+) {
+
+  companion object {
+
+    const val oneMillion = 1_000_000
+    val oneThousand = 1_000
+
+    // The pricing for the models was updated the May 2st, 2024
+    // Be sure to update the pricing for each model
+
+    val gpt4Turbo =
+      ModelsPricing(
+        modelName = "gpt-4-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(10.0, oneMillion),
+        output = ModelsPricingItem(30.0, oneMillion)
+      )
+
+    val gpt4 =
+      ModelsPricing(
+        modelName = "gpt-4-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(30.0, oneMillion),
+        output = ModelsPricingItem(60.0, oneMillion)
+      )
+
+    val gpt3_5Turbo =
+      ModelsPricing(
+        modelName = "gpt-3.5-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(0.5, oneMillion),
+        output = ModelsPricingItem(1.5, oneMillion)
+      )
+  }
+}
+
+data class ModelsPricingItem(val price: Double, val perTokens: Int)
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt
@@ -1,17 +1,64 @@
 package com.xebia.functional.xef.evaluator.models
 
+import com.xebia.functional.xef.llm.models.MessageWithUsage
+import com.xebia.functional.xef.llm.models.MessagesUsage
 import kotlin.jvm.JvmSynthetic
 import kotlinx.serialization.Serializable
 
 @Serializable data class OutputDescription(val value: String)
 
 @Serializable
-data class OutputResponse(val description: OutputDescription, val value: String) {
+data class OutputResponse(
+  val description: OutputDescription,
+  val tokens: OutputTokens?,
+  val value: String
+) {
   companion object {
     @JvmSynthetic
     suspend operator fun invoke(
       description: OutputDescription,
-      block: suspend () -> String
-    ): OutputResponse = OutputResponse(description, block())
+      price: ModelsPricing?,
+      block: suspend () -> MessageWithUsage
+    ): OutputResponse {
+      val response = block()
+      return OutputResponse(
+        description,
+        response.usage?.let { OutputTokens(it, price) },
+        response.message
+      )
+    }
+  }
+}
+
+@Serializable
+data class OutputTokens(
+  val promptTokens: Int? = null,
+  val estimatePricePerToken: Double? = null,
+  val completionTokens: Int? = null,
+  val estimatePriceCompletionToken: Double? = null,
+  val totalTokens: Int? = null,
+  val estimatePriceTotalToken: Double? = null,
+  val currency: String?
+) {
+  companion object {
+    @JvmSynthetic
+    operator fun invoke(usage: MessagesUsage, price: ModelsPricing?): OutputTokens {
+      val estimateInputPrice =
+        price?.let { usage.promptTokens.let { (it * price.input.price) / price.input.perTokens } }
+      val estimateOutputPrice =
+        price?.let {
+          usage.completionTokens.let { (it * price.output.price) / price.output.perTokens }
+        }
+      val estimateTotalPrice = estimateInputPrice?.plus(estimateOutputPrice ?: 0.0)
+      return OutputTokens(
+        usage.promptTokens,
+        estimateInputPrice,
+        usage.completionTokens,
+        estimateOutputPrice,
+        usage.totalTokens,
+        estimateTotalPrice,
+        price?.currency
+      )
+    }
   }
 }