From 4f24b2d756fe80c99e687e574273f0891983a305 Mon Sep 17 00:00:00 2001 From: Anish Shah Date: Wed, 5 Jun 2024 09:50:51 -0400 Subject: [PATCH] Create azure-weave-cookbook-colab.ipynb --- colabs/azure/azure-weave-cookbook-colab.ipynb | 882 ++++++++++++++++++ 1 file changed, 882 insertions(+) create mode 100644 colabs/azure/azure-weave-cookbook-colab.ipynb diff --git a/colabs/azure/azure-weave-cookbook-colab.ipynb b/colabs/azure/azure-weave-cookbook-colab.ipynb new file mode 100644 index 00000000..b3660b95 --- /dev/null +++ b/colabs/azure/azure-weave-cookbook-colab.ipynb @@ -0,0 +1,882 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "\"Open\n", + "" + ], + "metadata": { + "id": "XKTIwr_z-0AF" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Setup" + ], + "metadata": { + "id": "J8ry0yKIMpJW" + } + }, + { + "cell_type": "code", + "source": [ + "%%capture\n", + "!pip install weave openai" + ], + "metadata": { + "id": "kpZYVrQqEaMU" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "model_id = \"gpt-4-turbo\" # @param {type:\"string\"}\n", + "model_id = \"mistral-7b-instruct-weave\"\n", + "azure_model_option = \"openai\" # @param [\"openai\", \"ai_studio\"]" + ], + "metadata": { + "id": "0MSgoTl1fLed" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "wandb_entity = \"a-sh0ts\" # @param {type:\"string\"}\n", + "weave_project = \"azure-weave-cookbook\" # @param {type:\"string\"}\n", + "eval_dataset_name = \"customer_service_inquiries\" # @param {type:\"string\"}\n", + "publish_eval_data = True # @param {type:\"boolean\"}" + ], + "metadata": { + "id": "jUukcDuSlXVi" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from google.colab import userdata\n", + "import os\n", + "from openai import AzureOpenAI, OpenAI\n", + "\n", + "os.environ[\"WANDB_API_KEY\"] = userdata.get('WANDB_API_KEY')\n", + "\n", + "if azure_model_option == \"openai\":\n", + " os.environ[\"AZURE_OPENAI_ENDPOINT\"] = userdata.get('AZURE_OPENAI_ENDPOINT')\n", + " os.environ[\"AZURE_OPENAI_API_KEY\"] = userdata.get('AZURE_OPENAI_API_KEY')\n", + " client = AzureOpenAI(\n", + " api_key=os.getenv(\"AZURE_API_KEY\"),\n", + " api_version=\"2024-02-01\",\n", + " azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n", + " )\n", + "elif azure_model_option == \"ai_studio\":\n", + " os.environ[\"AZURE_AI_STUDIO_API_ENDPOINT\"] = userdata.get('AZURE_AI_STUDIO_API_ENDPOINT')\n", + " os.environ[\"AZURE_AI_STUDIO_API_KEY\"] = userdata.get('AZURE_AI_STUDIO_API_KEY')\n", + "\n", + " api_version = \"v1\"\n", + " client = OpenAI(\n", + " base_url=f\"{os.getenv('AZURE_AI_STUDIO_API_ENDPOINT')}/v1\",\n", + " api_key=os.getenv('AZURE_AI_STUDIO_API_KEY')\n", + " )\n", + "else:\n", + " print(\"Please us one of the above options\")" + ], + "metadata": { + "id": "u8f8n2yQEZeI" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "import weave\n", + "weave.init(weave_project)" + ], + "metadata": { + "id": "HP363H3VJZK6", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "e6a60fca-5895-4981-995c-ce3375e59c46" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Logged in as Weights & Biases user: a-sh0ts.\n", + "View Weave data at https://wandb.ai/a-sh0ts/azure-weave-cookbook/weave\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [] + }, + "metadata": {}, + "execution_count": 5 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Calling Azure directly" + ], + "metadata": { + "id": "3Hm8EOJkMtlP" + } + }, + { + "cell_type": "code", + "source": [ + "@weave.op()\n", + "def call_azure_chat(model_id: str, messages: list, max_tokens: int = 1000, temperature: float = 0.5):\n", + " response = client.chat.completions.create(\n", + " model=model_id,\n", + " messages=messages,\n", + " max_tokens=max_tokens,\n", + " temperature=temperature\n", + " )\n", + " return {\"status\": \"success\", \"response\": response.choices[0].message.content}" + ], + "metadata": { + "id": "ObIjLY5-HdIb" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "@weave.op()\n", + "def format_messages_for_mistral(messages: list):\n", + " system_message = messages[0][\"content\"]\n", + " formatted_messages = []\n", + "\n", + " for message in messages[1:]:\n", + " if message[\"role\"] == \"user\":\n", + " formatted_message = {\n", + " \"role\": \"user\",\n", + " \"content\": f\"[INST]\\n{system_message}\\n{message['content']}\\n[/INST]\"\n", + " }\n", + " else:\n", + " formatted_message = {\n", + " \"role\": message[\"role\"],\n", + " \"content\": message[\"content\"]\n", + " }\n", + " formatted_messages.append(formatted_message)\n", + "\n", + " return formatted_messages" + ], + "metadata": { + "id": "JUt1eJX1h5ul" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"Create a snack recipe for a dish called the Azure Weav-e-ohs\"}\n", + "]\n", + "if \"mistral\" in model_id.lower():\n", + " messages = format_messages_for_mistral(messages)\n", + "result = call_azure_chat(model_id, messages)\n", + "print(result)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "gb6nxJgiImtq", + "outputId": "287afb82-a9e7-4e35-a56a-5025e0609905" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "🍩 https://wandb.ai/a-sh0ts/azure-weave-cookbook/r/call/ead4a121-98e5-4cd8-bb73-2ccf4161c198\n", + "{'status': 'success', 'response': \"Certainly! The Azure Weav-e-ohs snack is inspired by its name, suggesting a vibrant blue color and a weave or interlaced pattern. This playful and creative snack can be a hit at parties or a fun project with kids. Let's dive into making this whimsical treat.\\n\\n### Azure Weav-e-ohs Recipe\\n\\n#### Ingredients:\\n- **Blue Corn Tortillas** - For the azure color and base.\\n- **Cream Cheese** - Acts as a delicious adhesive.\\n- **Blueberries** - For a burst of color and sweetness.\\n- **Blue Curacao Syrup** (Non-alcoholic) - For added color and a hint of citrus flavor.\\n- **Shredded Coconut** - Optional, for added texture.\\n- **Powdered Sugar** - For a light dusting.\\n\\n#### Equipment:\\n- **Baking Sheet**\\n- **Parchment Paper**\\n- **Knife**\\n- **Small Mixing Bowl**\\n\\n#### Instructions:\\n\\n1. **Prep the Tortillas:**\\n - Preheat your oven to 350°F (175°C).\\n - Take the blue corn tortillas and cut them into strips, about 1 inch wide. Then, cut the strips into 3-inch lengths. You'll want an even number of strips since they'll be woven together.\\n\\n2. **Weave the Tortillas:**\\n - On a parchment paper-lined baking sheet, start weaving the tortilla strips together. Lay half of them vertically, slightly apart. Then, weave the remaining strips horizontally, over and under the vertical strips. You're aiming for a checkerboard pattern.\\n - Lightly press down the woven tortilla sheet to ensure it stays together during baking.\\n\\n3. **Bake:**\\n - Bake in the preheated oven for 5-7 minutes, or until the edges start to turn crisp. Be careful not to overbake; you want them to be just crispy enough to hold their shape without becoming too hard.\\n - Remove from the oven and let them cool on the baking sheet.\\n\\n4. **Prepare the Topping:**\\n - In a small mixing bowl, mix cream cheese with a small amount of blue Curacao syrup until you get a light blue color. Adjust the amount of syrup based on the color intensity you desire.\\n\\n5. **Assemble the Azure Weav-e-ohs:**\\n - Once the woven tortilla bases have cooled, carefully spread the blue cream cheese mixture over the top. Be gentle to not break the weave.\\n - Sprinkle with blueberries and optional shredded coconut for added texture and flavor.\\n - For a final touch, dust lightly with powdered sugar.\\n\\n6. **Serve:**\\n - Serve immediately, or refrigerate for a short time before serving to allow the cream cheese to firm up a bit.\\n\\n#### Tips:\\n- For a more intense blue color, you can add a few drops of food coloring to the cream cheese mixture.\\n- Feel free to experiment with other blue fruits, like diced dragon fruit (pitaya) or blackberries, depending on their availability and your preference.\\n\\nEnjoy crafting and munching on your Azure Weav-e-ohs, a snack that's as fun to make as it is to eat!\"}\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Creating Functional LLM Apps" + ], + "metadata": { + "id": "T-W7qZRPMyPx" + } + }, + { + "cell_type": "code", + "source": [ + "@weave.op()\n", + "def format_prompt(prompt: str):\n", + " \"A formatting function for OpenAI models\"\n", + " system_prompt_formatted = \"You are a helpful assistant.\"\n", + "\n", + " human_prompt = \"\"\"\n", + " {prompt}\n", + " \"\"\"\n", + "\n", + " human_prompt_formatted = human_prompt.format(prompt=prompt)\n", + " messages = [{\"role\":\"system\", \"content\":system_prompt_formatted}, {\"role\":\"user\", \"content\":human_prompt_formatted}]\n", + " return messages" + ], + "metadata": { + "id": "czIAwNzmM2dv" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "@weave.op()\n", + "def run_chat(model_id: str, prompt: str):\n", + " formatted_messages = format_prompt(prompt=prompt)\n", + " if \"mistral\" in model.lower():\n", + " formatted_messages = format_messages_for_mistral(formatted_messages)\n", + " result = call_azure_chat(model_id, formatted_messages, max_tokens=1000)\n", + " return result" + ], + "metadata": { + "id": "eCfVI-yEUsR4" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "prompt = \"Give a full recipe for a Weights & Biases inspired cocktail. Ensure you provide a list of ingredients, tools, and step by step instructions\"" + ], + "metadata": { + "id": "FMggaRlzTpe2" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "result = run_chat(model_id, prompt)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PFMRR-w3UCTN", + "outputId": "7298e0aa-3be4-41d4-b8a2-66a7b03334d3" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "🍩 https://wandb.ai/a-sh0ts/azure-weave-cookbook/r/call/67eb7a86-faba-4160-a9a4-3ce145ea1b40\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "result['response']" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 105 + }, + "id": "91gjDjd4UX6U", + "outputId": "02a752bc-38e1-46e6-c002-d46e65812f45" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'Creating a cocktail inspired by Weights & Biases, let\\'s blend the themes of precision, experimentation, and the joy of discovery into a drink I\\'ll call the \"Gradient Descent Gimlet.\" This cocktail is a twist on the classic gimlet, incorporating elements that symbolize the iterative process of machine learning and the vibrant, dynamic nature of data visualization.\\n\\n### Ingredients:\\n\\n- 2 oz Gin (representing the base or \"model\" of your cocktail)\\n- 0.75 oz Fresh Lime Juice (for the sharp, precise acidity that mirrors the clarity of insights)\\n- 0.5 oz Simple Syrup (to balance, akin to tuning hyperparameters)\\n- 0.25 oz Blue Curacao (for a pop of color, symbolizing data visualization)\\n- A dash of Butterfly Pea Flower Tea (for a gradient effect, representing the iterative learning process)\\n- Ice\\n- Edible Glitter (optional, for that extra visual effect of data in motion)\\n\\n### Tools:\\n\\n- Cocktail Shaker\\n- Strainer\\n- Measuring Jigger\\n- Martini Glass or Coupe\\n\\n### Instructions:\\n\\n1. **Preparation:** Begin by chilling your glass. Fill it with ice or place it in the freezer for a few minutes. This ensures your cocktail stays cold, enhancing the experience.\\n\\n2. **Mix Your Base:** In a cocktail shaker, combine the gin, fresh lime juice, and simple syrup. These ingredients form the foundation of your cocktail, much like setting up the initial parameters for a machine learning model.\\n\\n3. **Add Color:** Pour in the blue curacao. This not only adds a unique flavor but also introduces a vibrant blue hue to your drink, reminiscent of the striking visuals often seen in data models and graphs.\\n\\n4. **Cool Down:** Fill your shaker with ice, sealing it tight. Shake vigorously for about 15 seconds. This not only chills the drink but also dilutes it slightly, ensuring a smooth, well-balanced cocktail.\\n\\n5. **Create the Gradient:** Strain the mixture into your chilled glass. Slowly pour a dash of the butterfly pea flower tea over the back of a spoon or through a small strainer onto the drink. Watch as it creates a beautiful gradient effect, symbolizing the gradual learning and improvement of a model.\\n\\n6. **Final Touch:** For an added visual effect, sprinkle a tiny amount of edible glitter over the top. This represents the spark of discovery and the magic of insights gleaned from data analysis.\\n\\n7. **Enjoy:** Sip and enjoy the \"Gradient Descent Gimlet,\" a cocktail that not only delights the senses but also pays homage to the iterative, enlightening journey of working with Weights & Biases.\\n\\nThis cocktail is perfect for data scientists, engineers, and anyone who appreciates the beauty of machine learning and the art of mixology. Cheers to discovery and innovation!'" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + } + }, + "metadata": {}, + "execution_count": 13 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Create LLM Model Classes to iterate over hyperparameters" + ], + "metadata": { + "id": "JUfZiRBzTqbM" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IcGm-l9jESks" + }, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "\n", + "@dataclass\n", + "class PromptTemplate:\n", + " system_prompt: str\n", + " human_prompt: str\n", + "\n", + " @weave.op()\n", + " def format_prompt(self, email_content: str):\n", + " \"A formatting function for OpenAI models\"\n", + " system_prompt_formatted = self.system_prompt.format()\n", + " human_prompt_formatted = self.human_prompt.format(email_content=email_content)\n", + " messages = [{\"role\":\"system\", \"content\":system_prompt_formatted}, {\"role\":\"user\", \"content\":human_prompt_formatted}]\n", + " return messages" + ] + }, + { + "cell_type": "code", + "source": [ + "from weave import Model\n", + "from typing import Tuple\n", + "\n", + "class AzureEmailAssistant(Model):\n", + " model_id: str = model_id\n", + " prompt_template: PromptTemplate\n", + " max_tokens: int = 2048\n", + " temperature: float = 0.0\n", + "\n", + " @weave.op()\n", + " def format_doc(self, doc: str) -> list:\n", + " \"Read and format the document\"\n", + " messages = self.prompt_template.format_prompt(doc)\n", + " return messages\n", + "\n", + " @weave.op()\n", + " def respond(self, doc: str) -> dict:\n", + " \"Generate a response to the email inquiry\"\n", + " messages = self.format_doc(doc)\n", + " if \"mistral\" in self.model_id.lower():\n", + " messages = format_messages_for_mistral(messages)\n", + " output = call_azure_chat(\n", + " self.model_id,\n", + " messages=messages,\n", + " max_tokens=self.max_tokens,\n", + " temperature=self.temperature)\n", + " return output\n", + "\n", + " @weave.op()\n", + " async def predict(self, email_content: str) -> str:\n", + " return self.respond(email_content)[\"response\"]\n" + ], + "metadata": { + "id": "AjwPUM4PEYbj" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "%%writefile customer_inquiry.txt\n", + "Subject: Inquiry about Order Delay\n", + "\n", + "Hello,\n", + "\n", + "I placed an order last week for the new UltraGlow Skin Serum, but I have not received a shipping update yet. My order number is 12345. Could you please update me on the status of my shipment?\n", + "\n", + "Thank you,\n", + "Jane Doe" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "jyo-LElfa5Rf", + "outputId": "b583c4f0-3064-4229-e50d-49e7428e18de" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Overwriting customer_inquiry.txt\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "system_prompt = \"\"\"\n", + "# Instructions\n", + "You are a customer service response assistant. Our goal is to provide clear, concise, and polite responses to customer inquiries about products, shipping, and any issues they may have encountered. Some rules to remember:\n", + "- Always be courteous and respectful.\n", + "- Provide accurate and helpful information.\n", + "- Responses should be concise and to the point.\n", + "- Use formal language suitable for professional communication.\n", + "## Formatting Rules\n", + "Maintain a formal greeting and closing in each response. Do not use slang or overly casual language. Ensure all provided information is correct and double-check for typographical errors.\n", + "\"\"\"\n", + "\n", + "human_prompt = \"\"\"\n", + "Here is a customer inquiry received via email. Craft a suitable response based on the guidelines provided:\n", + "\n", + "{email_content}\n", + "\n", + "\"\"\"" + ], + "metadata": { + "id": "055TCwJCreFc" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "prompt_template = PromptTemplate(\n", + " system_prompt=system_prompt,\n", + " human_prompt=human_prompt)" + ], + "metadata": { + "id": "_fU5h0GVMESp" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "weave.init(weave_project) # Colab specific" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "x9IOgy5apHAI", + "outputId": "47e51318-4726-4d18-bd44-ffe5a2d715fc" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Logged in as Weights & Biases user: a-sh0ts.\n", + "View Weave data at https://wandb.ai/a-sh0ts/azure-weave-cookbook/weave\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [] + }, + "metadata": {}, + "execution_count": 18 + } + ] + }, + { + "cell_type": "code", + "source": [ + "from pathlib import Path" + ], + "metadata": { + "id": "wd4wN7sqriAv" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "doc = Path('customer_inquiry.txt').read_text()\n", + "model = AzureEmailAssistant(model_id=model_id, prompt_template=prompt_template)\n", + "response = model.respond(doc)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5ixTluHFbu0k", + "outputId": "7140d0b4-1531-4b86-df73-3ef6df480a8c" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "🍩 https://wandb.ai/a-sh0ts/azure-weave-cookbook/r/call/589a01db-901e-49a2-8749-3849856e569d\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "print(response[\"response\"])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qmEh7O2Bb2kr", + "outputId": "b16891b9-b429-4f7b-c890-3e65dcad2b70" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Subject: Update on Your Order #12345 - UltraGlow Skin Serum\n", + "\n", + "Dear Jane Doe,\n", + "\n", + "Thank you for reaching out to us regarding your recent order of the UltraGlow Skin Serum. We understand how important it is for you to receive your order promptly and apologize for any inconvenience the delay may have caused.\n", + "\n", + "Upon reviewing your order #12345, we have noticed that there has been an unexpected delay in the shipping process. We are actively working with our shipping partners to resolve this issue and ensure your order is dispatched as soon as possible.\n", + "\n", + "We anticipate that your order will be shipped within the next 2-3 business days, and you will receive a shipping confirmation email with a tracking number to monitor the delivery status.\n", + "\n", + "We appreciate your patience and understanding in this matter. If you have any further questions or concerns, please do not hesitate to contact us.\n", + "\n", + "Thank you for choosing our UltraGlow Skin Serum. We are confident you will be delighted with your purchase.\n", + "\n", + "Warm regards,\n", + "\n", + "[Your Name]\n", + "Customer Service Team\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## [Optional] Publish synthetically generated Evaluation data to Weave" + ], + "metadata": { + "id": "jZlihC0AisvA" + } + }, + { + "cell_type": "code", + "source": [ + "if publish_eval_data:\n", + " from weave import Dataset\n", + " dataset = Dataset(name=eval_dataset_name, rows=[\n", + " {'id': '1', 'email_content': 'Subject: Inquiry about Order Delay\\n\\nHello,\\n\\nI placed an order last week for the new UltraGlow Skin Serum, but I have not received a shipping update yet. My order number is 12345. Could you please update me on the status of my shipment?\\n\\nThank you,\\nJane Doe'},\n", + " {'id': '2', 'email_content': 'Subject: Damaged Item Received\\n\\nHello,\\n\\nI received my order yesterday, but one of the items, a glass vase, was broken. My order number is 67890. How can I get a replacement or a refund?\\n\\nBest regards,\\nJohn Smith'},\n", + " {'id': '3', 'email_content': 'Subject: Wrong Item Delivered\\n\\nHi,\\n\\nI ordered a pair of blue sneakers, but I received a black pair instead. My order number is 54321. Could you please assist me with this issue?\\n\\nThank you,\\nEmily Johnson'},\n", + " {'id': '4', 'email_content': 'Subject: Request for Return Instructions\\n\\nDear Customer Service,\\n\\nI would like to return a dress I purchased last week as it does not fit well. My order number is 11223. Could you please provide the return instructions?\\n\\nSincerely,\\nLaura Davis'},\n", + " {'id': '5', 'email_content': 'Subject: Missing Items in Order\\n\\nHello,\\n\\nI just received my order, but two items are missing. My order number is 33445. Could you please help me resolve this?\\n\\nKind regards,\\nMichael Brown'},\n", + " {'id': '6', 'email_content': 'Subject: Delay in Order Confirmation\\n\\nDear Support Team,\\n\\nI placed an order two days ago but have not received a confirmation email yet. My order number is 99887. Can you confirm if my order was processed?\\n\\nThank you,\\nSarah Wilson'},\n", + " {'id': '7', 'email_content': 'Subject: Inquiry About Product Availability\\n\\nHi,\\n\\nI\\'m interested in purchasing the Professional Chef Knife Set, but it appears to be out of stock. Can you let me know when it will be available again?\\n\\nBest regards,\\nDavid Martinez'},\n", + " {'id': '8', 'email_content': 'Subject: Request for Invoice\\n\\nDear Customer Service,\\n\\nCould you please send me an invoice for my recent purchase? My order number is 55667. I need it for my records.\\n\\nThank you,\\nJessica Taylor'},\n", + " {'id': '9', 'email_content': 'Subject: Issue with Discount Code\\n\\nHello,\\n\\nI tried using the discount code SAVE20 during checkout, but it did not apply. My order number is 77654. Could you please assist me?\\n\\nSincerely,\\nRobert Anderson'},\n", + " {'id': '10', 'email_content': 'Subject: Request for Expedited Shipping\\n\\nHi,\\n\\nI need my order delivered urgently. Is it possible to upgrade to expedited shipping? My order number is 44556.\\n\\nThank you,\\nLinda Thompson'},\n", + " {'id': '11', 'email_content': 'Subject: Order Cancellation Request\\n\\nDear Support Team,\\n\\nI would like to cancel my recent order as I made a mistake while ordering. My order number is 33221. Can you please process the cancellation?\\n\\nBest regards,\\nWilliam Clark'}\n", + " ])\n", + " # Publish the dataset\n", + " weave.publish(dataset)" + ], + "metadata": { + "id": "JdVIiLnbi5Pb" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Run Evaluation whilst logging results to Weave" + ], + "metadata": { + "id": "2i7jVn9fjGTt" + } + }, + { + "cell_type": "code", + "source": [ + "dataset_uri = f\"weave:///{wandb_entity}/{weave_project}/object/{eval_dataset_name}:latest\"\n", + "dataset = weave.ref(dataset_uri).get()" + ], + "metadata": { + "id": "VZO7mbymlECL" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Scoring function checking length of summary\n", + "@weave.op()\n", + "def check_conciseness(model_output: str) -> dict:\n", + " result = len(model_output.split()) < 300\n", + " return {'conciseness': result}" + ], + "metadata": { + "id": "hVSUMnMXjIet" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "evaluation = weave.Evaluation(\n", + " dataset=dataset, scorers=[check_conciseness],\n", + ")" + ], + "metadata": { + "id": "TpqRzO0nmAqZ" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "await evaluation.evaluate(model)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 342 + }, + "id": "fvzDPo_ymDjj", + "outputId": "116f2bfd-1306-4966-eef1-59c81478969d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 1 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 2 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 3 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 4 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 5 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m6\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 6 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m7\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 7 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m8\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 8 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m9\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 9 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m10\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 10 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluated \u001b[1;36m11\u001b[0m of \u001b[1;36m11\u001b[0m examples\n" + ], + "text/html": [ + "
Evaluated 11 of 11 examples\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Evaluation summary\n", + "\u001b[1m{\u001b[0m\n", + " \u001b[32m'check_conciseness'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'conciseness'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'true_count'\u001b[0m: \u001b[1;36m11\u001b[0m, \u001b[32m'true_fraction'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n", + " \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m59.57047505812211\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[1m}\u001b[0m\n" + ], + "text/html": [ + "
Evaluation summary\n",
+              "{\n",
+              "    'check_conciseness': {'conciseness': {'true_count': 11, 'true_fraction': 1.0}},\n",
+              "    'model_latency': {'mean': 59.57047505812211}\n",
+              "}\n",
+              "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "🍩 https://wandb.ai/a-sh0ts/azure-weave-cookbook/r/call/26f22ec3-a719-4fc2-82f6-49210a53fc42\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'check_conciseness': {'conciseness': {'true_count': 11,\n", + " 'true_fraction': 1.0}},\n", + " 'model_latency': {'mean': 59.57047505812211}}" + ] + }, + "metadata": {}, + "execution_count": 25 + } + ] + } + ] +} \ No newline at end of file