Day_1_(Additional)_Prompt_Engineering_Automation_(Prompt_Bench).ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "nETMGv52qGnB"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting promptbench\n",
            "  Using cached promptbench-0.0.3-py3-none-any.whl.metadata (15 kB)\n",
            "Collecting openai==1.3.7\n",
            "  Using cached openai-1.3.7-py3-none-any.whl.metadata (17 kB)\n",
            "Collecting anyio<4,>=3.5.0 (from openai==1.3.7)\n",
            "  Using cached anyio-3.7.1-py3-none-any.whl.metadata (4.7 kB)\n",
            "Collecting distro<2,>=1.7.0 (from openai==1.3.7)\n",
            "  Using cached distro-1.9.0-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting httpx<1,>=0.23.0 (from openai==1.3.7)\n",
            "  Using cached httpx-0.27.0-py3-none-any.whl.metadata (7.2 kB)\n",
            "Collecting pydantic<3,>=1.9.0 (from openai==1.3.7)\n",
            "  Using cached pydantic-2.8.2-py3-none-any.whl.metadata (125 kB)\n",
            "Collecting sniffio (from openai==1.3.7)\n",
            "  Using cached sniffio-1.3.1-py3-none-any.whl.metadata (3.9 kB)\n",
            "Collecting tqdm>4 (from openai==1.3.7)\n",
            "  Using cached tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.5 in ./.venv/lib/python3.9/site-packages (from openai==1.3.7) (4.12.2)\n",
            "Collecting autocorrect==2.6.1 (from promptbench)\n",
            "  Using cached autocorrect-2.6.1.tar.gz (622 kB)\n",
            "  Installing build dependencies ... \u001b[?25ldone\n",
            "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
            "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
            "\u001b[?25hCollecting accelerate==0.25.0 (from promptbench)\n",
            "  Using cached accelerate-0.25.0-py3-none-any.whl.metadata (18 kB)\n",
            "Collecting datasets>=2.15.0 (from promptbench)\n",
            "  Using cached datasets-2.20.0-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting nltk==3.8.1 (from promptbench)\n",
            "  Using cached nltk-3.8.1-py3-none-any.whl.metadata (2.8 kB)\n",
            "Collecting sentencepiece==0.1.99 (from promptbench)\n",
            "  Downloading sentencepiece-0.1.99-cp39-cp39-macosx_11_0_arm64.whl.metadata (7.7 kB)\n",
            "Collecting tokenizers==0.15.0 (from promptbench)\n",
            "  Downloading tokenizers-0.15.0-cp39-cp39-macosx_11_0_arm64.whl.metadata (6.7 kB)\n",
            "Collecting torch==2.1.1 (from promptbench)\n",
            "  Downloading torch-2.1.1-cp39-none-macosx_11_0_arm64.whl.metadata (25 kB)\n",
            "Collecting tqdm>4 (from openai==1.3.7)\n",
            "  Using cached tqdm-4.66.1-py3-none-any.whl.metadata (57 kB)\n",
            "Collecting transformers==4.38.0 (from promptbench)\n",
            "  Downloading transformers-4.38.0-py3-none-any.whl.metadata (131 kB)\n",
            "Collecting Pillow==10.3.0 (from promptbench)\n",
            "  Downloading pillow-10.3.0-cp39-cp39-macosx_11_0_arm64.whl.metadata (9.2 kB)\n",
            "Collecting google-generativeai==0.4.0 (from promptbench)\n",
            "  Downloading google_generativeai-0.4.0-py3-none-any.whl.metadata (6.2 kB)\n",
            "Collecting dashscope==1.14.1 (from promptbench)\n",
            "  Downloading dashscope-1.14.1-py3-none-any.whl.metadata (6.7 kB)\n",
            "Collecting einops==0.7.0 (from promptbench)\n",
            "  Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting transformers-stream-generator==0.0.5 (from promptbench)\n",
            "  Downloading transformers-stream-generator-0.0.5.tar.gz (13 kB)\n",
            "  Installing build dependencies ... \u001b[?25ldone\n",
            "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
            "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
            "\u001b[?25hCollecting torchvision==0.16.1 (from promptbench)\n",
            "  Downloading torchvision-0.16.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (6.6 kB)\n",
            "Collecting matplotlib==3.8.3 (from promptbench)\n",
            "  Downloading matplotlib-3.8.3-cp39-cp39-macosx_11_0_arm64.whl.metadata (5.8 kB)\n",
            "Collecting tiktoken==0.6.0 (from promptbench)\n",
            "  Downloading tiktoken-0.6.0-cp39-cp39-macosx_11_0_arm64.whl.metadata (6.6 kB)\n",
            "Collecting numpy>=1.17 (from accelerate==0.25.0->promptbench)\n",
            "  Downloading numpy-2.0.1-cp39-cp39-macosx_14_0_arm64.whl.metadata (60 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in ./.venv/lib/python3.9/site-packages (from accelerate==0.25.0->promptbench) (24.1)\n",
            "Requirement already satisfied: psutil in ./.venv/lib/python3.9/site-packages (from accelerate==0.25.0->promptbench) (6.0.0)\n",
            "Collecting pyyaml (from accelerate==0.25.0->promptbench)\n",
            "  Downloading PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (2.1 kB)\n",
            "Collecting huggingface-hub (from accelerate==0.25.0->promptbench)\n",
            "  Downloading huggingface_hub-0.24.2-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting safetensors>=0.3.1 (from accelerate==0.25.0->promptbench)\n",
            "  Downloading safetensors-0.4.3-cp39-cp39-macosx_11_0_arm64.whl.metadata (3.8 kB)\n",
            "Collecting aiohttp (from dashscope==1.14.1->promptbench)\n",
            "  Downloading aiohttp-3.9.5-cp39-cp39-macosx_11_0_arm64.whl.metadata (7.5 kB)\n",
            "Collecting requests (from dashscope==1.14.1->promptbench)\n",
            "  Downloading requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
            "Collecting google-ai-generativelanguage==0.4.0 (from google-generativeai==0.4.0->promptbench)\n",
            "  Downloading google_ai_generativelanguage-0.4.0-py3-none-any.whl.metadata (5.1 kB)\n",
            "Collecting google-auth>=2.15.0 (from google-generativeai==0.4.0->promptbench)\n",
            "  Downloading google_auth-2.32.0-py2.py3-none-any.whl.metadata (4.7 kB)\n",
            "Collecting google-api-core (from google-generativeai==0.4.0->promptbench)\n",
            "  Downloading google_api_core-2.19.1-py3-none-any.whl.metadata (2.7 kB)\n",
            "Collecting protobuf (from google-generativeai==0.4.0->promptbench)\n",
            "  Downloading protobuf-5.27.2-cp38-abi3-macosx_10_9_universal2.whl.metadata (592 bytes)\n",
            "Collecting contourpy>=1.0.1 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading contourpy-1.2.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (5.8 kB)\n",
            "Collecting cycler>=0.10 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n",
            "Collecting fonttools>=4.22.0 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading fonttools-4.53.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (162 kB)\n",
            "Collecting kiwisolver>=1.3.1 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading kiwisolver-1.4.5-cp39-cp39-macosx_11_0_arm64.whl.metadata (6.4 kB)\n",
            "Collecting numpy>=1.17 (from accelerate==0.25.0->promptbench)\n",
            "  Downloading numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl.metadata (61 kB)\n",
            "Collecting pyparsing>=2.3.1 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading pyparsing-3.1.2-py3-none-any.whl.metadata (5.1 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in ./.venv/lib/python3.9/site-packages (from matplotlib==3.8.3->promptbench) (2.9.0.post0)\n",
            "Collecting importlib-resources>=3.2.0 (from matplotlib==3.8.3->promptbench)\n",
            "  Downloading importlib_resources-6.4.0-py3-none-any.whl.metadata (3.9 kB)\n",
            "Collecting click (from nltk==3.8.1->promptbench)\n",
            "  Downloading click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
            "Collecting joblib (from nltk==3.8.1->promptbench)\n",
            "  Downloading joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n",
            "Collecting regex>=2021.8.3 (from nltk==3.8.1->promptbench)\n",
            "  Downloading regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl.metadata (40 kB)\n",
            "Collecting filelock (from torch==2.1.1->promptbench)\n",
            "  Downloading filelock-3.15.4-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting sympy (from torch==2.1.1->promptbench)\n",
            "  Downloading sympy-1.13.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting networkx (from torch==2.1.1->promptbench)\n",
            "  Downloading networkx-3.2.1-py3-none-any.whl.metadata (5.2 kB)\n",
            "Collecting jinja2 (from torch==2.1.1->promptbench)\n",
            "  Downloading jinja2-3.1.4-py3-none-any.whl.metadata (2.6 kB)\n",
            "Collecting fsspec (from torch==2.1.1->promptbench)\n",
            "  Downloading fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting proto-plus<2.0.0dev,>=1.22.3 (from google-ai-generativelanguage==0.4.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading proto_plus-1.24.0-py3-none-any.whl.metadata (2.2 kB)\n",
            "Collecting protobuf (from google-generativeai==0.4.0->promptbench)\n",
            "  Downloading protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl.metadata (541 bytes)\n",
            "Collecting idna>=2.8 (from anyio<4,>=3.5.0->openai==1.3.7)\n",
            "  Downloading idna-3.7-py3-none-any.whl.metadata (9.9 kB)\n",
            "Requirement already satisfied: exceptiongroup in ./.venv/lib/python3.9/site-packages (from anyio<4,>=3.5.0->openai==1.3.7) (1.2.2)\n",
            "Collecting pyarrow>=15.0.0 (from datasets>=2.15.0->promptbench)\n",
            "  Downloading pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl.metadata (3.3 kB)\n",
            "Collecting pyarrow-hotfix (from datasets>=2.15.0->promptbench)\n",
            "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl.metadata (3.6 kB)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets>=2.15.0->promptbench)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting pandas (from datasets>=2.15.0->promptbench)\n",
            "  Downloading pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl.metadata (19 kB)\n",
            "INFO: pip is looking at multiple versions of datasets to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting datasets>=2.15.0 (from promptbench)\n",
            "  Downloading datasets-2.19.2-py3-none-any.whl.metadata (19 kB)\n",
            "Collecting xxhash (from datasets>=2.15.0->promptbench)\n",
            "  Downloading xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets>=2.15.0->promptbench)\n",
            "  Downloading multiprocess-0.70.16-py39-none-any.whl.metadata (7.2 kB)\n",
            "Collecting fsspec (from torch==2.1.1->promptbench)\n",
            "  Downloading fsspec-2024.3.1-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting certifi (from httpx<1,>=0.23.0->openai==1.3.7)\n",
            "  Downloading certifi-2024.7.4-py3-none-any.whl.metadata (2.2 kB)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai==1.3.7)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai==1.3.7)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Collecting annotated-types>=0.4.0 (from pydantic<3,>=1.9.0->openai==1.3.7)\n",
            "  Downloading annotated_types-0.7.0-py3-none-any.whl.metadata (15 kB)\n",
            "Collecting pydantic-core==2.20.1 (from pydantic<3,>=1.9.0->openai==1.3.7)\n",
            "  Downloading pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (6.6 kB)\n",
            "Collecting aiosignal>=1.1.2 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl.metadata (4.0 kB)\n",
            "Collecting attrs>=17.3.0 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading attrs-23.2.0-py3-none-any.whl.metadata (9.5 kB)\n",
            "Collecting frozenlist>=1.1.1 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl.metadata (12 kB)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl.metadata (4.2 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl.metadata (31 kB)\n",
            "Collecting async-timeout<5.0,>=4.0 (from aiohttp->dashscope==1.14.1->promptbench)\n",
            "  Downloading async_timeout-4.0.3-py3-none-any.whl.metadata (4.2 kB)\n",
            "Collecting googleapis-common-protos<2.0.dev0,>=1.56.2 (from google-api-core->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading googleapis_common_protos-1.63.2-py2.py3-none-any.whl.metadata (1.5 kB)\n",
            "Collecting cachetools<6.0,>=2.0.0 (from google-auth>=2.15.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading cachetools-5.4.0-py3-none-any.whl.metadata (5.3 kB)\n",
            "Collecting pyasn1-modules>=0.2.1 (from google-auth>=2.15.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading pyasn1_modules-0.4.0-py3-none-any.whl.metadata (3.4 kB)\n",
            "Collecting rsa<5,>=3.1.4 (from google-auth>=2.15.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading rsa-4.9-py3-none-any.whl.metadata (4.2 kB)\n",
            "Requirement already satisfied: zipp>=3.1.0 in ./.venv/lib/python3.9/site-packages (from importlib-resources>=3.2.0->matplotlib==3.8.3->promptbench) (3.19.2)\n",
            "Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib==3.8.3->promptbench) (1.16.0)\n",
            "Collecting charset-normalizer<4,>=2 (from requests->dashscope==1.14.1->promptbench)\n",
            "  Downloading charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl.metadata (33 kB)\n",
            "Collecting urllib3<3,>=1.21.1 (from requests->dashscope==1.14.1->promptbench)\n",
            "  Downloading urllib3-2.2.2-py3-none-any.whl.metadata (6.4 kB)\n",
            "Collecting MarkupSafe>=2.0 (from jinja2->torch==2.1.1->promptbench)\n",
            "  Downloading MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl.metadata (3.0 kB)\n",
            "Collecting pytz>=2020.1 (from pandas->datasets>=2.15.0->promptbench)\n",
            "  Downloading pytz-2024.1-py2.py3-none-any.whl.metadata (22 kB)\n",
            "Collecting tzdata>=2022.7 (from pandas->datasets>=2.15.0->promptbench)\n",
            "  Downloading tzdata-2024.1-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Collecting mpmath<1.4,>=1.1.0 (from sympy->torch==2.1.1->promptbench)\n",
            "  Downloading mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n",
            "Collecting grpcio<2.0dev,>=1.33.2 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.4.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading grpcio-1.65.1-cp39-cp39-macosx_10_9_universal2.whl.metadata (3.3 kB)\n",
            "Collecting grpcio-status<2.0.dev0,>=1.33.2 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.4.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading grpcio_status-1.65.1-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting pyasn1<0.7.0,>=0.4.6 (from pyasn1-modules>=0.2.1->google-auth>=2.15.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading pyasn1-0.6.0-py2.py3-none-any.whl.metadata (8.3 kB)\n",
            "INFO: pip is looking at multiple versions of grpcio-status to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting grpcio-status<2.0.dev0,>=1.33.2 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.4.0->google-generativeai==0.4.0->promptbench)\n",
            "  Downloading grpcio_status-1.64.1-py3-none-any.whl.metadata (1.1 kB)\n",
            "  Downloading grpcio_status-1.64.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "  Downloading grpcio_status-1.63.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "  Downloading grpcio_status-1.62.2-py3-none-any.whl.metadata (1.3 kB)\n",
            "Downloading openai-1.3.7-py3-none-any.whl (221 kB)\n",
            "Downloading promptbench-0.0.3-py3-none-any.whl (129 kB)\n",
            "Downloading accelerate-0.25.0-py3-none-any.whl (265 kB)\n",
            "Downloading dashscope-1.14.1-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m16.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n",
            "Downloading google_generativeai-0.4.0-py3-none-any.whl (137 kB)\n",
            "Downloading matplotlib-3.8.3-cp39-cp39-macosx_11_0_arm64.whl (7.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
            "\u001b[?25hDownloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m40.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pillow-10.3.0-cp39-cp39-macosx_11_0_arm64.whl (3.4 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading sentencepiece-0.1.99-cp39-cp39-macosx_11_0_arm64.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tiktoken-0.6.0-cp39-cp39-macosx_11_0_arm64.whl (923 kB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m923.4/923.4 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tokenizers-0.15.0-cp39-cp39-macosx_11_0_arm64.whl (2.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.5/2.5 MB\u001b[0m \u001b[31m21.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading torch-2.1.1-cp39-none-macosx_11_0_arm64.whl (59.6 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.6/59.6 MB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading torchvision-0.16.1-cp39-cp39-macosx_11_0_arm64.whl (1.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tqdm-4.66.1-py3-none-any.whl (78 kB)\n",
            "Downloading transformers-4.38.0-py3-none-any.whl (8.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.5/8.5 MB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading google_ai_generativelanguage-0.4.0-py3-none-any.whl (598 kB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m598.7/598.7 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading anyio-3.7.1-py3-none-any.whl (80 kB)\n",
            "Downloading datasets-2.19.2-py3-none-any.whl (542 kB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.1/542.1 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading distro-1.9.0-py3-none-any.whl (20 kB)\n",
            "Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "Downloading pydantic-2.8.2-py3-none-any.whl (423 kB)\n",
            "Downloading pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl (1.7 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sniffio-1.3.1-py3-none-any.whl (10 kB)\n",
            "Downloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\n",
            "Downloading contourpy-1.2.1-cp39-cp39-macosx_11_0_arm64.whl (244 kB)\n",
            "Downloading cycler-0.12.1-py3-none-any.whl (8.3 kB)\n",
            "Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "Downloading fonttools-4.53.1-cp39-cp39-macosx_11_0_arm64.whl (2.2 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fsspec-2024.3.1-py3-none-any.whl (171 kB)\n",
            "Downloading aiohttp-3.9.5-cp39-cp39-macosx_11_0_arm64.whl (390 kB)\n",
            "Downloading google_api_core-2.19.1-py3-none-any.whl (139 kB)\n",
            "Downloading google_auth-2.32.0-py2.py3-none-any.whl (195 kB)\n",
            "Downloading huggingface_hub-0.24.2-py3-none-any.whl (417 kB)\n",
            "Downloading idna-3.7-py3-none-any.whl (66 kB)\n",
            "Downloading importlib_resources-6.4.0-py3-none-any.whl (38 kB)\n",
            "Downloading kiwisolver-1.4.5-cp39-cp39-macosx_11_0_arm64.whl (66 kB)\n",
            "Downloading numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl (14.0 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.0/14.0 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m01\u001b[0m\n",
            "\u001b[?25hDownloading protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl (394 kB)\n",
            "Downloading pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl (27.2 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.2/27.2 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading pyparsing-3.1.2-py3-none-any.whl (103 kB)\n",
            "Downloading PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl (174 kB)\n",
            "Downloading regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl (278 kB)\n",
            "Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
            "Downloading certifi-2024.7.4-py3-none-any.whl (162 kB)\n",
            "Downloading safetensors-0.4.3-cp39-cp39-macosx_11_0_arm64.whl (411 kB)\n",
            "Downloading click-8.1.7-py3-none-any.whl (97 kB)\n",
            "Downloading filelock-3.15.4-py3-none-any.whl (16 kB)\n",
            "Downloading jinja2-3.1.4-py3-none-any.whl (133 kB)\n",
            "Downloading joblib-1.4.2-py3-none-any.whl (301 kB)\n",
            "Downloading multiprocess-0.70.16-py39-none-any.whl (133 kB)\n",
            "Downloading networkx-3.2.1-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl (11.3 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mMB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
            "\u001b[?25hDownloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
            "Downloading sympy-1.13.1-py3-none-any.whl (6.2 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading xxhash-3.4.1-cp39-cp39-macosx_11_0_arm64.whl (30 kB)\n",
            "Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Downloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\n",
            "Downloading attrs-23.2.0-py3-none-any.whl (60 kB)\n",
            "Downloading cachetools-5.4.0-py3-none-any.whl (9.5 kB)\n",
            "Downloading charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl (120 kB)\n",
            "Downloading frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl (53 kB)\n",
            "Downloading googleapis_common_protos-1.63.2-py2.py3-none-any.whl (220 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "Downloading MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl (18 kB)\n",
            "Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m31.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl (30 kB)\n",
            "Downloading proto_plus-1.24.0-py3-none-any.whl (50 kB)\n",
            "Downloading pyasn1_modules-0.4.0-py3-none-any.whl (181 kB)\n",
            "Downloading pytz-2024.1-py2.py3-none-any.whl (505 kB)\n",
            "Downloading rsa-4.9-py3-none-any.whl (34 kB)\n",
            "Downloading tzdata-2024.1-py2.py3-none-any.whl (345 kB)\n",
            "Downloading urllib3-2.2.2-py3-none-any.whl (121 kB)\n",
            "Downloading yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl (81 kB)\n",
            "Downloading grpcio-1.65.1-cp39-cp39-macosx_10_9_universal2.whl (10.5 MB)\n",
            "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.5/10.5 MB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m1m18.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
            "\u001b[?25hDownloading grpcio_status-1.62.2-py3-none-any.whl (14 kB)\n",
            "Downloading pyasn1-0.6.0-py2.py3-none-any.whl (85 kB)\n",
            "Building wheels for collected packages: autocorrect, transformers-stream-generator\n",
            "  Building wheel for autocorrect (pyproject.toml) ... \u001b[?25ldone\n",
            "\u001b[?25h  Created wheel for autocorrect: filename=autocorrect-2.6.1-py3-none-any.whl size=622365 sha256=1cdaa1bc9f8aec81b91ee1d22bd56934f3e3cccd680045cf8582da2a19f3435d\n",
            "  Stored in directory: /Users/kakao/Library/Caches/pip/wheels/ab/0f/23/3c010c3fd877b962146e7765f9e9b08026cac8b035094c5750\n",
            "  Building wheel for transformers-stream-generator (pyproject.toml) ... \u001b[?25ldone\n",
            "\u001b[?25h  Created wheel for transformers-stream-generator: filename=transformers_stream_generator-0.0.5-py3-none-any.whl size=12426 sha256=0b9d091f91859e0f370b1dce37fb035de9a1cbddf412cfc946ce77e3c869574a\n",
            "  Stored in directory: /Users/kakao/Library/Caches/pip/wheels/6e/00/32/da530fcb0b0a9aa69effd210297377a2bed10ec1980794c927\n",
            "Successfully built autocorrect transformers-stream-generator\n",
            "Installing collected packages: sentencepiece, pytz, mpmath, xxhash, urllib3, tzdata, tqdm, sympy, sniffio, safetensors, regex, pyyaml, pyparsing, pydantic-core, pyasn1, pyarrow-hotfix, protobuf, Pillow, numpy, networkx, multidict, MarkupSafe, kiwisolver, joblib, importlib-resources, idna, h11, grpcio, fsspec, frozenlist, fonttools, filelock, einops, distro, dill, cycler, click, charset-normalizer, certifi, cachetools, autocorrect, attrs, async-timeout, annotated-types, yarl, rsa, requests, pydantic, pyasn1-modules, pyarrow, proto-plus, pandas, nltk, multiprocess, jinja2, httpcore, googleapis-common-protos, contourpy, anyio, aiosignal, torch, tiktoken, matplotlib, huggingface-hub, httpx, grpcio-status, google-auth, aiohttp, torchvision, tokenizers, openai, google-api-core, dashscope, accelerate, transformers, datasets, transformers-stream-generator, google-ai-generativelanguage, google-generativeai, promptbench\n",
            "Successfully installed MarkupSafe-2.1.5 Pillow-10.3.0 accelerate-0.25.0 aiohttp-3.9.5 aiosignal-1.3.1 annotated-types-0.7.0 anyio-3.7.1 async-timeout-4.0.3 attrs-23.2.0 autocorrect-2.6.1 cachetools-5.4.0 certifi-2024.7.4 charset-normalizer-3.3.2 click-8.1.7 contourpy-1.2.1 cycler-0.12.1 dashscope-1.14.1 datasets-2.19.2 dill-0.3.8 distro-1.9.0 einops-0.7.0 filelock-3.15.4 fonttools-4.53.1 frozenlist-1.4.1 fsspec-2024.3.1 google-ai-generativelanguage-0.4.0 google-api-core-2.19.1 google-auth-2.32.0 google-generativeai-0.4.0 googleapis-common-protos-1.63.2 grpcio-1.65.1 grpcio-status-1.62.2 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 huggingface-hub-0.24.2 idna-3.7 importlib-resources-6.4.0 jinja2-3.1.4 joblib-1.4.2 kiwisolver-1.4.5 matplotlib-3.8.3 mpmath-1.3.0 multidict-6.0.5 multiprocess-0.70.16 networkx-3.2.1 nltk-3.8.1 numpy-1.26.4 openai-1.3.7 pandas-2.2.2 promptbench-0.0.3 proto-plus-1.24.0 protobuf-4.25.4 pyarrow-17.0.0 pyarrow-hotfix-0.6 pyasn1-0.6.0 pyasn1-modules-0.4.0 pydantic-2.8.2 pydantic-core-2.20.1 pyparsing-3.1.2 pytz-2024.1 pyyaml-6.0.1 regex-2024.7.24 requests-2.32.3 rsa-4.9 safetensors-0.4.3 sentencepiece-0.1.99 sniffio-1.3.1 sympy-1.13.1 tiktoken-0.6.0 tokenizers-0.15.0 torch-2.1.1 torchvision-0.16.1 tqdm-4.66.1 transformers-4.38.0 transformers-stream-generator-0.0.5 tzdata-2024.1 urllib3-2.2.2 xxhash-3.4.1 yarl-1.9.4\n"
          ]
        }
      ],
      "source": [
        "!pip install promptbench openai==1.3.7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ic_1os-LqKww",
        "outputId": "38d64978-e8b1-4696-fac8-9484020faeaf"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/Users/kakao/Documents/been.zino/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "All supported datasets: \n",
            "['sst2', 'cola', 'qqp', 'mnli', 'mnli_matched', 'mnli_mismatched', 'qnli', 'wnli', 'rte', 'mrpc', 'mmlu', 'squad_v2', 'un_multi', 'iwslt2017', 'math', 'bool_logic', 'valid_parentheses', 'gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking', 'last_letter_concat', 'numersense', 'qasc', 'bbh', 'drop', 'arc-easy', 'arc-challenge']\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading readme: 100%|██████████| 7.94k/7.94k [00:00<00:00, 9.32MB/s]\n",
            "Downloading data: 100%|██████████| 2.31M/2.31M [00:00<00:00, 4.09MB/s]\n",
            "Downloading data: 100%|██████████| 419k/419k [00:00<00:00, 933kB/s]\n",
            "Generating train split: 100%|██████████| 7473/7473 [00:00<00:00, 508600.54 examples/s]\n",
            "Generating test split: 100%|██████████| 1319/1319 [00:00<00:00, 476510.51 examples/s]\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "[{'content': \"Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\",\n",
              "  'label': '18'},\n",
              " {'content': 'A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?',\n",
              "  'label': '3'},\n",
              " {'content': 'Josh decides to try flipping a house.  He buys a house for $80,000 and then puts in $50,000 in repairs.  This increased the value of the house by 150%.  How much profit did he make?',\n",
              "  'label': '70000'}]"
            ]
          },
          "execution_count": 1,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import promptbench as pb\n",
        "\n",
        "# print all supported datasets in promptbench\n",
        "print('All supported datasets: ')\n",
        "print(pb.SUPPORTED_DATASETS)\n",
        "\n",
        "# load a dataset, sst2, for instance.\n",
        "# if the dataset is not available locally, it will be downloaded automatically.\n",
        "dataset_name = \"gsm8k\"\n",
        "dataset = pb.DatasetLoader.load_dataset(dataset_name)\n",
        "\n",
        "# print the first 3 examples\n",
        "dataset[:3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GboWIKBNwvnH",
        "outputId": "c0dc8535-0384-46e6-d1e7-616a38c1efe8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "All supported models: \n",
            "['google/flan-t5-large', 'llama2-7b', 'llama2-7b-chat', 'llama2-13b', 'llama2-13b-chat', 'llama2-70b', 'llama2-70b-chat', 'phi-1.5', 'phi-2', 'palm', 'gpt-3.5-turbo', 'gpt-4', 'gpt-4-1106-preview', 'gpt-3.5-turbo-1106', 'gpt-4-0125-preview', 'gpt-3.5-turbo-0125', 'vicuna-7b', 'vicuna-13b', 'vicuna-13b-v1.3', 'google/flan-ul2', 'gemini-pro', 'mistralai/Mistral-7B-v0.1', 'mistralai/Mistral-7B-Instruct-v0.1', 'mistralai/Mixtral-8x7B-v0.1', 'mistralai/Mixtral-8x7B-Instruct-v0.1', '01-ai/Yi-6B', '01-ai/Yi-34B', '01-ai/Yi-6B-Chat', '01-ai/Yi-34B-Chat', 'baichuan-inc/Baichuan2-7B-Base', 'baichuan-inc/Baichuan2-13B-Base', 'baichuan-inc/Baichuan2-7B-Chat', 'baichuan-inc/Baichuan2-13B-Chat']\n"
          ]
        }
      ],
      "source": [
        "# print all supported models in promptbench\n",
        "print('All supported models: ')\n",
        "print(pb.SUPPORTED_MODELS)\n",
        "\n",
        "# load a model, gpt-3.5-turbo, for instance.\n",
        "# If model is openai/palm, need to provide openai_key/palm_key\n",
        "# If model is llama, vicuna, need to provide model dir\n",
        "model = pb.LLMModel(model='gpt-3.5-turbo',\n",
        "                    api_key = '',\n",
        "                    max_new_tokens=150)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UFPQRC6DtXQ4",
        "outputId": "67b43094-a8cd-41f0-980a-2c6e6498762a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "All supported methods: \n",
            "['CoT', 'ZSCoT', 'least_to_most', 'generated_knowledge', 'expert_prompting', 'emotion_prompt', 'baseline']\n",
            "Supported datasets for each method: \n",
            "{'CoT': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'ZSCoT': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'expert_prompting': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'emotion_prompt': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'least_to_most': ['gsm8k', 'last_letter_concat'], 'generated_knowledge': ['csqa', 'numersense', 'qasc'], 'baseline': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking', 'last_letter_concat', 'numersense', 'qasc']}\n"
          ]
        }
      ],
      "source": [
        "# load method\n",
        "# print all methods and their supported datasets\n",
        "print('All supported methods: ')\n",
        "print(pb.SUPPORTED_METHODS)\n",
        "print('Supported datasets for each method: ')\n",
        "print(pb.METHOD_SUPPORT_DATASET)\n",
        "\n",
        "# load a method, emotion_prompt, for instance.\n",
        "# https://github.com/microsoft/promptbench/tree/main/promptbench/prompt_engineering\n",
        "method = pb.PEMethod(method='emotion_prompt',\n",
        "                        dataset=dataset_name,\n",
        "                        verbose=True,  # if True, print the detailed prompt and response\n",
        "                        prompt_id = 1  # for emotion_prompt\n",
        "                        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gAmaHUZCvqBT",
        "outputId": "4bcea33d-18c8-42d2-e81c-a06f5f9d0d75"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 1/1319 [00:02<1:01:22,  2.79s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\n",
            "This is very important to my career.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "To find out how much Janet makes at the farmers' market every day, we first need to calculate how many eggs she has left after eating three for breakfast and using four for muffins.\n",
            "\n",
            "Janet has 16 eggs per day.\n",
            "She eats 3 for breakfast and uses 4 for muffins, so she uses 3 + 4 = 7 eggs.\n",
            "This leaves her with 16 - 7 = 9 eggs to sell.\n",
            "\n",
            "She sells each egg for $2, so she makes 9 * $2 = $18 every day at the farmers' market.\n",
            "\n",
            "Therefore, Janet makes $18 every day at the farmers' market.##18\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 2/1319 [00:04<43:14,  1.97s/it]  "
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?\n",
            "This is very important to my career.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "To make a robe, you need 2 bolts of blue fiber and half that amount of white fiber, which is 1 bolt. Therefore, in total, it takes 2 bolts of blue fiber and 1 bolt of white fiber.\n",
            "\n",
            "Adding them together, the total number of bolts needed for the robe is 2 + 1 = 3 bolts.\n",
            "\n",
            "##3\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 3/1319 [00:06<46:51,  2.14s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Josh decides to try flipping a house.  He buys a house for $80,000 and then puts in $50,000 in repairs.  This increased the value of the house by 150%.  How much profit did he make?\n",
            "This is very important to my career.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "To find out how much profit Josh made, we first need to calculate the new value of the house after the repairs.\n",
            "\n",
            "The increase in value due to the repairs is 150% of the original value of the house, which is $80,000. So, the increase in value is 150/100 * $80,000 = $120,000.\n",
            "\n",
            "The new value of the house after the repairs is the original value plus the increase in value: $80,000 + $120,000 = $200,000.\n",
            "\n",
            "Josh's total investment in the house is the purchase price plus the cost of repairs: $80,000 + $50,000 = $130,000.\n",
            "\n",
            "Therefore, Josh's profit is the new value of the\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.6666666666666666"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results = method.test(dataset,\n",
        "                      model,\n",
        "                      num_samples=3 # if don't set the num_samples, method will use all examples in the dataset\n",
        "                      )\n",
        "results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hZaUgoBKvrvL",
        "outputId": "8b245892-3a09-4cba-e0e4-5a2c6d993b09"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "All supported methods: \n",
            "['CoT', 'ZSCoT', 'least_to_most', 'generated_knowledge', 'expert_prompting', 'emotion_prompt', 'baseline']\n",
            "Supported datasets for each method: \n",
            "{'CoT': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'ZSCoT': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'expert_prompting': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'emotion_prompt': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking'], 'least_to_most': ['gsm8k', 'last_letter_concat'], 'generated_knowledge': ['csqa', 'numersense', 'qasc'], 'baseline': ['gsm8k', 'csqa', 'bigbench_date', 'bigbench_object_tracking', 'last_letter_concat', 'numersense', 'qasc']}\n"
          ]
        }
      ],
      "source": [
        "# load method\n",
        "# print all methods and their supported datasets\n",
        "print('All supported methods: ')\n",
        "print(pb.SUPPORTED_METHODS)\n",
        "print('Supported datasets for each method: ')\n",
        "print(pb.METHOD_SUPPORT_DATASET)\n",
        "\n",
        "# load a method, emotion_prompt, for instance.\n",
        "# https://github.com/microsoft/promptbench/tree/main/promptbench/prompt_engineering\n",
        "method = pb.PEMethod(method='CoT',\n",
        "                        dataset=dataset_name,\n",
        "                        verbose=True,  # if True, print the detailed prompt and response\n",
        "                        prompt_id = 1  # for emotion_prompt\n",
        "                        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5-SQ6mqDyaS4",
        "outputId": "b9ef4414-d56c-4cf3-e16c-05d41c8f1d34"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 1/1319 [00:02<44:50,  2.04s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there\n",
            "will be 21 trees. How many trees did the grove workers plant today?\n",
            "A: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have\n",
            "been 21 - 15 = 6. The answer is 6.\n",
            "Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\n",
            "A: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n",
            "Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\n",
            "A: Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they\n",
            "had 74 - 35 = 39. The answer is 39.\n",
            "Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did\n",
            "Jason give to Denny?\n",
            "A: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8.\n",
            "The answer is 8.\n",
            "Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he\n",
            "have now?\n",
            "A: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9.\n",
            "The answer is 9.\n",
            "Q: There were nine computers in the server room. Five more computers were installed each day, from monday\n",
            "to thursday. How many computers are now in the server room?\n",
            "A: There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20\n",
            "computers were added. 9 + 20 is 29. The answer is 29.\n",
            "Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf\n",
            "balls did he have at the end of wednesday?\n",
            "A: Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he\n",
            "had 35 - 2 = 33 golf balls. The answer is 33.\n",
            "Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n",
            "A: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23\n",
            "- 15 is 8. The answer is 8.\n",
            "Q: Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\n",
            "A:\n",
            "\n",
            "Let's think step by step.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "1. Janet's ducks lay 16 eggs per day.\n",
            "2. She eats 3 eggs for breakfast every morning, so she has 16 - 3 = 13 eggs remaining.\n",
            "3. She bakes muffins for her friends every day with 4 eggs, so she has 13 - 4 = 9 eggs remaining.\n",
            "4. Janet sells the remainder at the farmers' market daily for $2 per fresh duck egg, so she makes 9 x 2 = 18 dollars every day at the farmers' market.\n",
            "\n",
            "Therefore, Janet makes $18 every day at the farmers' market. ##18\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 2/1319 [00:03<38:16,  1.74s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there\n",
            "will be 21 trees. How many trees did the grove workers plant today?\n",
            "A: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have\n",
            "been 21 - 15 = 6. The answer is 6.\n",
            "Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\n",
            "A: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n",
            "Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\n",
            "A: Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they\n",
            "had 74 - 35 = 39. The answer is 39.\n",
            "Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did\n",
            "Jason give to Denny?\n",
            "A: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8.\n",
            "The answer is 8.\n",
            "Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he\n",
            "have now?\n",
            "A: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9.\n",
            "The answer is 9.\n",
            "Q: There were nine computers in the server room. Five more computers were installed each day, from monday\n",
            "to thursday. How many computers are now in the server room?\n",
            "A: There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20\n",
            "computers were added. 9 + 20 is 29. The answer is 29.\n",
            "Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf\n",
            "balls did he have at the end of wednesday?\n",
            "A: Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he\n",
            "had 35 - 2 = 33 golf balls. The answer is 33.\n",
            "Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n",
            "A: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23\n",
            "- 15 is 8. The answer is 8.\n",
            "Q: A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?\n",
            "A:\n",
            "\n",
            "Let's think step by step.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "1. A robe takes 2 bolts of blue fiber.\n",
            "2. It also takes half that much white fiber, which is 2 / 2 = 1 bolt.\n",
            "3. So in total, the robe takes 2 (blue) + 1 (white) = 3 bolts.\n",
            "\n",
            "Therefore, it takes 3 bolts in total. ##3\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 3/1319 [00:06<45:09,  2.06s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there\n",
            "will be 21 trees. How many trees did the grove workers plant today?\n",
            "A: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have\n",
            "been 21 - 15 = 6. The answer is 6.\n",
            "Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\n",
            "A: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n",
            "Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\n",
            "A: Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they\n",
            "had 74 - 35 = 39. The answer is 39.\n",
            "Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did\n",
            "Jason give to Denny?\n",
            "A: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8.\n",
            "The answer is 8.\n",
            "Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he\n",
            "have now?\n",
            "A: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9.\n",
            "The answer is 9.\n",
            "Q: There were nine computers in the server room. Five more computers were installed each day, from monday\n",
            "to thursday. How many computers are now in the server room?\n",
            "A: There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20\n",
            "computers were added. 9 + 20 is 29. The answer is 29.\n",
            "Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf\n",
            "balls did he have at the end of wednesday?\n",
            "A: Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he\n",
            "had 35 - 2 = 33 golf balls. The answer is 33.\n",
            "Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n",
            "A: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23\n",
            "- 15 is 8. The answer is 8.\n",
            "Q: Josh decides to try flipping a house.  He buys a house for $80,000 and then puts in $50,000 in repairs.  This increased the value of the house by 150%.  How much profit did he make?\n",
            "A:\n",
            "\n",
            "Let's think step by step.\n",
            "Please output your answer at the end as ##<your answer (arabic numerals)>\n",
            "\n",
            "Step 1: Calculate the increase in value of the house after repairs.\n",
            "Increase in value = 150% of $80,000\n",
            "Increase in value = 150/100 * $80,000\n",
            "Increase in value = $120,000\n",
            "\n",
            "Step 2: Calculate the total value of the house after repairs.\n",
            "Total value = Initial value + Increase in value\n",
            "Total value = $80,000 + $120,000\n",
            "Total value = $200,000\n",
            "\n",
            "Step 3: Calculate the profit made by Josh.\n",
            "Profit = Total value - Cost of purchase - Cost of repairs\n",
            "Profit = $200,000 - $80,000 - $50,000\n",
            "Profit = $70,000\n",
            "\n",
            "##$70,000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results = method.test(dataset,\n",
        "                      model,\n",
        "                      num_samples=3 # if don't set the num_samples, method will use all examples in the dataset\n",
        "                      )\n",
        "results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QOWlobl4ybyz"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}