Google_Sentence_Encoder_evaluation_script.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "sT8P1i7tNWDO",
        "u0liaIe6at2j",
        "3ua2QQAobkRl",
        "sYYWjBD2fLdK",
        "GeQDq-U3WP2e",
        "jWJeOug3g4zT",
        "74H2ugx4h0e_",
        "5qMaMwzQANBY",
        "hVg_Us8nKuv1"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install openai\n"
      ],
      "metadata": {
        "id": "smrMl4FaOiVI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d3e16c8-56c2-409a-c2bf-a01b1fd2a470"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.27.4-py3-none-any.whl (70 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.3/70.3 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.9/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from openai) (4.65.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (23.1.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (269 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m269.4/269.4 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Installing collected packages: multidict, frozenlist, async-timeout, yarl, aiosignal, aiohttp, openai\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 frozenlist-1.3.3 multidict-6.0.4 openai-0.27.4 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import openai\n",
        "import pandas as pd\n"
      ],
      "metadata": {
        "id": "5wL9hYqVa7yu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Creating arrays for context-topic-topic data"
      ],
      "metadata": {
        "id": "sT8P1i7tNWDO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import time\n",
        "\n",
        "data_ctt = open(\"gpt3_dev_context_topic_topic_data (1).jsonl\", 'r')\n",
        "lines_ctt = data_ctt.readlines()\n",
        "dict_ctt = []\n",
        "prompt_ctt = []\n",
        "completion_ctt = []\n",
        "for line in lines_ctt:\n",
        "  temp = json.loads(line)\n",
        "  prompt_ctt.append(temp[\"prompt\"])\n",
        "  completion_ctt.append(temp[\"completion\"].partition(\" END\")[0])"
      ],
      "metadata": {
        "id": "Tdgu-p8yKyH_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preds_ctt = []\n",
        "\n",
        "model_context_topic = \"babbage:ft-personal-2023-04-26-06-19-00\"\n",
        "prompt_ctt_json = json.dumps(prompt_ctt)\n",
        "\n",
        "openai.api_key = \"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\"\n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "for i in range(0, len(prompt_ctt)):\n",
        "\n",
        "  response = openai.Completion.create(\n",
        "    model= model_context_topic,\n",
        "    prompt=prompt_ctt[i],\n",
        "    temperature=1,\n",
        "    max_tokens=50,\n",
        "    top_p=1,\n",
        "    frequency_penalty=0,\n",
        "    presence_penalty=0, \n",
        "    stop = [\" END\"]\n",
        "  )\n",
        "  response_txt = response['choices'][0]['text']\n",
        "  preds_ctt.append(response_txt)\n",
        "  if(i%60==0 and i!=0):\n",
        "    print(i)\n",
        "    print(time.time())\n",
        "    print(start_time)\n",
        "    sleep_time = 60 - (time.time()-start_time)%60\n",
        "    print(sleep_time)\n",
        "    time.sleep(sleep_time)\n",
        "    start_time = time.time()\n",
        "    print(start_time)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Apw02b4hNZy1",
        "outputId": "221852c2-082a-4c46-fd46-2823a73add9d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "60\n",
            "1682490637.5840008\n",
            "1682490617.268429\n",
            "39.68436288833618\n",
            "1682490677.3082476\n",
            "120\n",
            "1682490691.8868742\n",
            "1682490677.3082476\n",
            "45.421321630477905\n",
            "1682490737.3404002\n",
            "180\n",
            "1682490752.1486468\n",
            "1682490737.3404002\n",
            "45.19057846069336\n",
            "1682490797.3477676\n",
            "240\n",
            "1682490812.2496605\n",
            "1682490797.3477676\n",
            "45.09758448600769\n",
            "1682490857.3931718\n",
            "300\n",
            "1682490870.8060386\n",
            "1682490857.3931718\n",
            "46.58668613433838\n",
            "1682490917.4268193\n",
            "360\n",
            "1682490931.6944458\n",
            "1682490917.4268193\n",
            "45.731250286102295\n",
            "1682490977.443206\n",
            "420\n",
            "1682490993.042288\n",
            "1682490977.443206\n",
            "44.40015983581543\n",
            "1682491037.4880552\n",
            "480\n",
            "1682491051.2636287\n",
            "1682491037.4880552\n",
            "46.22366809844971\n",
            "1682491097.5148304\n",
            "540\n",
            "1682491113.9660838\n",
            "1682491097.5148304\n",
            "43.5487105846405\n",
            "1682491157.5577295\n",
            "600\n",
            "1682491172.1470068\n",
            "1682491157.5577295\n",
            "45.409899950027466\n",
            "1682491217.5788248\n",
            "660\n",
            "1682491233.1171584\n",
            "1682491217.5788248\n",
            "44.4605176448822\n",
            "1682491277.6189697\n",
            "720\n",
            "1682491292.3843484\n",
            "1682491277.6189697\n",
            "45.2340829372406\n",
            "1682491337.6597376\n",
            "780\n",
            "1682491352.063516\n",
            "1682491337.6597376\n",
            "45.59499526023865\n",
            "1682491397.700982\n",
            "840\n",
            "1682491414.073303\n",
            "1682491397.700982\n",
            "43.62762522697449\n",
            "1682491457.7148135\n",
            "900\n",
            "1682491473.089517\n",
            "1682491457.7148135\n",
            "44.62453103065491\n",
            "1682491517.7388089\n",
            "960\n",
            "1682491531.6736338\n",
            "1682491517.7388089\n",
            "46.06439709663391\n",
            "1682491577.7853215\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"pred_ctt_final.json\", \"w\") as outfile:\n",
        "  json.dump(preds_ctt, outfile)"
      ],
      "metadata": {
        "id": "baBzl7_RxSWC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Evaluating with TF"
      ],
      "metadata": {
        "id": "9AGAKLNjVxbv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lVjNK8shFKOC"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip3 install seaborn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zwty8Z6mAkdV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "da14aae2-635c-4d47-e1d5-1443eec688cf",
        "cellView": "form"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "module https://tfhub.dev/google/universal-sentence-encoder/4 loaded\n"
          ]
        }
      ],
      "source": [
        "#@title Load the Universal Sentence Encoder's TF Hub module\n",
        "from absl import logging\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow_hub as hub\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import seaborn as sns\n",
        "\n",
        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
        "model = hub.load(module_url)\n",
        "print (\"module %s loaded\" % module_url)\n",
        "def embed(input):\n",
        "  return model(input)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q8F4LNGFqOiq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5dfca4ee-47b3-4b98-960f-8b933c9795fc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.2859293073415756\n",
            "0.45486633479595184\n",
            "0.31623045\n"
          ]
        }
      ],
      "source": [
        "#@title Compute a representation for each message, showing various lengths supported. preds_ctt\n",
        "corr_pred = []\n",
        "for i in range(0, len(preds_ctt)):\n",
        "  messages = [preds_ctt[i], completion_ctt[i]]\n",
        "  # print(messages)\n",
        "  message_embeddings = embed(messages)\n",
        "  corr_pred.append(np.inner(message_embeddings, message_embeddings)[0][1])\n",
        "\n",
        "# Reduce logging output.\n",
        "logging.set_verbosity(logging.ERROR)\n",
        "corr_pred.sort()\n",
        "print(np.percentile(corr_pred, 50))\n",
        "print(np.percentile(corr_pred, 75))\n",
        "print(np.array(corr_pred).mean())"
      ]
    }
  ]
}