abastractive_summarization_on_Reddit.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CS224N_abastractive_summarization.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VkxnFhKA9TgN",
        "outputId": "aa7893fe-2f32-4e8b-ec26-5e657f4fb5a3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/My Drive/code/CS224N\n",
            "/content/drive/My Drive/code/CS224N\n",
            "mydata.csv  mydata.pkl\tSCBAll.csv\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "%cd /content/drive/My\\ Drive/code/CS224N\n",
        "! pwd\n",
        "! ls data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from tqdm import tqdm\n",
        "\n",
        "!pip install transformers\n",
        "!pip install sentencepiece\n",
        "!pip install wandb -q\n",
        "\n",
        "import torch\n",
        "from transformers import AutoModelWithLMHead, AutoTokenizer\n",
        "from transformers import BertForQuestionAnswering\n",
        "from transformers import BertTokenizer\n",
        "from transformers import AdamW"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SmtD3eyP9VhX",
        "outputId": "a74f838e-3d3a-44c5-cb48-4b627e5158c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.17.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.11.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.47)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.96)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "_5QCk1eYCMbt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.memory_reserved(0)/1e6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4N0YYBm1oyeu",
        "outputId": "1cffd1df-3bbb-48db-9ded-0a1957eaf55e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.memory_allocated(0)/1e6"
      ],
      "metadata": {
        "id": "1B2UFwvHCOwZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "41250455-6230-4204-d294-e8c5e995d794"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.memory_reserved(0)/1e6, torch.cuda.memory_allocated(0)/1e6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SyNOgL4jqE7x",
        "outputId": "aa23378a-3d53-44d3-dfa4-9cddd52a1a76"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(983.564288, 891.614208)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.get_device_properties(0).total_memory/1e9"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "toNLadejn4Ns",
        "outputId": "d2b03f9a-5a11-440c-c0cc-18db3f5a83c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17.071734784"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"tuner007/t5_abs_qa\")\n",
        "model_original = AutoModelWithLMHead.from_pretrained(\"tuner007/t5_abs_qa\")\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-bCY9suK9wph",
        "outputId": "13290f75-f2fc-4530-efc6-e6b4127d0664"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/models/auto/modeling_auto.py:882: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
            "  FutureWarning,\n",
            "The `xla_device` argument has been deprecated in v4.4.0 of Transformers. It is ignored and you can safely remove it from your `config.json` file.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = model_original.to(device)\n",
        "def get_answer(question, context): ### check\n",
        "  input_text = \"context: %s <question for context: %s </s>\" % (context,question)\n",
        "  features = tokenizer([input_text], return_tensors='pt')\n",
        "  out = model.generate(input_ids=features['input_ids'].to(device), attention_mask=features['attention_mask'].to(device))\n",
        "  print(out[0])\n",
        "  # for i in out[0]:\n",
        "  #   print(i,tokenizer.decode(i))\n",
        "  return tokenizer.decode(out[0]) ### test if we dont store the output"
      ],
      "metadata": {
        "id": "aC1fN3KE-ts-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# gpu_info = !nvidia-smi\n",
        "# gpu_info = '\\n'.join(gpu_info)\n",
        "# if gpu_info.find('failed') >= 0:\n",
        "#   print('Not connected to a GPU')\n",
        "# else:\n",
        "#   print(gpu_info)"
      ],
      "metadata": {
        "id": "92tdQO2_9zNy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from psutil import virtual_memory\n",
        "# ram_gb = virtual_memory().total / 1e9\n",
        "# print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "# if ram_gb < 20:\n",
        "#   print('Not using a high-RAM runtime')\n",
        "# else:\n",
        "#   print('You are using a high-RAM runtime!')"
      ],
      "metadata": {
        "id": "oljWZfrC91T6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## quick demo"
      ],
      "metadata": {
        "id": "0vPjMnYI_Dx_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "### to suppress output\n",
        "model.eval()"
      ],
      "metadata": {
        "id": "EaL-nEsD93e-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(['<pad> It is a hall of worship ruled by Odin.</s>'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpZcOQEN4UAS",
        "outputId": "84cc3b62-9b68-4a4a-ee91-b6357aa06ac3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [[0, 94, 19, 3, 9, 6358, 13, 7373, 3, 16718, 57, 9899, 77, 5, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"In Norse mythology, Valhalla is a majestic, enormous hall located in Asgard, ruled over by the god Odin.\"\n",
        "question = \"What is Valhalla?\"\n",
        "get_answer(question, context)\n",
        "# output: 'It is a hall of worship ruled by Odin.'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "nC5ywWxM-_9K",
        "outputId": "12175578-435e-4e44-ded0-fca58e9c2c19"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([    0,    94,    19,     3,     9,  6358,    13,  7373,     3, 16718,\n",
            "           57,  9899,    77,     5,     1], device='cuda:0')\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<pad> It is a hall of worship ruled by Odin.</s>'"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## eval without finetuning"
      ],
      "metadata": {
        "id": "0auaHmdT_K4n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('data/mydata.csv')"
      ],
      "metadata": {
        "id": "3OrSa9bk_QFa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['old QA'] = df['answer'].copy()"
      ],
      "metadata": {
        "id": "HLywyqST_XoV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "hhJ9ZPKQAo7J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#  len(df['article'].iloc[193])"
      ],
      "metadata": {
        "id": "qx7aOfVTCdZ9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for i in range(len(df)):\n",
        "for i in range(10):\n",
        "    if i%100==0 or i<10:\n",
        "        print(i)\n",
        "    try:\n",
        "        df['old QA'].iloc[i] = get_answer(df['teaser'].iloc[i], df['article'].iloc[i])\n",
        "    except RuntimeError as e:\n",
        "        print(e)\n",
        "        df['old QA'].iloc[i] = 'Too long, GPU memory exceeded'\n",
        "    if i%10 ==0:\n",
        "        df.to_csv('mydata2.csv')\n",
        "    # torch.cuda.empty_cache()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ka5rQMUM_ZaM",
        "outputId": "12ef1bbe-7928-435e-fb91-c6434665150b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n",
            "tensor([   0,  465, 1525,  347,   16, 2625,    1], device='cuda:0')\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self._setitem_single_block(indexer, value, name)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1\n",
            "tensor([   0,  328,  435,    3,    9,  126, 1876, 1096,   70, 1683,  915,    5,\n",
            "           1], device='cuda:0')\n",
            "2\n",
            "tensor([   0,  451,  133,  281,   12,  217,  160, 7370,   11,  129,  160, 1268,\n",
            "        1340,    5,    1], device='cuda:0')\n",
            "3\n",
            "tensor([    0,    37, 11511,   789,    65, 18094,    15,    26,   529,    18,\n",
            "         8185, 10646,  1623,     5,     1], device='cuda:0')\n",
            "4\n",
            "tensor([   0,  328,   33,   59,   30, 1641,  544,    5,    1], device='cuda:0')\n",
            "5\n",
            "tensor([   0,  465, 1525,  347,   16, 2625,    1], device='cuda:0')\n",
            "6\n",
            "tensor([   0,  465, 1525,  347,   16, 2625,    1], device='cuda:0')\n",
            "7\n",
            "tensor([   0,   37, 4836,    1], device='cuda:0')\n",
            "8\n",
            "tensor([   0,  465, 1525,  347,   16, 2625,    1], device='cuda:0')\n",
            "9\n",
            "tensor([   0,   27,  183,    3,    9,  306,  496, 9999, 3763,    5,    1],\n",
            "       device='cuda:0')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# df.to_csv('mydata1.csv')"
      ],
      "metadata": {
        "id": "l380jO9S_674"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# print(df.iloc[50]['article'])\n",
        "print(df.iloc[50]['teaser'])\n",
        "print(df.iloc[50]['answer'])\n",
        "print(df.iloc[50]['old QA'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HEv7fYaaMcI3",
        "outputId": "9731ecef-290e-47a4-dcda-8539fe0d3507"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Why did 'Pawn Star' Richard \"The Olde Man\" Harrison intentionally disinherit his youngest son Christopher, cutting him from any inheritance a few months before dying?\n",
            "You never find out. After scrolling longer than a CVS receipt you learn it's a private family matter they aren't talking about.\n",
            "You never find out. After scrolling longer than a CVS receipt you learn it's a private family matter they aren't talking about.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## finetune further"
      ],
      "metadata": {
        "id": "JXtYMff0_GRG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "optim = AdamW(model.parameters(), lr=5.0e-5)\n",
        "train_test_split = 2000 ### leaving 300+ for test\n",
        "N_epochs = 8"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KsKJMtCs_BdJ",
        "outputId": "90fc1b5f-cbb3-4512-f22f-5a20eab1e04a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.memory_reserved(0)/1e6, torch.cuda.memory_allocated(0)/1e6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KzbISJWnqdbg",
        "outputId": "6212e99f-50e1-45f2-c11e-3c914f3246f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(989.855744, 891.614208)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "### to suppress output\n",
        "model.train()"
      ],
      "metadata": {
        "id": "xobp5TBnAeZ5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"In Norse mythology, Valhalla is a majestic, enormous hall located in Asgard, ruled over by the god Odin.\"\n",
        "question = \"What is Valhalla?\"\n",
        "input_text = \"context: %s <question for context: %s </s>\" % (context,question)\n",
        "features = tokenizer([input_text], return_tensors='pt')\n",
        "# targets = tokenizer(['<pad> It is a hall of worship ruled by Odin.</s>'], return_tensors='pt')['input_ids'].to(device)\n",
        "targets = tokenizer(['<pad> It is a Dummy Answer.</s>'], return_tensors='pt')['input_ids'].to(device)"
      ],
      "metadata": {
        "id": "hxX9KiQBNTvL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_test_split"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KyiZNsUzsIOr",
        "outputId": "0e2e564a-40f8-49bc-e1f7-7c87ebdf42e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2000"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "below is some debugging effort to figure out why we run out of GPU memory after finetuning. The current best guess is that after the model encounter super long text and crashes due to running out of memory, it fails to clear all allocated memory. So the workaround is to prune the data and avoid trainig on too long texts."
      ],
      "metadata": {
        "id": "ezxoV-qgz1M3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df['article'].iloc[2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "id": "iixpLpb0vps2",
        "outputId": "c1236c65-384e-471d-a391-d1cdac1ab87a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Is there anything more important than a birthday to a kid? All year long, they wait and wait for their special day where everyone celebrates them and showers them with gifts, excited to finally be a year older. Some kids have parents that have separated – but hey, it’s a reason to celebrate twice! Kelsey Frederick is one such teenager with two divorced parents who both still love her very much. Her parents Christin and Schaffen seemed to have no problem coordinating two separate parties to spend with Kelsey. But when her birthday came around, it turned out they had very different plans in mind. Unhappy Birthday To You Kelsey Frederick recently turned 13, and even had two birthday parties to look forward to in order to celebrate with each of her parents. It was a routine the whole family had grown accustomed to over the years. But this year, she was extra excited because her mom had decided to treat her to highlights in her hair for the very first time! Afterward, she went on to see her dad as expected – but what wasn’t expected, was his reaction…\\n\\nStrong Single Mother Having two families is not an uncommon experience for many young people today, like Kelsey. She spends plenty of time with both of her parents, but she and her mother Christin don’t only live together; they’re practically best friends, too. It’s the kind of bond many mothers and daughters only dream of. And while she loves her father and he loves her dearly, he doesn’t always love hearing about her young life the way her mom does.\\n\\nDays With Dad Although Christin and Schaffen didn’t work out in the end, they found themselves able to mostly agree on how to parent Kelsey. Her father had even managed to find another successful relationship with a woman who worked with him at the fire station. But it wasn’t a problem for the two parents, as long as they remained partners in raising the daughter they shared. Kelsey also welcomed it – she was happy that he was happy.\\n\\nSurrounded By Strong Women Kelsey was also used to having many strong women around her in life. Her aunt Kelly was often around and like another mother to her, giving her support and company. She also had her godmother, Haylee Ann, who was full of advice and perspectives on life from all of her experiences. So with many motherly women around her, another woman close to her father wasn’t threatening to Kelsey. And they were all about to play a big role in the events around her birthday.\\n\\nNot Your Average Girl Getting gifts for a young girl is really never easy, but Kelsey proved to be an exceptional challenge. Her favorite hobbies tended to separate her from other young girls her age, and didn’t make for the most practical gifts. She loved to play softball with her team, but already had everything she needed. She loved going fishing with her cousins, but that’s no easy gift to plan in the middle of a party. Plus, she was already pretty happy.\\n\\nIt’s My Birthday! For Kelsey, her birthday seemed to be taking an eternity to arrive, as usual. But for Christin, it was hard to believe her little girl was growing up so fast…where had 13 come from all of a sudden? This year, she knew Kelsey was hitting a new milestone, and it made her want to prepare an extra special gift. Every year before, Christin had prepared a surprise for Kelsey’s birthday. But she wanted to give her newly teenage daughter the privilege of choosing her own.\\n\\nI Want Highlights But she wasn’t expecting Kelsey’s request. It sounds normal enough, but a mother is never ready for the day when her daughter asks for her first major makeover. Kelsey wanted to totally upgrade her brown hair with some bold, blonde highlights. Christin was definitely taken aback, but knew she had to follow through since she’d offered. Christin knew this would be no easy task, and didn’t think she was capable of doing it herself. But she had another plan.\\n\\nDay At The Salon She opted to go all out and give Kelsey the royal treatment for the procedure. Neither of them quite knew what to expect, so they turned a potentially nerve-wracking experiment into a day of birthday wishes and mother-daughter bonding at the local salon, waiting for Kelsey’s transformation to be complete. After a long and relaxing day, Christin put Kelsey in front of a mirror to show her the end result. Both of them were in for a huge surprise.\\n\\nIncredible Results Kelsey looked in the mirror, but couldn’t believe her own eyes: her highlights made her feel like a totally new person. Christin also could barely recognize her own daughter, but with the smile on the birthday girl’s face, she thought she’d never looked so beautiful. She just had to take a picture of her pride and joy and share it on facebook. Neither of them were ready for how someone in the audience would react – but they were still blissfully ignorant in the meantime.\\n\\nPaying Dad A Visit Kelsey was ready to take herself and her new hairdo anywhere in the world, but the next stop was obviously her dad. He also needed to lavish his dear daughter with birthday gifts and love, and he was waiting patiently for his turn. Christin was more than happy to pass their sweetheart along, so they got in the car and she got ready to take her to her dad’s place for continued celebrations, where she would be staying for the whole weekend.\\n\\nHorrific Haircut While Kelsey was with her other parents, she usually spent time away from her phone, so Christin wasn’t expecting to hear from her much. But when she showed up on her mom’s doorstep a few days later, it took her a minute to even realize it was Kelsey; at first, she didn’t know who she was looking at. All of her hair and expensive highlights were gone – they had clearly been butchered off over the weekend.\\n\\nShe Was Devastated It didn’t help Christin figure out who it was that Kelsey was mostly refusing to take her face out of her hands. She was clearly humiliated, and why wouldn’t she be? In a matter of moments, she had gone from feeling her most beautiful ever, to having no hair and a ruined birthday. Kelsey wouldn’t even look at herself in the mirror – her self-confidence seemed, for the time being, ruined. Christin had to find out what had happened.\\n\\nWhy Did They Do This To Her? Since she couldn’t get Kelsey to talk, Christin jumped in the car and headed to Schaffen’s home and proceeded to scream at him and his girlfriend, asking them what had happened to Kelsey’s hair – and why had no one called her? It turns out they couldn’t stand her new hair, and thought she should be punished for it. They insisted that “actions have consequences,” so Christin decided to show them just how true that could be.\\n\\nSharing Their Pain Online Christin was beyond wit’s end, and wanted the world to share in shaming the unjust actions from her ex-husband. So she took to social media in order to show a little sample of the pain and humiliation that such rash actions and extreme punishments can have on a young, sensitive person. When people saw how extremely the situation had escalated from some beautiful highlights to a hairless head, the post went viral.\\n\\nThey Went Viral However, a viral post wasn’t actually Christin’s intentions – she only wanted friends and family to back her up in criticizing Schaffen’s unreasonable reaction…and, probably, to let off some steam. But the photos reached far beyond that. In fact, the photos had been shared 24,000 times and gathered 33,000 reactions in just a week. People around the internet weren’t holding back from sharing their feelings about this incident at all – and for Kelsey, it was all a bit much.\\n\\nNational Outrage Christin was definitely getting the support she had hoped for – in fact, she was getting way more than she had bargained for. Strangers from all over were calling the father and stepmother’s actions “disgusting” and “ridiculous.” One person wrote, “What a cruel thing for a parent to do to their child.So sorry that happened…beautiful no matter what!” Kelsey, on the other hand, wasn’t exactly ready for her face and freshly chopped hair to be all over the internet.\\n\\nCaught In The Middle? It was only to be expected that most of the people who got wind of this incident would be critical of her father’s actions – but a few people believed the problem went beyond a controlling and impulsive father. One person wrote, “I think the daughter was unfortunately in the middle of a get back at you between both parents fighting.” Is it possible that Christin intentionally went ahead to upset her ex-husband? It’s hard to say – but the speculations didn’t stop there…\\n\\nHighlights Are Normal, No? As parents, a lot of people had different feelings about the highlights themselves upon hearing about the incident. It wasn’t hard to see that plenty of other people would have been upset with their daughter’s hair makeover as well – not that they were excusing Kelsey’s dad’s behavior. But 13 is considered young by plenty of people to start dying hair. Other parents would say, whatever – it’s temporary! But the chopped hair raised more concern than just being overprotective.\\n\\nConstructive Ideas Fortunately, a lot of people on the internet knew just how she was feeling. Christin was getting her fair share of comments against Schaffen, but without even asking, Kelsey also started being addressed by the people who were mostly sympathetic for how she must be feeling. People wrote her compliments, suggestions for new hairstyles, and other heartening sentiments. Struggling to understand how complete strangers could understand her more than her own father, Kelsey turned to the comments for comfort.\\n\\nTeam Kelsey Once Kelsey’s story had gotten so much attention on the internet, Christin realized it could be more helpful to them than they’d originally realized. With so many people reading and sharing the post, it seemed likely they could make a pretty successful fundraiser that might help Kelsey fix her destroyed hair. So Christin set up a GoFundMe account to spread awareness about situations like Kelsey’s, and to invite people to donate to Kelsey’s ruined birthday present. They didn’t yet know that the story would continue.\\n\\nImportant Updates Surprised by how much of an interest people were taking in Kelsey’s story, Christin decided to keep the page updated with the feelings and events going on in Kelsey’s life in the aftermath. She had been pretty up, down, and all over the place – but she started doing the things that she loved again, and showing her face with her short hair. Christin was enjoying posting updates about these things and showing the people on both sides of the issue what a good kid Kelsey was.\\n\\nTaking Her Mind Off Things People would see Kelsey playing her favorite sport, going fishing with her cousins – all bravely sporting this hairdo that she hadn’t asked for. As people started to relate to Kelsey even more and find inspiration from her positivity, another side of the drama began to unfold without Christin or Kelsey even knowing. It wasn’t until they got a knock on the door one day that they realized how far the situation had escalated over the internet.\\n\\nPolice Got Involved Some people paying attention to the story had gotten the idea in their heads that this had been a case of parental abuse – and filed a police report. With the police involved, interviews with Fox8, Haskins Police Chief Colby Carroll broadcasted that Kelsey had been forced to cut her hair as punishment by Schaffen and his wife. “I’ve been doing this since ’92 and I’ve never had a case I would say that’s like this,” the cop said. It wasn’t long before Schaffen had police on his doorstep.\\n\\nActions Have Consequences An investigation began, and in the end the local judges determined that the haircutting couple should have a price to pay as well. They were then suspended from their firefighter positions, with the Middletown Township fire department, placing both of them on “administrative leave.” People disagreed on whether this punishment fit the crime, but after all, this was the couple that so deeply believed that actions have consequences. The investigation may have been closed, but the family drama wasn’t over just yet…\\n\\nStrong Single Mother Having two families is not an uncommon experience for many young people today, like Kelsey. She spends plenty of time with both of her parents, but she and her mother Christin don’t only live together; they’re practically best friends, too. It’s the kind of bond many mothers and daughters only dream of. And while she loves her father and he loves her dearly, he doesn’t always love hearing about her young life the way her mom does.\\n\\nNot With Father Anymore The authorities went on to grant Christin full custody over Kelsey. “After 5 hours of testimony and ‘evidence’ provided by both parties, the magistrate’s decision was for Kelsey to live with mom and her siblings!” Christin wrote on Facebook. “A huge weight has been lifted off of everyone’s shoulders and we cannot be any [happier]!!” Both Christin and Kelsey were relieved to keep Kelsey away from her impulsive father – but it meant a pretty big sacrifice for Kelsey.\\n\\nAunt Kelly To The Rescue Kelsey didn’t end up having a chance to retrieve any of her possessions from her father’s house before being legally removed from it. She didn’t actually want to go back to get her stuff, because that would mean she had to see him. But she was giving up a lot of things that she loved. In order to make up for it, her aunt Kelly set up a GoFundMe account that could bring in some funds for Kelsey to begin building her normal life back up again.\\n\\nWords Of Support Although everyone in the family could pretty much agree that they’d achieved the best possible outcome, it didn’t take away the pain of the whole experience for Kelsey and Christin – or the fact that their relationship with her father had changed so much and so suddenly. But many of their GoFundMe followers had words of support, like Tatum Mathis, who wrote his sympathies to Kelsey directly. And things were going to get even better from there.\\n\\nTarget Smashed! After almost no time at all, the audience that had been so supportive of Kelsey were ready to put their money where their comments had been. Aunt Kelly had only set a target of $1,000, but it was nearly reached almost immediately – and then the unthinkable happened: one anonymous donor contributed a $2,000 in one sweep. That brought a total of $3,543 to their campaign. The only question left was, what would they spend the funds on to make Kelsey’s life feel normal again?\\n\\nReturn To The Salon The answer wasn’t hard to find: they would go back to the salon to repair Kelsey’s hair before doing anything else. At the Lady Jane salon they chose, it turned out the whole staff had heard about the hair-rowing (pun intended) ordeal Kelsey had gone through. In fact, her godmother Haylee Ann had shown up ahead of time to prepare a little surprise – so Kelsey had a welcome committee waiting for her when she arrived there.\\n\\nStunning Results At the salon, the entire staff had been anticipating her arrival and were ready to make magic happen for Kelsey. Having already gotten through so many big changes with so much strength, they encouraged her to go big or go home – anything she wanted was now possible. Even her godmother Haylee Ann had come along in support of the cause, giving Kelsey the courage to make the day count. So she took their advice and opted for another big change.\\n\\nShe’s Happy Again While she was waiting for her own natural roots to grow back, Kelsey opted for a wig in the meantime. She loved the salon’s choices, and was truly happy with the result. It almost made up for everything that had gone wrong. And her mom, who had been desperate for Kelsey to feel like herself again, was so grateful. “I’d like to thank the ladies…for making my baby feel more like herself!” Christin said. “We truly appreciate you ladies so much!”\\n\\nFairy Godmother Christin knew that without Haylee Ann, the salon wouldn’t have been nearly as prepared to help Kelsey and things couldn’t have turned out so well. She made sure to give her a shout out on their fundraiser for all the people following the news, to show them what a supportive family Kelsey still had despite things going south with her dad. And after she posted pictures of Day #2 at the salon with all the great women in Kelsey’s life, even more surprises were on the way.'"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# len(df['article'].iloc[i].split(' '))\n",
        "' '.join(df['article'].iloc[i].split(' ')[:1000])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K1ebMX6twx1R",
        "outputId": "e24a6249-ce2b-476c-86cb-bed64c6ba084"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2846"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lens = []\n",
        "for i in range(len(df)):\n",
        "  lens.append(len(df['article'].iloc[i].split(' ')))\n",
        "  # lens.append(len(df['article'].iloc[i]))\n",
        "  # if len(df['article'].iloc[i])==100000:\n",
        "  #   # print(df[['title','article']].iloc[i])\n",
        "  #   print(df['answer'].iloc[i])\n",
        "  #   print(df['article'].iloc[i])\n",
        "# print(max(lens))"
      ],
      "metadata": {
        "id": "3iEPtFHnxK30"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.hist(np.log10(lens),bins=20)\n",
        "plt.axvline(x=np.log10(1500),color='red')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        },
        "id": "MlZy9JeLyL7c",
        "outputId": "a38a234a-4e13-4224-832c-578fcfde5030"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.lines.Line2D at 0x7f6c0d5aab10>"
            ]
          },
          "metadata": {},
          "execution_count": 54
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQ2klEQVR4nO3dbYxcV33H8e+PPAAqCBOydSPbwSAsUFo1kFqpo1RVikWVB4QjNURBLTFRKkttSoOoRA0viqj6IrzhIW0FtRJapwWSKEDjJoE2yoNQXxBwIASISVkiR7aVxCaAAaVQBf59Mccw3ux6Z72zO+uT70cazbnnnpn7n2vPb++euXM3VYUkqS8vmHQBkqTxM9wlqUOGuyR1yHCXpA4Z7pLUIcNdkjo0UrgnWZXktiTfTrInyXlJTktyd5LvtPuXt7FJcn2S6SQPJzlnaV+CJGmmUY/cPwp8oapeB5wN7AG2A/dU1QbgnrYMcBGwod22AR8ba8WSpHllvi8xJXkZ8BDw6hoanORR4IKqeiLJGcD9VfXaJP/U2p+eOW7JXoUk6SgnjzDmVcAh4J+TnA08CFwLrB4K7CeB1a29Btg39Pj9rW/OcD/99NNr/fr1C6tc0vJ79NHB/WtfO9k6BMCDDz74vaqamm3dKOF+MnAO8M6qeiDJR/nVFAwAVVVJFnQdgyTbGEzbcOaZZ7J79+6FPFzSJFxwweD+/vsnWYWaJI/PtW6UOff9wP6qeqAt38Yg7J9q0zG0+4Nt/QFg3dDj17a+o1TVjqraWFUbp6Zm/cEjSTpO84Z7VT0J7Ety5PewzcAjwC5ga+vbCtze2ruAK9tZM5uAw863S9LyGmVaBuCdwCeTnAo8BlzF4AfDrUmuBh4HLm9j7wIuBqaBZ9pYSdIyGincq+ohYOMsqzbPMraAaxZZlyRpEfyGqiR1yHCXpA4Z7pLUIcNdkjpkuEtSh0Y9FVLScVi//c7jfuze6y4ZYyV6vvHIXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDI4V7kr1JvpHkoSS7W99pSe5O8p12//LWnyTXJ5lO8nCSc5byBUiSnmshR+5/UFWvr6qNbXk7cE9VbQDuacsAFwEb2m0b8LFxFStJGs1ipmW2ADtbeydw6VD/TTXwJWBVkjMWsR1J0gKNGu4F/FeSB5Nsa32rq+qJ1n4SWN3aa4B9Q4/d3/okScvk5BHH/V5VHUjy68DdSb49vLKqKkktZMPth8Q2gDPPPHMhD5WeF9Zvv/O4H7v3ukvGWIlORCMduVfVgXZ/EPgccC7w1JHplnZ/sA0/AKwbevja1jfzOXdU1caq2jg1NXX8r0CS9BzzhnuSX0vy0iNt4A+BbwK7gK1t2Fbg9tbeBVzZzprZBBwemr6RJC2DUaZlVgOfS3Jk/Keq6gtJvgLcmuRq4HHg8jb+LuBiYBp4Brhq7FVLko5p3nCvqseAs2fpfxrYPEt/AdeMpTpJ0nHxG6qS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUodGDvckJyX5WpI72vKrkjyQZDrJLUlObf0vbMvTbf36pSldkjSXhRy5XwvsGVr+IPDhqnoN8APg6tZ/NfCD1v/hNk6StIxGCvcka4FLgBvacoA3Are1ITuBS1t7S1umrd/cxkuSlsmoR+4fAd4D/KItvwL4YVU925b3A2taew2wD6CtP9zGS5KWybzhnuTNwMGqenCcG06yLcnuJLsPHTo0zqeWpOe9UY7czwfekmQvcDOD6ZiPAquSnNzGrAUOtPYBYB1AW/8y4OmZT1pVO6pqY1VtnJqaWtSLkCQdbd5wr6r3VtXaqloPXAHcW1V/DNwHXNaGbQVub+1dbZm2/t6qqrFWLUk6psWc5/7XwLuTTDOYU7+x9d8IvKL1vxvYvrgSJUkLdfL8Q36lqu4H7m/tx4BzZxnzU+CtY6hNknSc/IaqJHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOLejP7EnPN+u33znpEqTj4pG7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aN5wT/KiJF9O8vUk30rygdb/qiQPJJlOckuSU1v/C9vydFu/fmlfgiRpplGO3H8GvLGqzgZeD1yYZBPwQeDDVfUa4AfA1W381cAPWv+H2zhJ0jKaN9xr4Cdt8ZR2K+CNwG2tfydwaWtvacu09ZuTZGwVS5LmNdKce5KTkjwEHATuBr4L/LCqnm1D9gNrWnsNsA+grT8MvGKW59yWZHeS3YcOHVrcq5AkHWWkcK+qn1fV64G1wLnA6xa74araUVUbq2rj1NTUYp9OkjRkQWfLVNUPgfuA84BVSY5cMngtcKC1DwDrANr6lwFPj6VaSdJIRjlbZirJqtZ+MfAmYA+DkL+sDdsK3N7au9oybf29VVXjLFqSdGyj/LGOM4CdSU5i8MPg1qq6I8kjwM1J/g74GnBjG38j8K9JpoHvA1csQd2SpGOYN9yr6mHgDbP0P8Zg/n1m/0+Bt46lOknScfHP7EkdWsyfB9x73SVjrEST4uUHJKlDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWjecE+yLsl9SR5J8q0k17b+05LcneQ77f7lrT9Jrk8yneThJOcs9YuQJB1tlCP3Z4G/qqqzgE3ANUnOArYD91TVBuCetgxwEbCh3bYBHxt71ZKkY5o33Kvqiar6amv/GNgDrAG2ADvbsJ3Apa29BbipBr4ErEpyxtgrlyTNaUFz7knWA28AHgBWV9UTbdWTwOrWXgPsG3rY/tYnSVomI4d7kpcAnwHeVVU/Gl5XVQXUQjacZFuS3Ul2Hzp0aCEPlSTNY6RwT3IKg2D/ZFV9tnU/dWS6pd0fbP0HgHVDD1/b+o5SVTuqamNVbZyamjre+iVJsxjlbJkANwJ7qupDQ6t2AVtbeytw+1D/le2smU3A4aHpG0nSMjh5hDHnA28HvpHkodb3PuA64NYkVwOPA5e3dXcBFwPTwDPAVWOtWJI0r3nDvar+G8gcqzfPMr6AaxZZlyRpEfyGqiR1yHCXpA4Z7pLUIcNdkjo0ytky0glt/fY7J12CtOw8cpekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHfKSv5KOcqxLJN/82NMAXHGMMXuvu2TsNWnhPHKXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalD84Z7kk8kOZjkm0N9pyW5O8l32v3LW3+SXJ9kOsnDSc5ZyuIlSbMb5cj9X4ALZ/RtB+6pqg3APW0Z4CJgQ7ttAz42njIlSQsxb7hX1ReB78/o3gLsbO2dwKVD/TfVwJeAVUnOGFexkqTRHO/lB1ZX1ROt/SSwurXXAPuGxu1vfU8gLcKxvhIv6bkW/YFqVRVQC31ckm1JdifZfejQocWWIUkacrzh/tSR6ZZ2f7D1HwDWDY1b2/qeo6p2VNXGqto4NTV1nGVIkmZzvOG+C9ja2luB24f6r2xnzWwCDg9N30iSlsm8c+5JPg1cAJyeZD/wfuA64NYkVwOPA5e34XcBFwPTwDPAVUtQsyRpHvOGe1W9bY5Vm2cZW8A1iy1KkrQ4fkNVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOHe+FwyRpVou5yNve6y4ZYyXPbx65S1KHDHdJ6pDhLkkdcs5dy8Y/uCEtH4/cJalDhrskdchwl6QOGe6S1CHDXZI65NkyJyC/AShpPh65S1KHPHLXgniuupaSv5WOj0fuktQhw12SOmS4S1KHnHN/nnHOXHp+8MhdkjpkuEtSh5yWmRCnRyQtpSUJ9yQXAh8FTgJuqKrrlmI7knSE58gfbezTMklOAv4RuAg4C3hbkrPGvR1J0tyW4sj9XGC6qh4DSHIzsAV4ZAm2NVFOrUhaqZYi3NcA+4aW9wO/uwTbAQxYSYs3yRxZqimhiX2gmmQbsK0t/iTJo4t8ytOB7y3yOZaaNY6HNY7Hgms870jjg28eezGz6HIfzpQPLmr7r5xrxVKE+wFg3dDy2tZ3lKraAewY10aT7K6qjeN6vqVgjeNhjeOx0mtc6fXByq5xKc5z/wqwIcmrkpwKXAHsWoLtSJLmMPYj96p6NslfAP/J4FTIT1TVt8a9HUnS3JZkzr2q7gLuWornPoaxTfEsIWscD2scj5Ve40qvD1ZwjamqSdcgSRozry0jSR064cI9yYVJHk0ynWT7LOvfkeRQkofa7U+Xub5PJDmY5JtzrE+S61v9Dyc5ZznrG7HGC5IcHtqHfzOBGtcluS/JI0m+leTaWcZMbF+OWN9E92OSFyX5cpKvtxo/MMuYFya5pe3DB5KsX4E1TvQ9PVTHSUm+luSOWdZNdD/OqqpOmBuDD2i/C7waOBX4OnDWjDHvAP5hgjX+PnAO8M051l8MfB4IsAl4YAXWeAFwx4T/rc8AzmntlwL/M8u/9cT25Yj1TXQ/tv3yktY+BXgA2DRjzJ8DH2/tK4BbVmCNE31PD9XxbuBTs/2bTno/znY70Y7cf3lpg6r6P+DIpQ1WjKr6IvD9YwzZAtxUA18CViU5Y3mqGxihxomrqieq6qut/WNgD4NvPw+b2L4csb6JavvlJ23xlHab+SHbFmBna98GbE6SZSpx1BonLsla4BLghjmGTHQ/zuZEC/fZLm0w2xvqj9qv6bclWTfL+kka9TVM2nntV+XPJ/nNSRbSfsV9A4OjumErYl8eoz6Y8H5sUwkPAQeBu6tqzn1YVc8Ch4FXrLAaYfLv6Y8A7wF+Mcf6ie/HmU60cB/FfwDrq+q3gbv51U9Tje6rwCur6mzg74F/n1QhSV4CfAZ4V1X9aFJ1zGWe+ia+H6vq51X1egbfFD83yW8tdw3zGaHGib6nk7wZOFhVDy7ndhfrRAv3eS9tUFVPV9XP2uINwO8sU22jGunyDJNUVT868qtyDb6zcEqS05e7jiSnMAjOT1bVZ2cZMtF9OV99K2U/tu3/ELgPuHDGql/uwyQnAy8Dnl7e6gbmqnEFvKfPB96SZC+DqeA3Jvm3GWNWzH484kQL93kvbTBjzvUtDOZCV5JdwJXtTI9NwOGqemLSRQ1L8htH5guTnMvg/8my/kdt278R2FNVH5pj2MT25Sj1TXo/JplKsqq1Xwy8Cfj2jGG7gK2tfRlwb7VPBVdKjZN+T1fVe6tqbVWtZ5A591bVn8wYNtH9OJsT6s/s1RyXNkjyt8DuqtoF/GWStwDPMvjQ8B3LWWOSTzM4S+L0JPuB9zP4kIiq+jiDb+5eDEwDzwBXLWd9I9Z4GfBnSZ4F/he4YgL/Uc8H3g58o83HArwPOHOozknuy1Hqm/R+PAPYmcEf0HkBcGtV3THj/XIj8K9Jphm8X65YxvpGrXGi7+m5rLD9+Bx+Q1WSOnSiTctIkkZguEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KH/B3T+tRPN5O8MAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for i in range(200):\n",
        "i=2\n",
        "# if i%10==0:\n",
        "#   print(i)\n",
        "# context = df['article'].iloc[i]\n",
        "context = ' '.join(df['article'].iloc[i].split(' ')[:1500])\n",
        "question = df['teaser'].iloc[i]\n",
        "input_text = \"context: %s <question for context: %s </s>\" % (context,question)\n",
        "features = tokenizer([input_text], return_tensors='pt')\n",
        "targets = tokenizer(['<pad> '+ df['answer'].iloc[i] +'</s>'], return_tensors='pt')['input_ids'].to(device) ### check\n",
        "targets_in = targets[:,:-1].contiguous()\n",
        "targets_out = targets[:,1:].clone().detach() ### check\n",
        "targets_out[targets[:,1:] == tokenizer.pad_token_id] = -100\n",
        "try:\n",
        "  outputs = model(input_ids=features['input_ids'].to(device), attention_mask=features['attention_mask'].to(device), decoder_input_ids=targets_in, labels = targets_out)\n",
        "  loss = outputs[0]\n",
        "  loss.backward()\n",
        "  optim.step()\n",
        "except Exception as e:\n",
        "  print(e)\n",
        "  # continue\n",
        "  pass\n"
      ],
      "metadata": {
        "id": "zf4qwbqH3Swk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.memory_reserved(0)/1e6, torch.cuda.memory_allocated(0)/1e6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "axW4MYafqzZx",
        "outputId": "e6ac3839-aaba-47e8-f11b-ceadc834e09d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(15556.673536, 3690.210304)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "del features, targets, targets_out, targets_in, outputs, loss"
      ],
      "metadata": {
        "id": "pNqKypjuq-X3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with torch.no_grad():\n",
        "  print(torch.cuda.memory_reserved(0)/1e6, torch.cuda.memory_allocated(0)/1e6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "37aAPncJtLaR",
        "outputId": "8e4a93f1-5d24-4a49-b9d6-dfa653ad5476"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "13895.729152 11454.027776\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# del optim\n",
        "# del model\n",
        "# torch.cuda.empty_cache()\n",
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()\n",
        "# model.eval()"
      ],
      "metadata": {
        "id": "QkEBrsLrqxow"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "End of debugging effort"
      ],
      "metadata": {
        "id": "cDgy7R5CzxHv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(N_epochs):\n",
        "  print('Epoch',epoch)\n",
        "  for i in tqdm(range(train_test_split)):\n",
        "    context = df['article'].iloc[i]\n",
        "    question = df['teaser'].iloc[i]\n",
        "    input_text = \"context: %s <question for context: %s </s>\" % (context,question)\n",
        "    features = tokenizer([input_text], return_tensors='pt')\n",
        "    targets = tokenizer(['<pad> '+ df['answer'].iloc[i] +'</s>'], return_tensors='pt')['input_ids'].to(device) ### check\n",
        "    targets_in = targets[:,:-1].contiguous()\n",
        "    targets_out = targets[:,1:].clone().detach() ### check\n",
        "    targets_out[targets[:,1:] == tokenizer.pad_token_id] = -100\n",
        "    try:\n",
        "      outputs = model(input_ids=features['input_ids'].to(device), attention_mask=features['attention_mask'].to(device), decoder_input_ids=targets_in, labels = targets_out)\n",
        "      loss = outputs[0]\n",
        "      loss.backward()\n",
        "      optim.step()\n",
        "    except:\n",
        "      continue\n",
        "    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7h-zLKuzNW6J",
        "outputId": "acf7907b-c564-47a4-a48f-6bb0957ce74b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:40<00:00, 49.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 51.07it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 50.02it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 50.45it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 50.04it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 50.99it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:39<00:00, 51.05it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 2000/2000 [00:38<00:00, 51.39it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "### to suppress output\n",
        "model.eval()"
      ],
      "metadata": {
        "id": "8tze_S0PS9qS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['after 10 epochs'] = df['answer'].copy()\n",
        "for i in tqdm(range(len(df))):\n",
        "    try:\n",
        "        df['after 10 epochs'].iloc[i] = get_answer(df['teaser'].iloc[i], df['article'].iloc[i])\n",
        "    except RuntimeError as e:\n",
        "        # print(e)\n",
        "        df['after 10 epochs'].iloc[i] = 'Too long, GPU memory exceeded'\n",
        "    if i%10 ==0:\n",
        "        df.to_csv('mydata_10epochs.csv')\n",
        "    # torch.cuda.empty_cache()\n",
        "df.to_csv('mydata_10epochs.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hvqK8texYnVX",
        "outputId": "7a80ab6a-28a8-429a-e618-056a89aa44f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/2385 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self._setitem_single_block(indexer, value, name)\n",
            "100%|██████████| 2385/2385 [02:20<00:00, 16.99it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# df['after 2 epochs'] = df['answer'].copy()\n",
        "# for i in tqdm(range(len(df))):\n",
        "#     try:\n",
        "#         df['after 2 epochs'].iloc[i] = get_answer(df['teaser'].iloc[i], df['article'].iloc[i])\n",
        "#     except RuntimeError as e:\n",
        "#         # print(e)\n",
        "#         df['after 2 epochs'].iloc[i] = 'Too long, GPU memory exceeded'\n",
        "#     if i%10 ==0:\n",
        "#         df.to_csv('mydata_2epochs.csv')\n",
        "#     # torch.cuda.empty_cache()\n",
        "# df.to_csv('mydata_2epochs.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IUgYYzdGNZTI",
        "outputId": "40a70fe5-53c1-436c-da30-07a98928cf29"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/2385 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  self._setitem_single_block(indexer, value, name)\n",
            "100%|██████████| 2385/2385 [07:27<00:00,  5.33it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['after 2 epochs'].iloc[20]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "pG8FNbIOQGrx",
        "outputId": "cd01a2d7-2cca-40ce-9b3c-a4338e3c3dab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<pad> Taylor Swift is a fan of the folklore album.</s>'"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['after 2 epochs'].equals(df['old QA'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TnFM48GkSv8K",
        "outputId": "02e66d85-9b09-4964-82eb-758ed91f8801"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.loc[df['after 2 epochs'] != df['old QA']][['after 2 epochs','old QA']]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 655
        },
        "id": "dfTRCe41Vrhr",
        "outputId": "53631d10-922c-40fc-d1e5-36f2bb47fdf3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f85a646b-e688-480b-8da7-55be0d89b1da\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>after 2 epochs</th>\n",
              "      <th>old QA</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>&lt;pad&gt; She would go to see her dad and get her ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>&lt;pad&gt; He was able to help Andrella with her ca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>&lt;pad&gt; She was a size 8 and had a large ovary.&lt;/s&gt;</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>&lt;pad&gt; She was unable to find the information s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>&lt;pad&gt; They are becoming a standard in other ci...</td>\n",
              "      <td>&lt;pad&gt; The city is considering a plan to make t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2332</th>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>&lt;pad&gt; Apple is rumored to be replacing the 30-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2335</th>\n",
              "      <td>&lt;pad&gt; The final season of the series is set fo...</td>\n",
              "      <td>&lt;pad&gt; The final season of the series will debu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2364</th>\n",
              "      <td>&lt;pad&gt; The show will return to Netflix in 2021....</td>\n",
              "      <td>&lt;pad&gt; The show will be set in a different loca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2366</th>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>&lt;pad&gt; Moderna or Pfizer booster shots are the ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2368</th>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>&lt;pad&gt; Andrew Garfield was replaced by Tom Holl...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>230 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f85a646b-e688-480b-8da7-55be0d89b1da')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f85a646b-e688-480b-8da7-55be0d89b1da button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f85a646b-e688-480b-8da7-55be0d89b1da');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                         after 2 epochs                                             old QA\n",
              "2                         Too long, GPU memory exceeded  <pad> She would go to see her dad and get her ...\n",
              "12                        Too long, GPU memory exceeded  <pad> He was able to help Andrella with her ca...\n",
              "14                        Too long, GPU memory exceeded  <pad> She was a size 8 and had a large ovary.</s>\n",
              "16                        Too long, GPU memory exceeded  <pad> She was unable to find the information s...\n",
              "19    <pad> They are becoming a standard in other ci...  <pad> The city is considering a plan to make t...\n",
              "...                                                 ...                                                ...\n",
              "2332           <pad> No answer available in context</s>  <pad> Apple is rumored to be replacing the 30-...\n",
              "2335  <pad> The final season of the series is set fo...  <pad> The final season of the series will debu...\n",
              "2364  <pad> The show will return to Netflix in 2021....  <pad> The show will be set in a different loca...\n",
              "2366           <pad> No answer available in context</s>  <pad> Moderna or Pfizer booster shots are the ...\n",
              "2368           <pad> No answer available in context</s>  <pad> Andrew Garfield was replaced by Tom Holl...\n",
              "\n",
              "[230 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.loc[df['after 2 epochs'] != df['old QA']][['after 2 epochs','old QA']]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        },
        "id": "t3RK1hPXV8YQ",
        "outputId": "cadac4ee-ff9c-4824-edc4-79196952e294"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-659aac3c4d81>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'after 2 epochs'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'old QA'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'after 2 epochs'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'old QA'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('mydata_10epochs.csv')"
      ],
      "metadata": {
        "id": "awfK742wyuuX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.loc[df['after 2 epochs'] != df['old QA']][['answer','old QA','after 2 epochs','after 10 epochs']]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1251
        },
        "id": "tLB8V67Z1MB3",
        "outputId": "13dbf28c-f77f-4b5d-8118-8edbf6efb964"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4b9ecf96-ef38-4b8b-b0a4-b16e1d31c811\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>answer</th>\n",
              "      <th>old QA</th>\n",
              "      <th>after 2 epochs</th>\n",
              "      <th>after 10 epochs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Makes a viral social media post, wins full cus...</td>\n",
              "      <td>&lt;pad&gt; She would go to see her dad and get her ...</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>She had three kids in the back that weren't bu...</td>\n",
              "      <td>&lt;pad&gt; He was able to help Andrella with her ca...</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>It's a 60lbs ovarian cyst, no cops are called ...</td>\n",
              "      <td>&lt;pad&gt; She was a size 8 and had a large ovary.&lt;/s&gt;</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Husband with secret second family. Except it w...</td>\n",
              "      <td>&lt;pad&gt; She was unable to find the information s...</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Cyclists don't like being run over by cars.</td>\n",
              "      <td>&lt;pad&gt; The city is considering a plan to make t...</td>\n",
              "      <td>&lt;pad&gt; They are becoming a standard in other ci...</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2332</th>\n",
              "      <td>Might replace lightning connector with USB-C</td>\n",
              "      <td>&lt;pad&gt; Apple is rumored to be replacing the 30-...</td>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2335</th>\n",
              "      <td>Debut is April 14</td>\n",
              "      <td>&lt;pad&gt; The final season of the series will debu...</td>\n",
              "      <td>&lt;pad&gt; The final season of the series is set fo...</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2364</th>\n",
              "      <td>[Season 3 will return sometime in 2021. Joe wi...</td>\n",
              "      <td>&lt;pad&gt; The show will be set in a different loca...</td>\n",
              "      <td>&lt;pad&gt; The show will return to Netflix in 2021....</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2366</th>\n",
              "      <td>Pfizer , Moderna , J&amp;J</td>\n",
              "      <td>&lt;pad&gt; Moderna or Pfizer booster shots are the ...</td>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2368</th>\n",
              "      <td>He decided not to attend a The Amazing Spider-...</td>\n",
              "      <td>&lt;pad&gt; Andrew Garfield was replaced by Tom Holl...</td>\n",
              "      <td>&lt;pad&gt; No answer available in context&lt;/s&gt;</td>\n",
              "      <td>Too long, GPU memory exceeded</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>230 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b9ecf96-ef38-4b8b-b0a4-b16e1d31c811')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b9ecf96-ef38-4b8b-b0a4-b16e1d31c811 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b9ecf96-ef38-4b8b-b0a4-b16e1d31c811');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                 answer  ...                after 10 epochs\n",
              "2     Makes a viral social media post, wins full cus...  ...  Too long, GPU memory exceeded\n",
              "12    She had three kids in the back that weren't bu...  ...  Too long, GPU memory exceeded\n",
              "14    It's a 60lbs ovarian cyst, no cops are called ...  ...  Too long, GPU memory exceeded\n",
              "16    Husband with secret second family. Except it w...  ...  Too long, GPU memory exceeded\n",
              "19          Cyclists don't like being run over by cars.  ...  Too long, GPU memory exceeded\n",
              "...                                                 ...  ...                            ...\n",
              "2332       Might replace lightning connector with USB-C  ...  Too long, GPU memory exceeded\n",
              "2335                                  Debut is April 14  ...  Too long, GPU memory exceeded\n",
              "2364  [Season 3 will return sometime in 2021. Joe wi...  ...  Too long, GPU memory exceeded\n",
              "2366                             Pfizer , Moderna , J&J  ...  Too long, GPU memory exceeded\n",
              "2368  He decided not to attend a The Amazing Spider-...  ...  Too long, GPU memory exceeded\n",
              "\n",
              "[230 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "FuHu6iyZ1Mh7"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}