Lesson 1 intro to torchtext with text classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's examine what the data looks like.\n",
    "\n",
    "(Note: This repo does not contain the full data. To get the full data, go to the [Kaggle competition page](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge) and download the data for yourself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"data/train.csv\").head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000eefc67a2c930f</td>\n",
       "      <td>Radial symmetry \\n\\nSeveral now extinct lineag...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000f35deef84dc4a</td>\n",
       "      <td>There's no need to apologize. A Wikipedia arti...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  000eefc67a2c930f  Radial symmetry \\n\\nSeveral now extinct lineag...      0   \n",
       "1  000f35deef84dc4a  There's no need to apologize. A Wikipedia arti...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"data/valid.csv\").head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apparently we have to predict 6 labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00001cee341fdb12</td>\n",
       "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000247867823ef7</td>\n",
       "      <td>== From RfC == \\n\\n The title is fine as it is...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text\n",
       "0  00001cee341fdb12  Yo bitch Ja Rule is more succesful then you'll...\n",
       "1  0000247867823ef7  == From RfC == \\n\\n The title is fine as it is..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"data/test.csv\").head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declaring Fields"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Field class determines how the data is preprocessed and converted into a numeric format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import Field"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want comment_text field to be converted to lowercase, tokenized on whitespace, and preprocessed. So we tell that to the Field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize = lambda x: x.split()\n",
    "TEXT = Field(sequential=True, tokenize=tokenize, lower=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That was simple. The preprocessing of the labels is even easier, since they are already converted into a binary encoding.\n",
    "All we need to do is to tell the Field class that the labels are already processed. We do this by passing the use_vocab=False keyword to the constructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL = Field(sequential=False, use_vocab=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating the Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use the TabularDataset class to read our data, since it is in csv format (TabularDataset handles csv, tsv, and json files as of now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import TabularDataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the train and validation data, we need to process the labels. The fields we pass in must be in the same order as the columns. For fields we don't use, we pass in a tuple where the second element is None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.57 ms, sys: 1.81 ms, total: 6.37 ms\n",
      "Wall time: 5.25 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tv_datafields = [(\"id\", None), # we won't be needing the id, so we pass in None as the field\n",
    "                 (\"comment_text\", TEXT), (\"toxic\", LABEL),\n",
    "                 (\"severe_toxic\", LABEL), (\"threat\", LABEL),\n",
    "                 (\"obscene\", LABEL), (\"insult\", LABEL),\n",
    "                 (\"identity_hate\", LABEL)]\n",
    "\n",
    "trn, vld = TabularDataset.splits(\n",
    "        path=\"data\", # the root directory where the data lies\n",
    "        train='train.csv', validation=\"valid.csv\",\n",
    "        format='csv',\n",
    "        skip_header=True, # if your csv header has a header, make sure to pass this to ensure it doesn't get proceesed as data!\n",
    "        fields=tv_datafields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the test data, we don't have any labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.98 ms, sys: 877 µs, total: 2.86 ms\n",
      "Wall time: 2.17 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tst_datafields = [(\"id\", None), # we won't be needing the id, so we pass in None as the field\n",
    "                 (\"comment_text\", TEXT)\n",
    "]\n",
    "\n",
    "tst = TabularDataset(\n",
    "        path=\"data/test.csv\", # the file path\n",
    "        format='csv',\n",
    "        skip_header=True, # if your csv header has a header, make sure to pass this to ensure it doesn't get proceesed as data!\n",
    "        fields=tst_datafields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the TEXT field to convert words into integers, it needs to be told what the entire vocabulary is. To do this, we run TEXT.build_vocab, passing in the dataset to build the vocabulary on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.29 ms, sys: 207 µs, total: 2.5 ms\n",
      "Wall time: 2.54 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "TEXT.build_vocab(trn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look at what the vocab looks like.\n",
    "\n",
    "The vocab.freqs is a collections.Counter object, so we can take a look at the most frequent words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 78),\n",
       " ('to', 41),\n",
       " ('you', 33),\n",
       " ('of', 30),\n",
       " ('and', 26),\n",
       " ('a', 26),\n",
       " ('is', 24),\n",
       " ('that', 22),\n",
       " ('i', 20),\n",
       " ('if', 19)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.freqs.most_common(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also instructive to take a look inside the Dataset. Datasets can be indexed like normal lists, so we'll look at the first element."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torchtext.data.example.Example at 0x11023a860>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each element of the dataset is an Example object that bundles the attributes of a single data point together."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['comment_text', 'toxic', 'severe_toxic', 'threat', 'obscene', 'insult', 'identity_hate'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0].__dict__.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that the comment text is already tokenized for us."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['explanation', 'why', 'the']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn[0].comment_text[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking good. Now, let's build the Iterator which will allow us to load the data into our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating the Iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import Iterator, BucketIterator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "During training, we'll be using a special kind of Iterator, called the **BucketIterator**.\n",
    "\n",
    "When we pass data into a neural network, we want the data to be padded to be the same length so that we can process them in batch:\n",
    "\n",
    "e.g.\n",
    "\\[ \n",
    "\\[3, 15, 2, 7\\],\n",
    "\\[4, 1\\], \n",
    "\\[5, 5, 6, 8, 1\\] \n",
    "\\] -> \\[ \n",
    "\\[3, 15, 2, 7, **0**\\],\n",
    "\\[4, 1, **0**, **0**, **0**\\], \n",
    "\\[5, 5, 6, 8, 1\\] \n",
    "\\] \n",
    "\n",
    "If the sequences differ greatly in length, the padding will consume a lot of wasteful memory and time.\n",
    "\n",
    "The BucketIterator groups sequences of similar lengths together for each batch to minimize padding. Handy, right?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter, val_iter = BucketIterator.splits(\n",
    "        (trn, vld), # we pass in the datasets we want the iterator to draw data from\n",
    "        batch_sizes=(64, 64),\n",
    "        device=-1, # if you want to use the GPU, specify the GPU number here\n",
    "        sort_key=lambda x: len(x.comment_text), # the BucketIterator needs to be told what function it should use to group the data.\n",
    "        sort_within_batch=False,\n",
    "        repeat=False # we pass repeat=False because we want to wrap this Iterator layer.\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look at what the output of the BucketIterator looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[torchtext.data.batch.Batch of size 25]\n",
       "\t[.comment_text]:[torch.LongTensor of size 494x25]\n",
       "\t[.toxic]:[torch.LongTensor of size 25]\n",
       "\t[.severe_toxic]:[torch.LongTensor of size 25]\n",
       "\t[.threat]:[torch.LongTensor of size 25]\n",
       "\t[.obscene]:[torch.LongTensor of size 25]\n",
       "\t[.insult]:[torch.LongTensor of size 25]\n",
       "\t[.identity_hate]:[torch.LongTensor of size 25]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch = next(train_iter.__iter__()); batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The batch has all the fields we passed to the Dataset as attributes. The batch data can be accessed through the attribute with the same name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['batch_size', 'dataset', 'train', 'fields', 'comment_text', 'toxic', 'severe_toxic', 'threat', 'obscene', 'insult', 'identity_hate'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch.__dict__.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the test set, we don't want the data to be shuffled. This is why we'll be using a standard Iterator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_iter = Iterator(tst, batch_size=64, device=-1, sort=False, sort_within_batch=False, repeat=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wrapping the Iterator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Currently, the iterator returns a custom datatype called torchtext.data.Batch. This makes code reuse difficult (since each time the column names change, we need to modify the code), and makes torchtext hard to use with other libraries for some use cases (like torchsample and fastai). \n",
    "\n",
    "I hope this will be dealt with in the future (I'm considering filing a PR if I can decide what the API should look like), but in the meantime, we'll hack on a simple wrapper to make the batches easy to use. \n",
    "\n",
    "Concretely, we'll convert the batch to a tuple in the form (x, y) where x is the independent variable (the input to the model) and y is the dependent variable (the supervision data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchWrapper:\n",
    "    def __init__(self, dl, x_var, y_vars):\n",
    "        self.dl, self.x_var, self.y_vars = dl, x_var, y_vars # we pass in the list of attributes for x and y\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for batch in self.dl:\n",
    "            x = getattr(batch, self.x_var) # we assume only one input in this wrapper\n",
    "            \n",
    "            if self.y_vars is not None: # we will concatenate y into a single tensor\n",
    "                y = torch.cat([getattr(batch, feat).unsqueeze(1) for feat in self.y_vars], dim=1).float()\n",
    "            else:\n",
    "                y = torch.zeros((1))\n",
    "\n",
    "            yield (x, y)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use this to wrap the BucketIterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = BatchWrapper(train_iter, \"comment_text\", [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"])\n",
    "valid_dl = BatchWrapper(val_iter, \"comment_text\", [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"])\n",
    "test_dl = BatchWrapper(test_iter, \"comment_text\", None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "   606   354   334  ...     63    15    15\n",
       "   693    63    55  ...      4   601    29\n",
       "   584     4   520  ...    664   242    21\n",
       "        ...          ⋱          ...       \n",
       "     1     1     1  ...      1     1    84\n",
       "     1     1     1  ...      1     1   118\n",
       "     1     1     1  ...      1     1    15\n",
       " [torch.LongTensor of size 494x25], Variable containing:\n",
       "     0     0     0     0     0     0\n",
       "     1     1     0     1     1     0\n",
       "     1     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     1     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       "     0     0     0     0     0     0\n",
       " [torch.FloatTensor of size 25x6])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(train_dl.__iter__())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we're ready to start training a model!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training a Text Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll use a simple LSTM as a baseline example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleBiLSTMBaseline(nn.Module):\n",
    "    def __init__(self, hidden_dim, emb_dim=300,\n",
    "                 spatial_dropout=0.05, recurrent_dropout=0.1, num_linear=1):\n",
    "        super().__init__() # don't forget to call this!\n",
    "        self.embedding = nn.Embedding(len(TEXT.vocab), emb_dim)\n",
    "        self.encoder = nn.LSTM(emb_dim, hidden_dim, num_layers=1, dropout=recurrent_dropout)\n",
    "        self.linear_layers = []\n",
    "        for _ in range(num_linear - 1):\n",
    "            self.linear_layers.append(nn.Linear(hidden_dim, hidden_dim))\n",
    "        self.linear_layers = nn.ModuleList(self.linear_layers)\n",
    "        self.predictor = nn.Linear(hidden_dim, 6)\n",
    "    \n",
    "    def forward(self, seq):\n",
    "        hdn, _ = self.encoder(self.embedding(seq))\n",
    "        feature = hdn[-1, :, :]\n",
    "        for layer in self.linear_layers:\n",
    "            feature = layer(feature)\n",
    "        preds = self.predictor(feature)\n",
    "        return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleBiLSTMBaseline(\n",
       "  (embedding): Embedding(784, 100)\n",
       "  (encoder): LSTM(100, 500, dropout=0.1)\n",
       "  (linear_layers): ModuleList(\n",
       "  )\n",
       "  (predictor): Linear(in_features=500, out_features=6)\n",
       ")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "em_sz = 100\n",
    "nh = 500\n",
    "nl = 3\n",
    "model = SimpleBiLSTMBaseline(nh, emb_dim=em_sz); model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you're using a GPU, remember to call model.cuda() to move your model to the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), lr=1e-2)\n",
    "loss_func = nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:17<00:00, 17.39s/it]/Users/keitakurita/anaconda/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: generator 'Iterator.__iter__' raised StopIteration\n",
      "  \n",
      "\n",
      "  0%|          | 0/1 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Training Loss: 13.7826, Validation Loss: 2.0098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:13<00:00, 13.78s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2, Training Loss: 3.9747, Validation Loss: 1.9117\n",
      "CPU times: user 1min 21s, sys: 3.43 s, total: 1min 25s\n",
      "Wall time: 32.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for epoch in range(1, epochs + 1):\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0\n",
    "    model.train() # turn on training mode\n",
    "    for x, y in tqdm.tqdm(train_dl): # thanks to our wrapper, we can intuitively iterate over our data!\n",
    "        opt.zero_grad()\n",
    "\n",
    "        preds = model(x)\n",
    "        loss = loss_func(preds, y)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        \n",
    "        running_loss += loss.data[0] * x.size(0)\n",
    "        \n",
    "    epoch_loss = running_loss / len(trn)\n",
    "    \n",
    "    # calculate the validation loss for this epoch\n",
    "    val_loss = 0.0\n",
    "    model.eval() # turn on evaluation mode\n",
    "    for x, y in valid_dl:\n",
    "        preds = model(x)\n",
    "        loss = loss_func(preds, y)\n",
    "        val_loss += loss.data[0] * x.size(0)\n",
    "\n",
    "    val_loss /= len(vld)\n",
    "    print('Epoch: {}, Training Loss: {:.4f}, Validation Loss: {:.4f}'.format(epoch, epoch_loss, val_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Writing Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we output the data in the format required by the competition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.BatchWrapper at 0x110269c18>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  2.42it/s]/Users/keitakurita/anaconda/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: generator 'Iterator.__iter__' raised StopIteration\n",
      "  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_preds = []\n",
    "for x, y in tqdm.tqdm(test_dl):\n",
    "    preds = model(x)\n",
    "    # if you're data is on the GPU, you need to move the data back to the cpu\n",
    "    # preds = preds.data.cpu().numpy()\n",
    "    preds = preds.data.numpy()\n",
    "    # the actual outputs of the model are logits, so we need to pass these values to the sigmoid function\n",
    "    preds = 1 / (1 + np.exp(-preds))\n",
    "    test_preds.append(preds)\n",
    "test_preds = np.hstack(test_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/test.csv\")\n",
    "for i, col in enumerate([\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]):\n",
    "    df[col] = test_preds[:, i]\n",
    "\n",
    "# if you want to write the submission file to disk, uncomment and run the below code\n",
    "# df.drop(\"comment_text\", axis=1).to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00001cee341fdb12</td>\n",
       "      <td>Yo bitch Ja Rule is more succesful then you'll...</td>\n",
       "      <td>0.068491</td>\n",
       "      <td>0.014765</td>\n",
       "      <td>0.00251</td>\n",
       "      <td>0.012214</td>\n",
       "      <td>0.014872</td>\n",
       "      <td>0.002175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000247867823ef7</td>\n",
       "      <td>== From RfC == \\n\\n The title is fine as it is...</td>\n",
       "      <td>0.068491</td>\n",
       "      <td>0.014765</td>\n",
       "      <td>0.00251</td>\n",
       "      <td>0.012214</td>\n",
       "      <td>0.014872</td>\n",
       "      <td>0.002175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00013b17ad220c46</td>\n",
       "      <td>\" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...</td>\n",
       "      <td>0.068491</td>\n",
       "      <td>0.014765</td>\n",
       "      <td>0.00251</td>\n",
       "      <td>0.012214</td>\n",
       "      <td>0.014872</td>\n",
       "      <td>0.002175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  \\\n",
       "0  00001cee341fdb12  Yo bitch Ja Rule is more succesful then you'll...   \n",
       "1  0000247867823ef7  == From RfC == \\n\\n The title is fine as it is...   \n",
       "2  00013b17ad220c46  \" \\n\\n == Sources == \\n\\n * Zawe Ashton on Lap...   \n",
       "\n",
       "      toxic  severe_toxic  obscene    threat    insult  identity_hate  \n",
       "0  0.068491      0.014765  0.00251  0.012214  0.014872       0.002175  \n",
       "1  0.068491      0.014765  0.00251  0.012214  0.014872       0.002175  \n",
       "2  0.068491      0.014765  0.00251  0.012214  0.014872       0.002175  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}