test_trec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 8051,
     "status": "ok",
     "timestamp": 1648761792279,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "thiGiJugAuoy"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchtext.legacy import data, datasets\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 391,
     "status": "ok",
     "timestamp": 1648761792667,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "iOgPHnQtAutO",
    "outputId": "0a1bd2a1-bcb4-4f12-b055-f27162d5f23e"
   },
   "outputs": [],
   "source": [
    "seed = 966\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 2435,
     "status": "ok",
     "timestamp": 1648761795099,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "dDVEivhqAuxA"
   },
   "outputs": [],
   "source": [
    "TEXT = data.Field(tokenize='spacy', lower=True)\n",
    "LABEL = data.LabelField()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1943,
     "status": "ok",
     "timestamp": 1648761797040,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "BxiaBGivAu3K",
    "outputId": "d901b255-6c14-47a7-d232-bfbcbe5fbea4"
   },
   "outputs": [],
   "source": [
    "train, test = datasets.TREC.splits(TEXT, LABEL)\n",
    "train, val = train.split(random_state = random.seed(seed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1648761797041,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "Kxky-ODaRh0U",
    "outputId": "d5ac26d8-a58d-4060-8ff5-91275ecb6dc2"
   },
   "outputs": [],
   "source": [
    "vars(train[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1648761797041,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "JoM5PVpl5Kra"
   },
   "outputs": [],
   "source": [
    "TEXT.build_vocab(train, min_freq=2)\n",
    "LABEL.build_vocab(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1648761797041,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "uKAd1xen2o5b",
    "outputId": "b6e86d3b-76be-48b5-d112-37e3e8efe007"
   },
   "outputs": [],
   "source": [
    "print(\"Vocabulary size of TEXT:\",len(TEXT.vocab.stoi))\n",
    "print(\"Vocabulary size of LABEL:\",len(LABEL.vocab.stoi))\n",
    "print(LABEL.vocab.stoi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1648761797042,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "zEiW7Ri9S-el"
   },
   "outputs": [],
   "source": [
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train, val, test),\n",
    "    batch_size = 64,\n",
    "    sort_key=lambda x: len(x.text), \n",
    "    device=device\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 211,
     "status": "ok",
     "timestamp": 1648761797247,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "sfXWPHz3S-hR"
   },
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "  def __init__(self, vocabulary_size, embedding_size, \n",
    "               kernels_number, kernel_sizes, output_size, dropout_rate):\n",
    "    super().__init__()\n",
    "    self.embedding = nn.Embedding(vocabulary_size, embedding_size)\n",
    "    self.convolution_layers = nn.ModuleList([nn.Conv2d(in_channels=1, out_channels=kernels_number, kernel_size=(k, embedding_size)) \n",
    "                                            for k in kernel_sizes])\n",
    "    self.dropout = nn.Dropout(dropout_rate)\n",
    "    self.fully_connected = nn.Linear(len(kernel_sizes) * kernels_number, output_size)\n",
    "  def forward(self, text):\n",
    "    text = text.permute(1, 0)\n",
    "    input_embeddings = self.embedding(text)\n",
    "    input_embeddings = input_embeddings.unsqueeze(1)\n",
    "    conved = [F.relu(convolution_layer(input_embeddings)).squeeze(3) for convolution_layer in self.convolution_layers]\n",
    "    pooled = [F.max_pool1d(conv, conv.shape[2]).squeeze(2) for conv in conved]\n",
    "    concat = self.dropout(torch.cat(pooled, dim=1))\n",
    "    final_output = self.fully_connected(concat)\n",
    "    return final_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1648761797248,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "pCURSmjUS-kV"
   },
   "outputs": [],
   "source": [
    "input_size = len(TEXT.vocab)\n",
    "embedding_size = 100\n",
    "kernels_number = 100\n",
    "kernel_sizes = [2, 3, 4]\n",
    "output_size = len(LABEL.vocab)\n",
    "dropout_rate = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1648761797248,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "XOcYKu_SmQvy"
   },
   "outputs": [],
   "source": [
    "model = CNN(input_size, embedding_size, kernels_number, kernel_sizes, output_size, dropout_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1648761797248,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "SMjhG8IOmhZf",
    "outputId": "662a55ce-768f-4768-81ee-25caf8651911"
   },
   "outputs": [],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 68254,
     "status": "ok",
     "timestamp": 1648761865499,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "aKcm7ZJQm9Oe",
    "outputId": "fbda9754-d089-4a3b-d956-1092b570db94"
   },
   "outputs": [],
   "source": [
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uMoLI86g0FA_"
   },
   "source": [
    "**Train and Evaluate Functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1648761865500,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "DUYzZfU9nBdP"
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "criterion = criterion.to(device)\n",
    "\n",
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1648761865501,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "Bk5boWJWnld4"
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, actual_label):\n",
    "    max_predictions = predictions.argmax(dim = 1, keepdim = True, )\n",
    "    correct_predictions = max_predictions.squeeze(1).eq(actual_label)\n",
    "    accuracy = correct_predictions.sum() / torch.cuda.FloatTensor([actual_label.shape[0]])\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1648761865501,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "ynHqY1ysoT57"
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "\n",
    "    model.train()\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    for batch in iterator:\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        predictions = model(batch.text)\n",
    "        \n",
    "        loss = criterion(predictions, batch.label)\n",
    "        \n",
    "        acc = accuracy(predictions, batch.label)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1648761865501,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "XoIQYPOJo164"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "\n",
    "    model.eval()\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            predictions = model(batch.text)\n",
    "            \n",
    "            loss = criterion(predictions, batch.label)\n",
    "            \n",
    "            acc = accuracy(predictions, batch.label)\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_CF2U8pZBq1t"
   },
   "source": [
    "**Training the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 39520,
     "status": "ok",
     "timestamp": 1648761952448,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "6Jd1eYGupW0x",
    "outputId": "ca13c6fe-59d4-41a6-a657-0e3323e7997b"
   },
   "outputs": [],
   "source": [
    "number_of_epochs = 20\n",
    "\n",
    "best_acc = float('-inf')\n",
    "\n",
    "for epoch in range(number_of_epochs):\n",
    "    \n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    if valid_acc > best_acc:\n",
    "        # Write the code here\n",
    "        best_acc = valid_acc\n",
    "        torch.save(model.state_dict(), 'trec.pt')\n",
    "    \n",
    "    print(f'Epoch {epoch+1} ')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Validation Loss: {valid_loss:.3f} |  Validation Acc: {valid_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 121,
     "status": "ok",
     "timestamp": 1648761958903,
     "user": {
      "displayName": "Zhongyu Pan",
      "userId": "00060245009473894752"
     },
     "user_tz": 420
    },
    "id": "mAcltNzgqgDb",
    "outputId": "b94b1cde-8498-40ce-9123-68653d68be0a"
   },
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('trec.pt'))\n",
    "\n",
    "test_loss, test_acc = evaluate(model, test_iterator, criterion)\n",
    "\n",
    "print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "id": "PcZeq-C33vBk"
   },
   "outputs": [],
   "source": ["**very old torchtext**"]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyNFDVxWzGY43ZcMIg/rj2ot",
   "collapsed_sections": [],
   "name": "03-05.ipynb",
   "provenance": [
    {
     "file_id": "1y2j-h25Qln4T01_mrfa_3nZoX_lS5Ka3",
     "timestamp": 1642736374693
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}