rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.autograd import Variable \n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import classification_report\n",
    "DATASET_PATH = 'output/train_1_arabert.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def edit_categories(x):\n",
    "    if x == 'info_news':\n",
    "        return 0\n",
    "    elif x == 'celebrity':\n",
    "        return 1\n",
    "    elif x == 'plan':\n",
    "        return 2\n",
    "    elif x == 'requests':\n",
    "        return 3\n",
    "    elif x == 'rumors':\n",
    "        return 4\n",
    "    elif x == 'advice':\n",
    "        return 5\n",
    "    elif x == 'restrictions':\n",
    "        return 6\n",
    "    elif x == 'personal':\n",
    "        return 7\n",
    "    elif x == 'unrelated':\n",
    "        return 8\n",
    "    elif x == 'others':\n",
    "        return 9\n",
    "    else:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# build the pytorch dataset\n",
    "class ArabertDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, dataset_path):\n",
    "        dataset = pd.read_pickle(dataset_path)\n",
    "        print(dataset['stance'].value_counts())\n",
    "        pad_embedding = torch.zeros(1, 768)\n",
    "        max_len = 0\n",
    "        embeddings = dataset['embeddings'].values\n",
    "        for i in range(len(embeddings)):\n",
    "            if embeddings[i].shape[0] > max_len:\n",
    "                max_len = embeddings[i].shape[0]\n",
    "        for i in range(len(embeddings)):\n",
    "            if embeddings[i].shape[0] < max_len:\n",
    "                pad = torch.zeros(max_len - embeddings[i].shape[0], 768).to(torch.device('cuda'))\n",
    "                embeddings[i] = torch.cat((embeddings[i], pad), dim=0)\n",
    "        categories = dataset['category'].apply(edit_categories)\n",
    "        self.embeddings = embeddings # already a tensor\n",
    "        self.stance = dataset['stance']\n",
    "        self.category = categories\n",
    "        self.stance = torch.tensor(self.stance.values)\n",
    "        self.category = torch.tensor(self.category.values)\n",
    "\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.stance)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.embeddings[idx], self.stance[idx], self.category[idx]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2    5538\n",
      "1    1012\n",
      "0     438\n",
      "Name: stance, dtype: int64\n",
      "2    804\n",
      "1    126\n",
      "0     70\n",
      "Name: stance, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# build the pytorch dataloader\n",
    "train_dataset = ArabertDataset(DATASET_PATH)\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# load the dev set\n",
    "dev_dataset = ArabertDataset('output/dev_1_arabert.pkl')\n",
    "dev_loader = torch.utils.data.DataLoader(dev_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# build the model\n",
    "class RNN(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super(RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.rnn = torch.nn.RNN(input_size, hidden_size, num_layers, batch_first=True)\n",
    "        self.fc = torch.nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
    "\n",
    "        out, _ = self.rnn(x, h0)\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 74.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 79.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 81.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 79.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 82.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 81.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       1.00      0.01      0.02       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.15      0.10      0.07      1000\n",
      "weighted avg       0.43      0.55      0.39      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 81.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 81.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 85.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 85.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 85.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       1.00      0.01      0.01       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.15      0.10      0.07      1000\n",
      "weighted avg       0.44      0.55      0.39      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 84.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.55      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.55      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 81.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      1.00      0.71       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 83.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 110/110 [00:01<00:00, 85.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      1.00      0.70       545\n",
      "           1       0.00      0.00      0.00       145\n",
      "           2       0.00      0.00      0.00        82\n",
      "           3       0.00      0.00      0.00        20\n",
      "           4       0.00      0.00      0.00        15\n",
      "           5       0.00      0.00      0.00        10\n",
      "           6       0.00      0.00      0.00         2\n",
      "           7       0.00      0.00      0.00       128\n",
      "           8       0.00      0.00      0.00        36\n",
      "           9       0.00      0.00      0.00        17\n",
      "\n",
      "    accuracy                           0.54      1000\n",
      "   macro avg       0.05      0.10      0.07      1000\n",
      "weighted avg       0.30      0.54      0.38      1000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# train the stance_model\n",
    "stance_model = RNN(768, 256, 1, 10).to(device)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(stance_model.parameters(), lr=1e-3)\n",
    "for epoch in range(20):\n",
    "    stance_model.train()\n",
    "    for i, (embedding, stance, category) in enumerate(tqdm(train_loader)):\n",
    "        embedding = embedding.to(device)\n",
    "        stance = stance.to(device)\n",
    "        category = category.to(device)\n",
    "\n",
    "        outputs = stance_model(embedding)\n",
    "        loss = criterion(outputs, category)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # calculate the total epoch accuracy\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total = stance.size(0)\n",
    "        correct = (predicted == category).sum().item()\n",
    "        accuracy = correct / total\n",
    "\n",
    "        if (i+1) % len(train_loader) == 0:\n",
    "            pass\n",
    "            # print(f'Epoch [{epoch+1}/{10}], Step [{i+1}/{len(train_loader)}], Loss: {loss.item():.4f}')\n",
    "            # print(f'Epoch [{epoch+1}/{10}], Step [{i+1}/{len(train_loader)}], Accuracy: {accuracy:.4f}')\n",
    "\n",
    "    # inference mode\n",
    "    stance_model.eval()\n",
    "    # get accuracy on development set\n",
    "    with torch.no_grad():\n",
    "        predicted = np.empty(len(dev_dataset), dtype=int)\n",
    "        stances = []\n",
    "        for i, (embedding, stance, category) in enumerate(dev_loader):\n",
    "            embedding = embedding.to(device)\n",
    "            stance = stance.to(device)\n",
    "            category = category.to(device)\n",
    "\n",
    "            outputs = stance_model(embedding)\n",
    "            predicted[i*64:(i+1)*64] = torch.argmax(outputs.data, 1).cpu().numpy()\n",
    "        stances = dev_dataset.stance\n",
    "        print(classification_report(dev_dataset.category, predicted, zero_division=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results\n",
    "# RNN + train_1_arabert.pkl --> 80.4% dev accuracy\n",
    "# RNN + train_2_arabert.pkl --> 71.0% dev accuracy\n",
    "# RNN + train_3_arabert.pkl --> 7.2% dev accuracy\n",
    "# RNN + train_4_arabert.pkl --> 28.9% dev accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # train the category_model\n",
    "# category_model = RNN(768, 512, 4, 10).to(device)\n",
    "# criterion = torch.nn.CrossEntropyLoss()\n",
    "# optimizer = torch.optim.Adam(category_model.parameters(), lr=1e-3)\n",
    "# category_model.train()\n",
    "# for epoch in range(10):\n",
    "#     for i, (embedding, stance, category) in enumerate(tqdm(train_loader)):\n",
    "#         embedding = embedding.to(device)\n",
    "#         stance = stance.to(device)\n",
    "#         category = category.to(device)\n",
    "\n",
    "#         outputs = category_model(embedding)\n",
    "#         loss = criterion(outputs, category)\n",
    "\n",
    "#         optimizer.zero_grad()\n",
    "#         loss.backward()\n",
    "#         optimizer.step()\n",
    "\n",
    "#         # calculate the total epoch accuracy\n",
    "#         _, predicted = torch.max(outputs.data, 1)\n",
    "#         total = category.size(0)\n",
    "#         correct = (predicted == category).sum().item()\n",
    "#         accuracy = correct / total\n",
    "\n",
    "#         if (i+1) % len(train_loader) == 0:\n",
    "#             print(f'Epoch [{epoch+1}/{10}], Step [{i+1}/{len(train_loader)}], Loss: {loss.item():.4f}')\n",
    "#             print(f'Epoch [{epoch+1}/{10}], Step [{i+1}/{len(train_loader)}], Accuracy: {accuracy:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # inference mode\n",
    "# category_model.eval()\n",
    "\n",
    "# # load the dev set\n",
    "# dev_dataset = ArabertDataset('output/dev_1_arabert.pkl')\n",
    "# dev_loader = torch.utils.data.DataLoader(dev_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# # get accuracy on development set\n",
    "# with torch.no_grad():\n",
    "#     correct = 0\n",
    "#     total = 0\n",
    "#     for embedding, stance, category in dev_loader:\n",
    "#         embedding = embedding.to(device)\n",
    "#         stance = stance.to(device)\n",
    "#         category = category.to(device)\n",
    "\n",
    "#         outputs = category_model(embedding)\n",
    "#         _, predicted = torch.max(outputs.data, 1)\n",
    "#         total += category.size(0)\n",
    "#         correct += (predicted == category).sum().item()\n",
    "#     print(f'Accuracy of the model on the dev set: {100 * correct / total}%')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results\n",
    "# RNN + train_1_arabert.pkl --> 54.5% dev accuracy\n",
    "# RNN + train_2_arabert.pkl --> 54.4% dev accuracy\n",
    "# RNN + train_3_arabert.pkl --> 7.2% dev accuracy\n",
    "# RNN + train_4_arabert.pkl --> 54.5% dev accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5 (default, Sep  4 2020, 07:30:14) \n[GCC 7.3.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "0eefc6a63c6d719296cee5685f23fefb92aa63e2e9fdaf52ddbdc4ce266c7bb3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}