Roger - BERT - Text Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:16.899967900Z",
     "start_time": "2023-06-04T21:35:13.942677800Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Roger\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import metrics\n",
    "import torch\n",
    "import transformers\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:16.922849200Z",
     "start_time": "2023-06-04T21:35:16.900973100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "                                        comment_text                list\n0  Explanation\\nWhy the edits made under my usern...  [0, 0, 0, 0, 0, 0]\n1  D'aww! He matches this background colour I'm s...  [0, 0, 0, 0, 0, 0]\n2  Hey man, I'm really not trying to edit war. It...  [0, 0, 0, 0, 0, 0]\n3  \"\\nMore\\nI can't make any real suggestions on ...  [0, 0, 0, 0, 0, 0]\n4  You, sir, are my hero. Any chance you remember...  [0, 0, 0, 0, 0, 0]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>comment_text</th>\n      <th>list</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Explanation\\nWhy the edits made under my usern...</td>\n      <td>[0, 0, 0, 0, 0, 0]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>D'aww! He matches this background colour I'm s...</td>\n      <td>[0, 0, 0, 0, 0, 0]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Hey man, I'm really not trying to edit war. It...</td>\n      <td>[0, 0, 0, 0, 0, 0]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n      <td>[0, 0, 0, 0, 0, 0]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>You, sir, are my hero. Any chance you remember...</td>\n      <td>[0, 0, 0, 0, 0, 0]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"./resources/jigsaw-toxic-comment-classification-challenge/train.csv/train.csv\")\n",
    "df['list'] = df[df.columns[2:]].values.tolist()\n",
    "# cols = df.column_names\n",
    "# df = df.map(lambda x : {\"labels\": [x[c] for c in cols if c != \"comment_text\"]})\n",
    "new_df = df[['comment_text', 'list']].copy()\n",
    "new_df.head()\n",
    "# id2label = {0: \"toxic\", 1: \"severe_toxic\", 2: \"obscene\", 3: \"threat\", 4: \"insult\", 5: \"identity_hate\"}\n",
    "# label2id = {\"toxic\": 0, \"severe_toxic\": 1, \"obscene\": 2, \"threat\": 3, \"insult\": 4, \"identity_hate\": 5}"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:17.853077600Z",
     "start_time": "2023-06-04T21:35:16.922849200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "MAX_LEN = 200\n",
    "TRAIN_BATCH_SIZE = 8\n",
    "VALID_BATCH_SIZE = 4\n",
    "EPOCHS = 1\n",
    "LEARNING_RATE = 1e-05\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', problem_type=\"multi_label_classification\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:18.027438400Z",
     "start_time": "2023-06-04T21:35:17.853077600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "class CustomDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.comment_text = dataframe.comment_text\n",
    "        self.targets = self.data.list\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.comment_text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        comment_text = str(self.comment_text[index])\n",
    "        comment_text = \" \".join(comment_text.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            comment_text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            pad_to_max_length=True,\n",
    "            return_token_type_ids=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:18.033141600Z",
     "start_time": "2023-06-04T21:35:18.031629900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL Dataset: (159571, 2)\n",
      "TRAIN Dataset: (127657, 2)\n",
      "TEST Dataset: (31914, 2)\n"
     ]
    }
   ],
   "source": [
    "train_size = 0.8\n",
    "train_dataset=new_df.sample(frac=train_size,random_state=200)\n",
    "test_dataset=new_df.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "\n",
    "print(\"FULL Dataset: {}\".format(new_df.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(test_dataset.shape))\n",
    "\n",
    "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "testing_set = CustomDataset(test_dataset, tokenizer, MAX_LEN)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:18.071139Z",
     "start_time": "2023-06-04T21:35:18.035143200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "testing_loader = DataLoader(testing_set, **test_params)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:18.072139800Z",
     "start_time": "2023-06-04T21:35:18.064249600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": "BERTClass(\n  (l1): BertModel(\n    (embeddings): BertEmbeddings(\n      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n      (position_embeddings): Embedding(512, 768)\n      (token_type_embeddings): Embedding(2, 768)\n      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n      (dropout): Dropout(p=0.1, inplace=False)\n    )\n    (encoder): BertEncoder(\n      (layer): ModuleList(\n        (0-11): 12 x BertLayer(\n          (attention): BertAttention(\n            (self): BertSelfAttention(\n              (query): Linear(in_features=768, out_features=768, bias=True)\n              (key): Linear(in_features=768, out_features=768, bias=True)\n              (value): Linear(in_features=768, out_features=768, bias=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n            (output): BertSelfOutput(\n              (dense): Linear(in_features=768, out_features=768, bias=True)\n              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n              (dropout): Dropout(p=0.1, inplace=False)\n            )\n          )\n          (intermediate): BertIntermediate(\n            (dense): Linear(in_features=768, out_features=3072, bias=True)\n            (intermediate_act_fn): GELUActivation()\n          )\n          (output): BertOutput(\n            (dense): Linear(in_features=3072, out_features=768, bias=True)\n            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (pooler): BertPooler(\n      (dense): Linear(in_features=768, out_features=768, bias=True)\n      (activation): Tanh()\n    )\n  )\n  (l2): Linear(in_features=768, out_features=6, bias=True)\n)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating the customized model, by adding a drop out and a dense layer on top of distil bert to get the final output for the model.\n",
    "\n",
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.l2 = torch.nn.Linear(768, 6)\n",
    "\n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids, return_dict = False)\n",
    "        output_2 = self.l2(output_1)\n",
    "        # output = self.l3(output_2)\n",
    "        return output_2\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:19.278376300Z",
     "start_time": "2023-06-04T21:35:18.070138900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:19.285424600Z",
     "start_time": "2023-06-04T21:35:19.279377700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:19.310029Z",
     "start_time": "2023-06-04T21:35:19.283426600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    model.train()\n",
    "    for _,data in enumerate(training_loader, 0):\n",
    "        ids = data['ids'].to(device, dtype = torch.long)\n",
    "        mask = data['mask'].to(device, dtype = torch.long)\n",
    "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "        targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "        outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss = loss_fn(outputs, targets)\n",
    "        if _%5000==0:\n",
    "            print(f'Epoch: {epoch}, Loss:  {loss.item()}')\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T21:35:19.341608Z",
     "start_time": "2023-06-04T21:35:19.312027200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "C:\\Users\\Roger\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\transformers\\tokenization_utils_base.py:2364: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss:  0.7792837619781494\n",
      "Epoch: 0, Loss:  0.07940429449081421\n",
      "Epoch: 0, Loss:  0.03864573687314987\n",
      "Epoch: 0, Loss:  0.03751469776034355\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    train(epoch)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T22:33:07.161631500Z",
     "start_time": "2023-06-04T21:35:19.315611500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "def validation(epoch):\n",
    "    model.eval()\n",
    "    fin_targets=[]\n",
    "    fin_outputs=[]\n",
    "    with torch.no_grad():\n",
    "        for _, data in enumerate(testing_loader, 0):\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['targets'].to(device, dtype = torch.float)\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "            fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "            fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "    return fin_outputs, fin_targets"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T22:33:07.177693Z",
     "start_time": "2023-06-04T22:33:07.174624Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score = 0.9174970232499843\n",
      "F1 Score (Micro) = 0.7785252432891385\n",
      "F1 Score (Macro) = 0.633087549404111\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    outputs, targets = validation(epoch)\n",
    "    outputs = np.array(outputs) >= 0.5\n",
    "    accuracy = metrics.accuracy_score(targets, outputs)\n",
    "    f1_score_micro = metrics.f1_score(targets, outputs, average='micro')\n",
    "    f1_score_macro = metrics.f1_score(targets, outputs, average='macro')\n",
    "    print(f\"Accuracy Score = {accuracy}\")\n",
    "    print(f\"F1 Score (Micro) = {f1_score_micro}\")\n",
    "    print(f\"F1 Score (Macro) = {f1_score_macro}\")\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-06-04T22:38:21.134696700Z",
     "start_time": "2023-06-04T22:33:07.185695Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}