DL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import json\n",
    "import sklearn\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "from torcheval.metrics.functional import multiclass_accuracy,multiclass_f1_score,multiclass_confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 151836\n",
    "\n",
    "def setSeed(seed=seed):\n",
    "    \"\"\"\n",
    "    Setting the seed for reproducibility\n",
    "    \"\"\"\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "setSeed()\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class dataset(Dataset):\n",
    "    def __init__(self,x,y):\n",
    "        self.x = torch.from_numpy(x).type(torch.float32).to(device)\n",
    "        self.y = torch.from_numpy(y).type(torch.long).to(device)\n",
    "        self.length = self.x.shape[0]\n",
    "    def __getitem__(self, index):\n",
    "        return self.x[index],self.y[index]\n",
    "    def __len__(self):\n",
    "        return self.length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'dataset/finetuned.json'\n",
    "\n",
    "\n",
    "\n",
    "with open(path,'r',encoding='utf-8') as f:\n",
    "    input_dataset = json.load(f)\n",
    "\n",
    "labels = {'CODE':0, 'DATETIME':1, 'DEM':2, 'LOC':3, 'MISC':4, 'ORG':5, 'PERSON':6, 'QUANTITY':7}\n",
    "\n",
    "#we will need numerical encodings to use PyTorch\n",
    "\n",
    "X = np.array([element.get('embedding',None) for element in input_dataset])\n",
    "y = np.array([labels[element.get('label',None)] for element in input_dataset])\n",
    "#y = np.array([element.get('label',None) for element in dataset])\n",
    "\n",
    "setSeed()\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train,X_val,y_train,y_val = train_test_split(X_train,y_train,test_size= 0.125, random_state=42)\n",
    "print(len(X_train),len(X_test),len(X_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = dataset(X_train,y_train)\n",
    "test_dataset = dataset(X_test,y_test)\n",
    "val_dataset = dataset(X_val,y_val)\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset,shuffle=False,batch_size=100)\n",
    "val_dataloader = DataLoader(val_dataset,shuffle=False,batch_size=val_dataset.__len__())\n",
    "test_dataloader = DataLoader(train_dataset,shuffle=False,batch_size=test_dataset.__len__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'dataset/poisoned.json'\n",
    "\n",
    "with open(path,'r',encoding='utf-8') as f:\n",
    "    poisoned_dataset = json.load(f)\n",
    "\n",
    "labels = {'CODE':0, 'DATETIME':1, 'DEM':2, 'LOC':3, 'MISC':4, 'ORG':5, 'PERSON':6, 'QUANTITY':7}\n",
    "\n",
    "#we will need numerical encodings to use PyTorch\n",
    "\n",
    "X = np.array([element.get('embedding',None) for element in poisoned_dataset])\n",
    "y = np.array([labels[element.get('label',None)] for element in poisoned_dataset])\n",
    "#y = np.array([element.get('label',None) for element in dataset])\n",
    "\n",
    "setSeed()\n",
    "\n",
    "X_train,X_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train,X_val,y_train,y_val = train_test_split(X_train,y_train,test_size= 0.125, random_state=42)\n",
    "print(len(X_train),len(X_test),len(X_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "poisoned_test_dataset = dataset(X_test,y_test)\n",
    "poisoned_test_dataloader = DataLoader(poisoned_test_dataset,shuffle=False,batch_size=poisoned_test_dataset.__len__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(model,x,y_true):\n",
    "  with torch.inference_mode():\n",
    "    y_logits = model(x)\n",
    "  y_pred = torch.softmax(y_logits,dim=1).argmax(dim=1)\n",
    "  return multiclass_accuracy(y_pred,y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class redactBuster_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer_1 = nn.Linear(in_features=768,out_features=512)\n",
    "        self.layer_2 = nn.Linear(in_features=512,out_features=256)\n",
    "        self.layer_3 = nn.Linear(in_features=256,out_features=128)\n",
    "        self.layer_4 = nn.Linear(in_features=128,out_features=64)\n",
    "        self.layer_5 = nn.Linear(in_features=64,out_features=8)\n",
    "        self.ReLU = nn.ReLU()\n",
    "    \n",
    "    def forward(self,x):\n",
    "        h = self.layer_1(x)\n",
    "        h = self.ReLU(h)\n",
    "        h = self.layer_2(h)\n",
    "        h = self.ReLU(h)\n",
    "        h = self.layer_3(h)\n",
    "        h = self.ReLU(h)\n",
    "        h = self.layer_4(h)\n",
    "        h = self.ReLU(h)\n",
    "        h = self.layer_5(h)\n",
    "        return h "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setSeed()\n",
    "model = redactBuster_model().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(),lr=5e-5) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setSeed()\n",
    "epochs = 200\n",
    "train_losses = []\n",
    "train_accuracies = []\n",
    "val_losses = []\n",
    "val_accuracies = []\n",
    "for epoch in range(epochs):\n",
    "    batch_losses=[]  \n",
    "    batch_accuracies=[]  \n",
    "    for idx,(x_train,y_train) in enumerate(train_dataloader):\n",
    "        model.train()\n",
    "        y_logits = model(x_train).squeeze() #for ff\n",
    "        \n",
    "        loss = loss_fn(y_logits,y_train)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        batch_losses.append(loss.item())\n",
    "        batch_accuracies.append(compute_accuracy(model,x_train,y_train))\n",
    "        \n",
    "   \n",
    "    train_losses.append(torch.mean(torch.Tensor(batch_losses)).item())\n",
    "    train_accuracies.append(torch.mean(torch.Tensor(batch_accuracies)).item())\n",
    "\n",
    "    batch_losses=[]  \n",
    "    batch_accuracies=[]  \n",
    "    for idx,(x_val,y_val) in enumerate(val_dataloader):\n",
    "        model.eval()\n",
    "        y_logits = model(x_val).squeeze()   \n",
    "        \n",
    "        loss = loss_fn(y_logits,y_val)\n",
    "        batch_losses.append(loss.item())\n",
    "        batch_accuracies.append(compute_accuracy(model,x_train,y_train))\n",
    "        \n",
    "\n",
    "        \n",
    "    val_losses.append(torch.mean(torch.Tensor(batch_losses)).item())\n",
    "    val_accuracies.append(torch.mean(torch.Tensor(batch_accuracies)).item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,5))\n",
    "plt.plot(list(range(epochs)),train_accuracies)\n",
    "plt.plot(list(range(epochs)),val_accuracies)\n",
    "plt.legend(['train accuracy','validation accuracy'])\n",
    "print('final train accuracy:',train_accuracies[-1])\n",
    "print('final validation accuracy:',val_accuracies[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,(x_test,y_test) in enumerate(test_dataloader):\n",
    "    test_acc=compute_accuracy(model,x_test,y_test)\n",
    "    \n",
    "print('testing accuracy: ',test_acc.item())\n",
    "\n",
    "for idx,(x_test,y_test) in enumerate(test_dataloader):\n",
    "    with torch.inference_mode():\n",
    "        y_logits = model(x_test)\n",
    "    y_pred = torch.softmax(y_logits,dim=1).argmax(dim=1)\n",
    "    F1 = multiclass_f1_score(y_pred,y_test,num_classes=8)\n",
    "\n",
    "print('F1 score:',F1.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,(x_test,y_test) in enumerate(poisoned_test_dataloader):    \n",
    "    test_acc=compute_accuracy(model,x_test,y_test)\n",
    "        \n",
    "print('poisoned testing accuracy: ',test_acc.item())\n",
    "\n",
    "for idx,(x_test,y_test) in enumerate(poisoned_test_dataloader):\n",
    "    with torch.inference_mode():\n",
    "        y_logits = model(x_test)\n",
    "    y_pred = torch.softmax(y_logits,dim=1).argmax(dim=1)\n",
    "    F1 = multiclass_f1_score(y_pred,y_test,num_classes=8)\n",
    "\n",
    "print('poisoned F1 score:', F1.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class redactBuster_CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer_1 = nn.Conv1d(in_channels=1,out_channels=16,kernel_size=16)\n",
    "        self.pool = nn.MaxPool1d(kernel_size=8,stride=2) \n",
    "        self.layer_2 = nn.Conv1d(in_channels=16,out_channels=16,kernel_size=16,stride=2)\n",
    "       \n",
    "\n",
    "        self.layer_3 = nn.Linear(in_features=2*8*86,out_features=2*344)\n",
    "        self.layer_4 = nn.Linear(in_features=2*344,out_features=2*172)\n",
    "        self.layer_5 = nn.Linear(in_features=2*172,out_features=2*86)\n",
    "        self.layer_6 = nn.Linear(in_features=2*86,out_features=8)\n",
    "        self.ReLU = nn.ReLU()\n",
    "\n",
    "    \n",
    "    def forward(self,x):\n",
    "        x = self.pool(self.ReLU(self.layer_1(x)))\n",
    "        x = self.pool(self.ReLU(self.layer_2(x)))\n",
    "        #x = self.pool(self.ReLU(self.layer_3(x)))\n",
    "        x = torch.flatten(x,1)\n",
    "        x = self.ReLU(self.layer_3(x))\n",
    "        x = self.ReLU(self.layer_4(x))\n",
    "        x = self.ReLU(self.layer_5(x))\n",
    "        x = self.layer_6(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setSeed()\n",
    "model = redactBuster_CNN().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(),lr=1e-4) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setSeed()\n",
    "epochs = 200\n",
    "train_losses = []\n",
    "train_accuracies = []\n",
    "val_losses = []\n",
    "val_accuracies = []\n",
    "for epoch in range(epochs):\n",
    "    batch_losses=[]  \n",
    "    batch_accuracies=[]  \n",
    "    for idx,(x_train,y_train) in enumerate(train_dataloader):\n",
    "        model.train()\n",
    "        \n",
    "        y_logits = model(x_train.unsqueeze(1)).squeeze()         \n",
    "        loss = loss_fn(y_logits,y_train)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        batch_losses.append(loss.item())\n",
    "        \n",
    "        batch_accuracies.append(compute_accuracy(model,x_train.unsqueeze(1),y_train))\n",
    "   \n",
    "    train_losses.append(torch.mean(torch.Tensor(batch_losses)).item())\n",
    "    train_accuracies.append(torch.mean(torch.Tensor(batch_accuracies)).item())\n",
    "\n",
    "    batch_losses=[]  \n",
    "    batch_accuracies=[]  \n",
    "    for idx,(x_val,y_val) in enumerate(val_dataloader):\n",
    "        model.eval()\n",
    "        \n",
    "        y_logits = model(x_val.unsqueeze(1)).squeeze()        \n",
    "        loss = loss_fn(y_logits,y_val)\n",
    "        batch_losses.append(loss.item())\n",
    "        \n",
    "        batch_accuracies.append(compute_accuracy(model,x_train.unsqueeze(1),y_train))\n",
    "\n",
    "        \n",
    "    val_losses.append(torch.mean(torch.Tensor(batch_losses)).item())\n",
    "    val_accuracies.append(torch.mean(torch.Tensor(batch_accuracies)).item())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,5))\n",
    "plt.plot(list(range(epochs)),train_accuracies)\n",
    "plt.plot(list(range(epochs)),val_accuracies)\n",
    "plt.legend(['train accuracy','validation accuracy'])\n",
    "print('final train accuracy:',train_accuracies[-1])\n",
    "print('final validation accuracy:',val_accuracies[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,(x_test,y_test) in enumerate(test_dataloader):\n",
    "    test_acc=compute_accuracy(model,x_test.unsqueeze(1),y_test)\n",
    "    \n",
    "print('testing accuracy: ',test_acc.item())\n",
    "\n",
    "for idx,(x_test,y_test) in enumerate(test_dataloader):\n",
    "    with torch.inference_mode():\n",
    "        y_logits = model(x_test.unsqueeze(1))\n",
    "    y_pred = torch.softmax(y_logits,dim=1).argmax(dim=1)\n",
    "    F1 = multiclass_f1_score(y_pred,y_test,num_classes=8)\n",
    "\n",
    "print('F1 score:',F1.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,(x_test,y_test) in enumerate(poisoned_test_dataloader):    \n",
    "    test_acc=compute_accuracy(model,x_test.unsqueeze(1),y_test)\n",
    "        \n",
    "print('poisoned testing accuracy: ',test_acc.item())\n",
    "\n",
    "for idx,(x_test,y_test) in enumerate(poisoned_test_dataloader):\n",
    "    with torch.inference_mode():\n",
    "        y_logits = model(x_test.unsqueeze(1))\n",
    "    y_pred = torch.softmax(y_logits,dim=1).argmax(dim=1)\n",
    "    F1 = multiclass_f1_score(y_pred,y_test,num_classes=8)\n",
    "\n",
    "print('poisoned F1 score:', F1.item())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}