CNN Multilabel Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example of a Convolutional Neural Network (Multilabel) for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Imports needed from pytorch\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "#Some built-in imports\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pickle\n",
    "from os.path import join, isfile\n",
    "from os import listdir\n",
    "\n",
    "#Imports from the repository\n",
    "from data_processing import get_weight_matrix, get_tokens\n",
    "import data_processing as dp\n",
    "from privacy_policies_dataset import PrivacyPoliciesDataset as PPD\n",
    "from cnn import CNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If this is not the first time you run the code and you are using the same embeddings and the same dataset then you can skip several cells and just run cell 5 to load the weights matrix, cell 7 to load labels and cell 10 to load datasets. Once you have run these cells you can jump straight into section 3 and run the CNN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confusion_matrices(model, dataset, threshold):\n",
    "    \n",
    "    metrics = {}\n",
    "    x = PPD.collate_data(dataset)[0]\n",
    "    y_hat  = model(x)\n",
    "    \"\"\"ge = greater equal than threshold\"\"\"\n",
    "    y_hat = torch.ge(y_hat, threshold).double() #predicted\n",
    "    y = dataset.labels_tensor.double()  #actual\n",
    "    tp = (y * y_hat).sum(0).numpy()\n",
    "    tn = ((1 - y) * (1 - y_hat)).sum(0).numpy()\n",
    "    fp = (y_hat * (1 - y)).sum(0).numpy()\n",
    "    fn = ((1 - y_hat) * y).sum(0).numpy()\n",
    "    metrics['TP'] = tp\n",
    "    metrics['TN'] = tn\n",
    "    metrics['FP'] = fp\n",
    "    metrics['FN'] = fn\n",
    "    \n",
    "    return metrics\n",
    "\n",
    "def confusion_matrices_change(metrics_05, metrics_best_t):\n",
    "    \n",
    "    labels = range(12)\n",
    "    fig = plt.figure(figsize=(15,10))\n",
    "    fig.suptitle(\"metrics differences\")\n",
    "    tp_ax = fig.add_subplot(221)\n",
    "    tn_ax = fig.add_subplot(222)\n",
    "    fp_ax = fig.add_subplot(223)\n",
    "    fn_ax = fig.add_subplot(224)\n",
    "    \n",
    "    tp_ax.plot(labels, metrics_05['TP'], label = 't = 0.5')\n",
    "    tn_ax.plot(labels, metrics_05['TN'], label = 't = 0.5')\n",
    "    fp_ax.plot(labels, metrics_05['FP'], label = 't = 0.5')\n",
    "    fn_ax.plot(labels, metrics_05['FN'], label = 't = 0.5')\n",
    "    \n",
    "    tp_ax.set_ylabel('TP')\n",
    "    tn_ax.set_ylabel('TN')\n",
    "    fp_ax.set_ylabel('FP')\n",
    "    fn_ax.set_ylabel('FN')\n",
    "    \n",
    "    tp_ax.plot(labels, metrics_best_t['TP'], label = 'best t')\n",
    "    tn_ax.plot(labels, metrics_best_t['TN'], label = 'best t')\n",
    "    fp_ax.plot(labels, metrics_best_t['FP'], label = 'best t')\n",
    "    fn_ax.plot(labels, metrics_best_t['FN'], label = 'best t')\n",
    "    \n",
    "    tp_ax.legend()\n",
    "    tn_ax.legend()\n",
    "    fp_ax.legend()\n",
    "    fn_ax.legend()\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Generating word embeddings matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We read from raw_data all the files and get all the different words we can find within all the files. If we already have a file named dictionary.pkl and read set to True, it will read the dictionary from this file. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from file dictionary.pkl\n"
     ]
    }
   ],
   "source": [
    "dictionary = get_tokens(\"raw_data\", \"embeddings_data\", read = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is to load the pretrained embeddings. We will get two python dictionaries. Both have the words as the keys of the python dictionaries and one has the vectors as the keys whilst the other one has the position on the dictionary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from file embeddings_data\\word2vector_fast_text_300.pkl\n"
     ]
    }
   ],
   "source": [
    "fast_text_model = 'data/fastText-0.1.0/corpus_vectors_default_300d'\n",
    "\n",
    "word2vector_fast_text = dp.get_fast_text_dicts(fast_text_model, \"embeddings_data\", 300, missing = False, read = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading from file word2vector_globe_300.pkl\n",
      "Number of words in the pretrained embeddings: 400001\n"
     ]
    }
   ],
   "source": [
    "word2vector_glove = dp.get_glove_dicts( \"data\",  \"glove.6B\", \"embeddings_data\", 300, read = True)\n",
    "\n",
    "print(\"Number of words in the pretrained embeddings: {}\".format(len(word2vector_glove)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we compute the matrix containing all the word embeddings that we will need for the embedding layer of the CNN and \n",
    "we obtain a word2idx of just all the words inside dictionary and not all the words present in the word embeddings. Usually the pretrained embeddings that we will use have more words than what we need, that is the reason why we need to obtain a new word2idx of just all the words that we found in the files inside train and test folders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`get_weigth_matrix()` variables:\n",
    "1. dimensions\n",
    "2. folder\n",
    "3. read\n",
    "4. oov_random\n",
    "5. kwargs: dictionary, word2vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset ...\n",
      "Some words were missing in the word2vector. 1119 words were not found.\n"
     ]
    }
   ],
   "source": [
    "#weights_matrix = get_weight_matrix(300, \"embeddings_data\", read = True)\n",
    "#weights_matrix = get_weight_matrix(300, \"embeddings_data\", oov_random = False, dictionary = dictionary, word2vector = word2vector_glove)\n",
    "weights_matrix = get_weight_matrix(300, \"embeddings_data\", oov_random = False, dictionary = dictionary, word2vector = word2vector_fast_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Creation of Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first step before obtaining the prefectly cleaned data that will be used in the CNN is to aggregate the labels. The raw_data folder provides a series of files in csv format with repeated sentences. The reason behind this is that some sentences have several labels assigned to them. The last step is to aggregate segments and obtain a list of labels per sentence. The following function gets all the data from raw_data folder and outputs the result in agg_data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset in one file ...\n"
     ]
    }
   ],
   "source": [
    "labels = OrderedDict([('First Party Collection/Use', 0),\n",
    "             ('Third Party Sharing/Collection', 1),\n",
    "             ('User Access, Edit and Deletion', 2),\n",
    "             ('Data Retention', 3),\n",
    "             ('Data Security', 4),\n",
    "             ('International and Specific Audiences', 5),\n",
    "             ('Do Not Track', 6),\n",
    "             ('Policy Change', 7),\n",
    "             ('User Choice/Control', 8),\n",
    "             ('Introductory/Generic', 9),\n",
    "             ('Practice not covered', 10),\n",
    "             ('Privacy contact information', 11)])\n",
    "\n",
    "with open(\"labels.pkl\", \"wb\") as f:\n",
    "    pickle.dump(labels, f)\n",
    "\n",
    "\n",
    "dp.aggregate_data(read = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the aggregated data in agg_data we will process all the sentences and transform them into a list of integers. The integers will refer to the position inside the word2idx dictionary. The labels will also be transformed into an n-dimensinal vector with 1s if a sentence has that label and 0s if it doesn't. All the data will be placed in the corresponding folder inside processed_data. We load the labels with which we want to perform the classification. We will also show them so that it is clearer to the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0. First Party Collection/Use\n",
      "1. Third Party Sharing/Collection\n",
      "2. User Access, Edit and Deletion\n",
      "3. Data Retention\n",
      "4. Data Security\n",
      "5. International and Specific Audiences\n",
      "6. Do Not Track\n",
      "7. Policy Change\n",
      "8. User Choice/Control\n",
      "9. Introductory/Generic\n",
      "10. Practice not covered\n",
      "11. Privacy contact information\n"
     ]
    }
   ],
   "source": [
    "labels_file = open(\"labels.pkl\",\"rb\")\n",
    "\n",
    "labels = pickle.load(labels_file)\n",
    "\n",
    "labels_file.close()\n",
    "\n",
    "for label, index in labels.items():\n",
    "    \n",
    "    print(str(index) + '. ' + label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset ...\n"
     ]
    }
   ],
   "source": [
    "sentence_matrices_all, labels_matrices_all = dp.process_dataset(labels, dictionary, read = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now create an PPD which stands for PrivacyPoliciesDataset containing the training and testing dataset. We will need to split the data in two to get the test training data and the data that will be used for training and validation. The function split_dataset_randomly is spliting the dataset 90/10 by default."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = PPD(sentence_matrices_all, labels_matrices_all, labels)\n",
    "\n",
    "# test_dataset, train_validation_dataset = dataset.split_dataset_randomly(0.2)\n",
    "# \n",
    "# validation_dataset, train_dataset = train_validation_dataset.split_dataset_randomly(0.2)\n",
    "# \n",
    "# test_dataset.pickle_dataset(\"datasets/test_dataset_label6.pkl\")\n",
    "# \n",
    "# train_dataset.pickle_dataset(\"datasets/train_dataset_label6.pkl\")\n",
    "# \n",
    "# validation_dataset.pickle_dataset(\"datasets/validation_dataset_label6.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In case we aready had all the data split and prepared we can load it like this: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = PPD.unpickle_dataset(\"datasets/test_dataset_label6.pkl\")\n",
    "\n",
    "train_dataset = PPD.unpickle_dataset(\"datasets/train_dataset_label6.pkl\")\n",
    "\n",
    "validation_dataset = PPD.unpickle_dataset(\"datasets/validation_dataset_label6.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of segments: 2425\n",
      "Num of labels: 4129\n",
      "Percentages with respect to number of segments ... \n",
      "0. First Party Collection/Use : 991 (40.865979381443296%)\n",
      "1. Third Party Sharing/Collection : 773 (31.876288659793815%)\n",
      "2. User Access, Edit and Deletion : 132 (5.443298969072165%)\n",
      "3. Data Retention : 107 (4.412371134020619%)\n",
      "4. Data Security : 237 (9.77319587628866%)\n",
      "5. International and Specific Audiences : 229 (9.443298969072165%)\n",
      "6. Do Not Track : 19 (0.7835051546391752%)\n",
      "7. Policy Change : 114 (4.701030927835052%)\n",
      "8. User Choice/Control : 382 (15.75257731958763%)\n",
      "9. Introductory/Generic : 526 (21.690721649484537%)\n",
      "10. Practice not covered : 423 (17.443298969072163%)\n",
      "11. Privacy contact information : 196 (8.082474226804123%)\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "Num of segments: 606\n",
      "Num of labels: 1040\n",
      "Percentages with respect to number of segments ... \n",
      "0. First Party Collection/Use : 244 (40.26402640264026%)\n",
      "1. Third Party Sharing/Collection : 186 (30.693069306930692%)\n",
      "2. User Access, Edit and Deletion : 45 (7.425742574257426%)\n",
      "3. Data Retention : 26 (4.29042904290429%)\n",
      "4. Data Security : 57 (9.405940594059405%)\n",
      "5. International and Specific Audiences : 51 (8.415841584158416%)\n",
      "6. Do Not Track : 4 (0.6600660066006601%)\n",
      "7. Policy Change : 34 (5.6105610561056105%)\n",
      "8. User Choice/Control : 122 (20.13201320132013%)\n",
      "9. Introductory/Generic : 119 (19.636963696369637%)\n",
      "10. Practice not covered : 100 (16.501650165016503%)\n",
      "11. Privacy contact information : 52 (8.58085808580858%)\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "Num of segments: 757\n",
      "Num of labels: 1284\n",
      "Percentages with respect to number of segments ... \n",
      "0. First Party Collection/Use : 284 (37.51651254953765%)\n",
      "1. Third Party Sharing/Collection : 227 (29.98678996036988%)\n",
      "2. User Access, Edit and Deletion : 53 (7.001321003963012%)\n",
      "3. Data Retention : 23 (3.038309114927345%)\n",
      "4. Data Security : 81 (10.700132100396301%)\n",
      "5. International and Specific Audiences : 73 (9.64332892998679%)\n",
      "6. Do Not Track : 9 (1.1889035667107002%)\n",
      "7. Policy Change : 44 (5.812417437252312%)\n",
      "8. User Choice/Control : 128 (16.90885072655218%)\n",
      "9. Introductory/Generic : 168 (22.192866578599737%)\n",
      "10. Practice not covered : 119 (15.71994715984148%)\n",
      "11. Privacy contact information : 75 (9.907529722589167%)\n",
      "---------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_dataset.labels_stats()\n",
    "print(\"-\" * 35 * 3)\n",
    "validation_dataset.labels_stats()\n",
    "print(\"-\" * 35 * 3)\n",
    "test_dataset.labels_stats()\n",
    "print(\"-\" * 35 * 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Creation of CNN and training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we set the 7 main parameters of the CNN we are using:\n",
    "1. Number of words in the dictionary\n",
    "2. Embeddings dimension\n",
    "3. Number of filters per kernel size\n",
    "4. Number of hidden units\n",
    "5. Number of labels to classify\n",
    "6. List of all the kernels sizes we want to use\n",
    "7. Name of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2425, 12])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset.labels_tensor.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6842, 300)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CNN(weights_matrix.shape[0], weights_matrix.shape[1], 200, [100], 12, [3], name = 'zeros_60-20-20_polisis')\n",
    "\n",
    "model.load_pretrained_embeddings(weights_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "We will also add the pretrained embeddings to the embedding layer of the CNN through load_pretrained_embeddings. The function called train_cn will need two more parameters:\n",
    "1. number of epochs \n",
    "2. learning rate\n",
    "3. momentum constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "last epoch finished: 299 -- progress: 100.0% -- time: 2.535688518100173 mins28873917 minss\n",
      "Training completed. Total training time: 69.27 mins\n"
     ]
    }
   ],
   "source": [
    "results = model.train_CNN(train_dataset, validation_dataset, lr = 0.01, epochs_num = 300, alpha = 0, momentum = 0.9)\n",
    "\n",
    "epochs, train_losses, validation_losses, f1_scores_validations, precisions_validations, recalls_validations = results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(epochs, train_losses, label = \"train\")\n",
    "plt.plot(epochs, validation_losses, label = \"validation\")\n",
    "plt.legend()\n",
    "plt.title(\"loss vs epoch\")\n",
    "plt.savefig(join(\"trained_models_pics\", model.cnn_name + '_loss.png'), format = 'png')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.plot(epochs, f1_scores_validations, label = \"validation\")\n",
    "plt.legend()\n",
    "plt.title(\"f1 vs epoch\")\n",
    "#plt.savefig(join(\"trained_models_pics\", model.cnn_name + '_loss.png'), format = 'png')\n",
    "plt.show()\n",
    "\n",
    "dict_path = join(\"trained_models\", model.cnn_name + \"_state.pt\")\n",
    "torch.save(model.state_dict(), dict_path)\n",
    "model.save_cnn_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We save all the parameters used in the CNN (weights of all the layers and the configurations of the CNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Evaluation of the CNN results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We extract the labels true labels from the training and testing data sets and predict the labels using both labels. The predictions are usually refered as y_hat. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "\n",
    "y_train = train_dataset.labels_tensor\n",
    "\n",
    "y_validation = validation_dataset.labels_tensor\n",
    "\n",
    "x_train = PPD.collate_data(train_dataset)[0]\n",
    "\n",
    "x_validation = PPD.collate_data(validation_dataset)[0]\n",
    "\n",
    "y_hat_train = model(x_train)\n",
    "\n",
    "y_hat_validation = model(x_validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now show how the F1 score changes for all possible thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4129.0 Labels T\n",
      "2425 Segments T\n",
      "1040.0 Labels V\n",
      "606 Segments V\n",
      "Scores with 0.5 threshold\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "f1 micro        |0.6984631419181824\n",
      "precision micro |0.7780401706695557\n",
      "recall micro    |0.6336538195610046\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "f1 macro        |0.5757095217704773\n",
      "precision macro |0.7288181185722351\n",
      "recall macro    |0.5022012591362\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "\n",
      "Score per label with 0.5 threshold\n",
      "---------------------------------------------------------------------------------------------------------\n",
      "Label                                           F1        Precision Recall    Count T.  Count V.  \n",
      "---------------------------------------------------------------------------------------------------------\n",
      "First Party Collection/Use                      0.81      0.81      0.8       0.41      0.41      \n",
      "Third Party Sharing/Collection                  0.82      0.78      0.85      0.32      0.31      \n",
      "User Access, Edit and Deletion                  0.45      0.78      0.32      0.06      0.08      \n",
      "Data Retention                                  0.32      0.84      0.2       0.05      0.05      \n",
      "Data Security                                   0.66      0.82      0.55      0.1       0.1       \n",
      "International and Specific Audiences            0.91      1.0       0.83      0.1       0.09      \n",
      "Do Not Track                                    0.0       0.0       0.0       0.01      0.01      \n",
      "Policy Change                                   0.58      0.84      0.45      0.05      0.06      \n",
      "User Choice/Control                             0.64      0.7       0.6       0.16      0.21      \n",
      "Introductory/Generic                            0.65      0.84      0.53      0.22      0.2       \n",
      "Practice not covered                            0.4       0.5       0.32      0.18      0.17      \n",
      "Privacy contact information                     0.74      0.87      0.64      0.09      0.09      \n"
     ]
    }
   ],
   "source": [
    "model.print_results(train_dataset, validation_dataset, 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the following block of code we will find the threshold that with which we obtain the best overall F1 score. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_f1_score, best_threshold_label = CNN.get_best_thresholds(y_validation, y_hat_validation, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label                                           F1        Precision Recall    \n",
      "---------------------------------------------------------------------------------------------------------\n",
      "First Party Collection/Use                      0.8       0.86      0.74      \n",
      "Third Party Sharing/Collection                  0.83      0.84      0.82      \n",
      "User Access, Edit and Deletion                  0.61      0.83      0.48      \n",
      "Data Retention                                  0.56      0.57      0.55      \n",
      "Data Security                                   0.66      0.91      0.52      \n",
      "International and Specific Audiences            0.94      0.92      0.96      \n",
      "Do Not Track                                    0.03      0.02      0.67      \n",
      "Policy Change                                   0.77      0.77      0.77      \n",
      "User Choice/Control                             0.66      0.65      0.67      \n",
      "Introductory/Generic                            0.65      0.64      0.66      \n",
      "Practice not covered                            0.5       0.41      0.63      \n",
      "Privacy contact information                     0.84      0.83      0.85      \n",
      "macro averages\n",
      "F1: 0.6484361290931702\n",
      "Precision: 0.6821401715278625\n",
      "Recall: 0.689123809337616\n"
     ]
    }
   ],
   "source": [
    "model.print_results_best_t(validation_dataset, torch.tensor(best_threshold_label).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6484361290931702, 0.6821401715278625, 0.689123809337616)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.f1_score(y_validation, y_hat_validation, torch.tensor(best_threshold_label).float(), macro=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6766641736030579, 0.6401383876800537, 0.7176105380058289)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.f1_score(y_validation, y_hat_validation, torch.tensor(best_threshold_label).float(), macro=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_05 = confusion_matrices(model, validation_dataset, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_best_t = confusion_matrices(model, validation_dataset, torch.tensor(best_threshold_label).float())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "confusion_matrices_change(metrics_05, metrics_best_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we show the results for the best combination of thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1        |tensor([0.7878, 0.8087, 0.4776, 0.0625, 0.6393, 0.8671, 0.0000, 0.7500, 0.6404,\n",
      "        0.6176, 0.4456, 0.8189])\n",
      "precision |tensor([0.8028, 0.8774, 0.9412, 1.0000, 0.8125, 1.0000, 0.0000, 0.9231, 0.7647,\n",
      "        0.6885, 0.6515, 0.8966])\n",
      "recall    |tensor([0.7733, 0.7500, 0.3200, 0.0323, 0.5270, 0.7654, 0.0000, 0.6316, 0.5508,\n",
      "        0.5600, 0.3386, 0.7536])\n"
     ]
    }
   ],
   "source": [
    "# We show the F1, precision and recall for the best threshold\n",
    "#f1, precision, recall = CNN.f1_score(y_validation, y_hat_validation, torch.tensor(best_t_label).float())\n",
    "f1, precision, recall = CNN.f1_score_per_label(y_validation, y_hat_validation, 0.5)\n",
    "\n",
    "print(\"f1        |\" + str(f1))\n",
    "\n",
    "print(\"precision |\" + str(precision))\n",
    "\n",
    "print(\"recall    |\" + str(recall))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We show a list of all the possible labels to remind the user which ones are available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index 0. First Party Collection/Use.\n",
      "index 1. Third Party Sharing/Collection.\n",
      "index 2. User Access, Edit and Deletion.\n",
      "index 3. Data Retention.\n",
      "index 4. Data Security.\n",
      "index 5. International and Specific Audiences.\n",
      "index 6. Do Not Track.\n",
      "index 7. Policy Change.\n",
      "index 8. User Choice/Control.\n",
      "index 9. Introductory/Generic.\n",
      "index 10. Practice not covered.\n",
      "index 11. Privacy contact information.\n"
     ]
    }
   ],
   "source": [
    "for label, i in labels.items():\n",
    "    \n",
    "    print(\"index {}. {}.\".format(i, label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also show how the F1 score changes for all possible thresholds in just one label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label: User Choice/Control\n",
      "\n",
      "f1_label        |0.6403940916061401\n",
      "precision_label |0.7647058963775635\n",
      "recall_label    |0.5508474707603455\n"
     ]
    }
   ],
   "source": [
    "threshold_list = np.arange(0.0, 1, 0.01)\n",
    "\n",
    "label = 'User Choice/Control'\n",
    "\n",
    "f1_scores_per_label = [CNN.f1_score_per_label(y_validation, y_hat_validation, t)[0][labels[label]].item() for t in threshold_list]\n",
    "\n",
    "plt.plot(threshold_list, f1_scores_per_label)\n",
    "\n",
    "plt.title(label + \" f1 score\" + \" vs threshold\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "f1_label, precision_label, recall_label = CNN.f1_score_per_label(y_validation, y_hat_validation, 0.5)\n",
    "\n",
    "f1_label = f1_label[labels[label]].item()\n",
    "\n",
    "precision_label = precision_label[labels[label]].item()\n",
    "\n",
    "recall_label = recall_label[labels[label]].item()\n",
    "\n",
    "print(\"Label: \" + label + \"\\n\")\n",
    "\n",
    "print(\"f1_label        |\" + str(f1_label))\n",
    "\n",
    "print(\"precision_label |\" + str(precision_label))\n",
    "\n",
    "print(\"recall_label    |\" + str(recall_label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Comparison between models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = 'trained_models/Thresholds'\n",
    "\n",
    "files = [join(folder, f) for f in listdir(folder) if isfile(join(folder, f)) and '.pt' in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_params.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-e0923db5daaf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparams_model1_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_params.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mparams_model1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams_model1_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_params.pkl'"
     ]
    }
   ],
   "source": [
    "params_model1_file = open('trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_params.pkl', 'rb')\n",
    "\n",
    "params_model1 = pickle.load(params_model1_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'params_model1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-6c7e10053adc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams_model1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_state.pt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmodel1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'params_model1' is not defined"
     ]
    }
   ],
   "source": [
    "model1 = CNN(**params_model1)\n",
    "\n",
    "model1.load_state_dict(torch.load('trained_models/Thresholds/cnn_300_200_[100]_12_[3]_e80_60-20-20_polisis_state.pt'))\n",
    "\n",
    "model1.print_results(train_dataset, validation_dataset, 0.5)\n",
    "\n",
    "y_validation = validation_dataset.labels_tensor\n",
    "\n",
    "x_validation = PPD.collate_data(validation_dataset)[0]\n",
    "\n",
    "y_hat_validation1 = model1(x_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'y_hat_validation1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-660edfc00f80>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbest_f1_score1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_t_label1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCNN\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_best_thresholds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_validation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_hat_validation1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_results_best_t\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_t_label1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'----macro averages----'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'y_hat_validation1' is not defined"
     ]
    }
   ],
   "source": [
    "best_f1_score1, best_t_label1 = CNN.get_best_thresholds(y_validation, y_hat_validation1, labels)\n",
    "\n",
    "model1.print_results_best_t(validation_dataset, torch.tensor(best_t_label1).float())\n",
    "\n",
    "print('----macro averages----')\n",
    "\n",
    "model1.f1_score(y_validation, y_hat_validation1, torch.tensor(best_t_label1).float(), macro=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-2fda407eacd4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfusion_matrices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_t_label1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TP'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mfp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'FP'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TN'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model1' is not defined"
     ]
    }
   ],
   "source": [
    "metrics = confusion_matrices(model1, validation_dataset, torch.tensor(best_t_label1).float())\n",
    "\n",
    "tp = sum(metrics['TP'])\n",
    "fp = sum(metrics['FP'])\n",
    "tn = sum(metrics['TN'])\n",
    "fn = sum(metrics['FN'])\n",
    "\n",
    "print('------------------best_t--------------------')\n",
    "\n",
    "print('f1: {}'.format(2 * tp / (2 * tp + fn + fp)))\n",
    "print('precision: {}'.format(tp / (tp + fp)))\n",
    "print('recall: {}'.format(tp / (tp + fn)))\n",
    "\n",
    "metrics = confusion_matrices(model1, validation_dataset, torch.tensor(0.5).float())\n",
    "\n",
    "tp = sum(metrics['TP'])\n",
    "fp = sum(metrics['FP'])\n",
    "tn = sum(metrics['TN'])\n",
    "fn = sum(metrics['FN'])\n",
    "\n",
    "print('-------------------t-0.5--------------------')\n",
    "\n",
    "print('f1: {}'.format(2 * tp / (2 * tp + fn + fp)))\n",
    "print('precision: {}'.format(tp / (tp + fp)))\n",
    "print('recall: {}'.format(tp / (tp + fn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_model2_file = open('trained_models/Thresholds/cnn_300_200_[100]_12_[3]_lr.005_e300_60-20-20_polisis_params.pkl', 'rb')\n",
    "\n",
    "params_model2 = pickle.load(params_model2_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = CNN(**params_model2)\n",
    "\n",
    "model2.load_state_dict(torch.load('trained_models/Thresholds/cnn_300_200_[100]_12_[3]_lr.005_e300_60-20-20_polisis_state.pt'))\n",
    "\n",
    "model2.print_results(train_dataset, validation_dataset, 0.5)\n",
    "\n",
    "y_validation = validation_dataset.labels_tensor\n",
    "\n",
    "x_validation = PPD.collate_data(validation_dataset)[0]\n",
    "\n",
    "y_hat_validation2 = model2(x_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_f1_score, best_t_label2 = CNN.get_best_thresholds(y_validation, y_hat_validation2, labels)\n",
    "\n",
    "model2.print_results_best_t(validation_dataset, torch.tensor(best_t_label2).float())\n",
    "\n",
    "print('----macro averages----')\n",
    "\n",
    "model2.f1_score(y_validation, y_hat_validation2, torch.tensor(best_t_label2).float(), macro=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = confusion_matrices(model2, validation_dataset, torch.tensor(best_t_label2).float())\n",
    "\n",
    "tp = sum(metrics['TP'])\n",
    "fp = sum(metrics['FP'])\n",
    "tn = sum(metrics['TN'])\n",
    "fn = sum(metrics['FN'])\n",
    "\n",
    "print('------------------best_t--------------------')\n",
    "\n",
    "print('f1: {}'.format(2 * tp / (2 * tp + fn + fp)))\n",
    "print('precision: {}'.format(tp / (tp + fp)))\n",
    "print('recall: {}'.format(tp / (tp + fn)))\n",
    "\n",
    "metrics = confusion_matrices(model2, validation_dataset, torch.tensor(0.5).float())\n",
    "\n",
    "tp = sum(metrics['TP'])\n",
    "fp = sum(metrics['FP'])\n",
    "tn = sum(metrics['TN'])\n",
    "fn = sum(metrics['FN'])\n",
    "\n",
    "print('-------------------t-0.5--------------------')\n",
    "\n",
    "print('f1: {}'.format(2 * tp / (2 * tp + fn + fp)))\n",
    "print('precision: {}'.format(tp / (tp + fp)))\n",
    "print('recall: {}'.format(tp / (tp + fn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.f1_score_per_label(y_validation, y_hat_validation2, torch.tensor(best_t_label2).float())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}