MLP&LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MLP&LSTM.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "10Gr27YNFCvS",
        "s9e8DZxZ6koo",
        "MQctqVsj6hO-",
        "9ZGH6T5V6oKN"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "vffi7Ee-KUSX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63cb1737-0ae6-4059-c216-bbde73000ab5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 43.8 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 5.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 48.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 52.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cEeLcBdpUClE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40645c9c-9915-4acd-9a28-6826ec4ffd50"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import"
      ],
      "metadata": {
        "id": "10Gr27YNFCvS"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nuQJ-Y2De8sC"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import random\n",
        "import scipy.sparse as sp\n",
        "import torch\n",
        "import os\n",
        "import time\n",
        "import pandas as pd\n",
        "import pickle\n",
        "import copy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "SEED = 42\n",
        "os.environ['PYTHONHASHSEED']=str(SEED)\n",
        "os.environ['TF_CUDNN_DETERMINISTIC'] = '1'  # TF 2.1\n",
        "random.seed(SEED)\n",
        "#seed(SEED)\n",
        "np.random.seed(SEED)"
      ],
      "metadata": {
        "id": "vHDOC0T3fclR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torch import optim\n",
        "import torch.nn as nn\n",
        "from transformers import GPT2Tokenizer, GPT2Model, GPT2Config\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "from sklearn.metrics import f1_score  \n",
        "from sklearn.metrics import roc_auc_score\n",
        "from torch.utils.data import DataLoader, TensorDataset"
      ],
      "metadata": {
        "id": "8_UBWYlpOOZr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Dataset"
      ],
      "metadata": {
        "id": "s9e8DZxZ6koo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def subwords_to_merge(tokenized_sequence, sentence, verbose):\n",
        "  for i in range(len(tokenized_sequence)) :\n",
        "    while 'Ġ' in tokenized_sequence[i] :\n",
        "      tokenized_sequence[i] = tokenized_sequence[i].replace('Ġ','')\n",
        "  words = sentence.split(' ')\n",
        "  if verbose :\n",
        "    print(\"words\", words)\n",
        "    print(\"token\", tokenized_sequence)\n",
        "  i = 0\n",
        "  j = 0\n",
        "  cpt = 0\n",
        "  n_words = len(words)\n",
        "  list_subwords_to_merge = []\n",
        "\n",
        "  while cpt != n_words :\n",
        "    if tokenized_sequence[j] == words[i] :\n",
        "      cpt = cpt + 1\n",
        "      i = i + 1\n",
        "      j = j + 1\n",
        "    else :\n",
        "      tmp_word = tokenized_sequence[j]\n",
        "      tmp_merge = [j]\n",
        "      while tmp_word != words[i] :\n",
        "        j = j+1\n",
        "        tmp_word = tmp_word + tokenized_sequence[j]\n",
        "        tmp_merge.append(j)\n",
        "      list_subwords_to_merge.append(tmp_merge)\n",
        "      cpt = cpt + 1\n",
        "      i = i + 1\n",
        "      j = j + 1\n",
        "  return list_subwords_to_merge\n",
        "\n",
        "\n",
        "def get_embedding(comment, tokenizer, model, verbose):\n",
        "  tokenized_sequence = tokenizer.tokenize(comment)\n",
        "  subwords_indices = subwords_to_merge(tokenized_sequence, comment, verbose)\n",
        "  encoded_input = tokenizer(comment, return_tensors='pt')\n",
        "  output = model(**encoded_input)\n",
        "  outputseq = output.last_hidden_state[0]\n",
        "\n",
        "  final_embd = []\n",
        "  i = 0\n",
        "\n",
        "  while i != len(outputseq):\n",
        "    inside = False\n",
        "    for k in subwords_indices :\n",
        "      first, last = k[0], k[-1]\n",
        "      if i in range(first, last) :\n",
        "        inside = True\n",
        "        if (last+1)>len(outputseq):\n",
        "          merge, _ = torch.max(outputseq[first::], 0)\n",
        "        else:\n",
        "          merge, _ = torch.max(outputseq[first:(last+1)], 0)\n",
        "        final_embd.append(list(merge.detach().numpy()))\n",
        "        i =  i + len(k)\n",
        "        \n",
        "    if inside == False:\n",
        "      final_embd.append(list(outputseq[i].detach().numpy()))\n",
        "      i = i + 1\n",
        "\n",
        "  final_embd = torch.tensor(np.array(final_embd))\n",
        "  return final_embd"
      ],
      "metadata": {
        "id": "8wsFfT2jlcMc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def sentence_process(sentence):\n",
        "  comment = sentence.replace('\\n', ' ')\n",
        "  while '\\\"' in comment :\n",
        "    comment = comment.replace('\\\"', '')\n",
        "  while \"\\'\" in comment :\n",
        "    comment = comment.replace(\"\\'\", '')\n",
        "  while ':' in comment :\n",
        "    comment = comment.replace(':', '')\n",
        "  while '.' in comment :\n",
        "    comment = comment.replace('.', '')\n",
        "  while '@' in comment :\n",
        "    comment = comment.replace('@', '')\n",
        "  while '+' in comment :\n",
        "    comment = comment.replace('+', '')\n",
        "  while '=' in comment:\n",
        "    comment = comment.replace('=', '')\n",
        "  while '&' in comment :\n",
        "    comment = comment.replace('&', '')\n",
        "  while ')' in comment or '(' in comment:\n",
        "    comment = comment.replace(')', '').replace('(', '')\n",
        "  while ',' in comment or ':' in comment or ';' in comment:\n",
        "    comment = comment.replace(\":\", '').replace(',', '').replace(';', '')\n",
        "  for c in comment :\n",
        "    if c.isascii() == False :\n",
        "      comment = comment.replace(c, '')\n",
        "  n1 = comment.count('!')\n",
        "  n2 = comment.count('?')\n",
        "  for i in range(n1):\n",
        "    comment = comment.replace('!', ' !')\n",
        "  for i in range(n2):\n",
        "    comment = comment.replace('?', ' ?')\n",
        "  while '  ' in comment :\n",
        "    comment = comment.replace('  ', ' ')\n",
        "  return comment\n",
        "  \n",
        "class AffineTransform(object):\n",
        "    def __init__(self, filepath, save_dir):\n",
        "        self.filepath = filepath\n",
        "        self.save_dir = save_dir\n",
        "\n",
        "    def process(self):\n",
        "\n",
        "      if not os.path.exists(self.save_dir + 'Affine_X.pt'):\n",
        "        tic = time.time()\n",
        "        train_df = pd.read_csv(self.filepath)\n",
        "\n",
        "        #delete comments that belongs to several classes\n",
        "        idx_to_del = []\n",
        "        for row in train_df.itertuples():\n",
        "          if sum(row[3::])> 1 :\n",
        "            idx_to_del.append(row[0])\n",
        "          \n",
        "        train_df = train_df.drop(idx_to_del)\n",
        "\n",
        "        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "        model = GPT2Model.from_pretrained('gpt2')\n",
        "\n",
        "        cpt = 0\n",
        "        n_too_long = 0\n",
        "\n",
        "\n",
        "        toxic = train_df[train_df['toxic'] == 0]\n",
        "        obscene = toxic[toxic['obscene']==0]\n",
        "        threat = obscene[obscene['threat']==0]\n",
        "        insult = threat[threat['insult']==0]\n",
        "        neutral = insult[insult['identity_hate']==0] #df with neutral comment\n",
        "\n",
        "        toxic = train_df[train_df['toxic'] == 1]\n",
        "        obscene = train_df[train_df['obscene'] == 1]\n",
        "        threat = train_df[train_df['threat'] == 1]\n",
        "        insult = train_df[train_df['insult'] == 1]\n",
        "        identity_hate = train_df[train_df['identity_hate'] == 1]\n",
        "\n",
        "        final_df = pd.concat([toxic, obscene, threat, insult, identity_hate, neutral]) #merge all the df together in the right order\n",
        "        X = []\n",
        "        labels = []\n",
        "        max = 0\n",
        "        for row in final_df.itertuples():\n",
        "      \n",
        "          cpt = cpt + 1\n",
        "          print(\"cpt:\", cpt)\n",
        "         \n",
        "          if cpt in [1344, 1345, 1445, 1446, 1528, 1529, 3243, 3244, 3773, 3776, 11888] :\n",
        "            continue\n",
        "          if len(X) == 10000 or len(labels) == 10000 :\n",
        "            if np.shape(X)[0] == np.shape(labels)[0] :\n",
        "              break\n",
        "            else : \n",
        "              print(\"Embedding matrix and labels different shapes\")\n",
        "              return -1\n",
        "          \n",
        "          \n",
        "          comment = sentence_process(row[2])\n",
        "          words = comment.split(' ')\n",
        "         \n",
        "          while '' in words :\n",
        "            words.remove('')\n",
        "          \n",
        "          for w in words :\n",
        "            if w.isnumeric() and len(w)>=5 :\n",
        "              words.remove(w)\n",
        "          \n",
        "          words = list(filter(lambda s: 'http' not in s, words))\n",
        "            \n",
        "          comment = \" \".join(words)\n",
        "\n",
        "          if len(words)>100 : #comments with more than 1024 words = cannot use with GPT2 or more than 996 : outliers comments\n",
        "            print(\"Too long\")\n",
        "            n_too_long += 1\n",
        "            continue\n",
        "\n",
        "\n",
        "          #print(\"comment:\", comment)\n",
        "          if row[3] == 0 and row[4] == 0 and row[5] ==0 and row[6] ==0 and row[7] == 0 and row[8] == 0:\n",
        "            labels.append(0)\n",
        "          elif row[3] == 1 :\n",
        "            labels.append(1)\n",
        "          elif row[4] == 1 :\n",
        "            labels.append(6)\n",
        "          elif row[5] == 1 :\n",
        "            labels.append(2)\n",
        "          elif row[6] == 1 :\n",
        "            labels.append(3)\n",
        "          elif row[7] == 1 :\n",
        "            labels.append(4)\n",
        "          else: \n",
        "            labels.append(5)\n",
        "      \n",
        "          #Get embeddings for the sentence\n",
        "          verbose = False\n",
        "          outputseq = get_embedding(comment, tokenizer, model, verbose)\n",
        "          if outputseq.shape[0]>max :\n",
        "            max = outputseq.shape[0]\n",
        "          \n",
        "          X.append(torch.flatten(outputseq))\n",
        "        for i, vector in enumerate(X) :\n",
        "          X[i] = torch.cat((X[i], torch.zeros(max*768 - list(X[i].shape)[0])))\n",
        "        X = torch.stack(X)\n",
        "        labels = torch.tensor(labels)\n",
        "        torch.save(X, self.save_dir + 'Affine_X.pt')\n",
        "        torch.save(labels, self.save_dir + 'Affine_labels.pt')\n",
        "      else :\n",
        "        X = torch.load(self.save_dir + 'Affine_X.pt')\n",
        "        labels = torch.load(self.save_dir + 'Affine_labels.pt')\n",
        "        \n",
        "      return X, labels\n",
        "\n",
        "dataset = AffineTransform('/content/drive/My Drive/IASD_tmp/NLP/train.csv', '/content/drive/My Drive/IASD_tmp/NLP/').process()"
      ],
      "metadata": {
        "id": "odes4cbBOiv5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## MLP"
      ],
      "metadata": {
        "id": "MQctqVsj6hO-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MLP(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
        "        super(MLP, self).__init__()\n",
        "        self.linear1 = nn.Linear(input_dim, hidden_dim)\n",
        "        self.batchnorm = nn.BatchNorm1d(hidden_dim)\n",
        "        self.dropout = nn.Dropout(0.2)\n",
        "        self.linear2 = nn.Linear(hidden_dim, output_dim)\n",
        "\n",
        "    def forward(self, input):\n",
        "        h = self.linear1(input)\n",
        "        h = self.batchnorm(h)\n",
        "        h = F.relu(h)\n",
        "        h = self.dropout(h)\n",
        "        h = self.linear2(h)\n",
        "        return h     "
      ],
      "metadata": {
        "id": "WmOvDeMKt1O-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(model, params, trainloader, class_weights) :\n",
        "  optimizer = params['optimizer'](model.parameters(), lr=params['learning_rate'])\n",
        "\n",
        "  for epoch in range(params['n_epochs']): \n",
        "    running_loss = 0.0\n",
        "    for i, data in enumerate(trainloader, 0):\n",
        "        inputs, labels = data\n",
        "        inputs = inputs.to(device)\n",
        "        labels = labels.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(inputs)\n",
        "        loss = F.cross_entropy(outputs, labels, weight = class_weights)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        running_loss += loss.item()\n",
        "        if i % 100 == 0 and params['verbose']:  \n",
        "            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n",
        "            running_loss = 0.0"
      ],
      "metadata": {
        "id": "xQX2k4Uwwd34"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(testloader, model):\n",
        "  num_correct = 0\n",
        "  num_tests = 0\n",
        "  for batched_graph, labels in testloader:\n",
        "    batched_graph = batched_graph.to(device)\n",
        "    labels = labels.to(device)\n",
        "    with torch.no_grad():\n",
        "      pred = model(batched_graph)\n",
        "    num_correct += (pred.argmax(1) == labels).sum().item()\n",
        "    num_tests += len(labels)\n",
        "\n",
        "  return num_correct / num_tests"
      ],
      "metadata": {
        "id": "dg4D6bDqkOwo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(testloader, model):\n",
        "  #preds = []\n",
        "  true_labels = []\n",
        "  for batched_graph, labels in testloader:\n",
        "    true_labels.append(labels)\n",
        "    batched_graph = batched_graph.to(device)\n",
        "    labels = labels.to(device)\n",
        "    with torch.no_grad():\n",
        "      pred = model(batched_graph)\n",
        "  #preds = torch.stack(preds)\n",
        "  true_labels = torch.cat(true_labels, dim=0)\n",
        "  return pred, true_labels"
      ],
      "metadata": {
        "id": "V3ns71muXD3y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "list_params = [{'n_epochs':20, 'learning_rate':0.001, 'batch_size':1024, 'hidden_dim':5000, 'optimizer': optim.Adam, 'verbose':False},\n",
        "               {'n_epochs':20, 'learning_rate':0.01, 'batch_size':1024, 'hidden_dim':5000, 'optimizer': optim.Adam, 'verbose':False},\n",
        "               {'n_epochs':20, 'learning_rate':0.001, 'batch_size':1024, 'hidden_dim':5000, 'optimizer': optim.SGD, 'verbose':False}, \n",
        "               {'n_epochs':20, 'learning_rate':0.001, 'batch_size':1024, 'hidden_dim':1000, 'optimizer': optim.Adam, 'verbose':False},\n",
        "               {'n_epochs':20, 'learning_rate':0.001, 'batch_size':1024, 'hidden_dim':500, 'optimizer': optim.Adam, 'verbose':False},\n",
        "               ]\n",
        "random_states = [0,1,2]\n",
        "\n",
        "for p, params in enumerate(list_params) :\n",
        "  tic = time.time()\n",
        "\n",
        "  test_seed_acc, val_seed_acc = [], []\n",
        "  test_seed_f1, val_seed_f1 = [], []\n",
        "  test_seed_auc, val_seed_auc = [], []\n",
        "  for r in random_states :\n",
        "    X, labels = shuffle(dataset[0], dataset[1], random_state=r)\n",
        "\n",
        "    torch.manual_seed(r)\n",
        "    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "    num_examples = X.shape[0]\n",
        "    num_train = int(0.80 * num_examples)\n",
        "    num_val = int((num_examples-num_train)/2)\n",
        "    X_train, y_train = X[:num_train], labels[:num_train]\n",
        "    class_weights = torch.from_numpy(compute_class_weight('balanced', classes = np.unique(y_train), y=y_train.cpu().numpy())).to(dtype=torch.float32).to(device)\n",
        "    X_val, y_val = X[num_train:(num_train+num_val)], labels[num_train : (num_train+num_val)]\n",
        "    X_test, y_test = X[(num_train+num_val) : (num_train+2*num_val)], labels[(num_train+num_val):(num_train+2*num_val)]\n",
        "    train_dataset = TensorDataset(X_train, y_train)\n",
        "    trainloader = DataLoader(train_dataset, batch_size=params['batch_size'])\n",
        "\n",
        "    val_dataset = TensorDataset(X_val, y_val)\n",
        "    val_loader = DataLoader(val_dataset, batch_size=params['batch_size'])\n",
        "\n",
        "    test_dataset = TensorDataset(X_test, y_test)\n",
        "    testloader = DataLoader(test_dataset, batch_size=params['batch_size'])\n",
        "    model = MLP(X_train.shape[1], params['hidden_dim'], 6).to(device)\n",
        "    train(model, params, trainloader, class_weights)\n",
        "\n",
        "    val_seed_acc.append(evaluate(val_loader, model))\n",
        "    test_seed_acc.append(evaluate(testloader, model))\n",
        "\n",
        "    val_preds, val_labels = predict(val_loader, model)\n",
        "    val_preds = val_preds.cpu()\n",
        "    val_labels = val_labels.cpu()\n",
        "    test_preds, test_labels = predict(testloader, model)\n",
        "    test_preds = test_preds.cpu()\n",
        "    test_labels = test_labels.cpu()\n",
        "\n",
        "    _, one_hot_val_preds = torch.max(val_preds, 1)\n",
        "    _, one_hot_test_preds = torch.max(test_preds, 1)\n",
        "    val_seed_f1.append(f1_score(val_labels, one_hot_val_preds, average='weighted'))\n",
        "    test_seed_f1.append(f1_score(test_labels, one_hot_test_preds, average='weighted'))\n",
        "\n",
        "    prob_val = nn.Softmax(dim=1)(val_preds)\n",
        "    prob_test = nn.Softmax(dim=1)(test_preds)\n",
        "    val_seed_auc.append(roc_auc_score(val_labels, prob_val, multi_class=\"ovo\", average=\"weighted\"))\n",
        "    test_seed_auc.append(roc_auc_score(test_labels, prob_test, multi_class=\"ovo\", average=\"weighted\"))\n",
        "\n",
        "  #Mesure incertitude en fct seed\n",
        "  validation_acc = np.mean(val_seed_acc)\n",
        "  test_acc = np.mean(test_seed_acc)\n",
        "\n",
        "  validation_f1 = np.mean(val_seed_f1)\n",
        "  test_f1 = np.mean(test_seed_f1)\n",
        "\n",
        "  validation_auc = np.mean(val_seed_auc)\n",
        "  test_auc = np.mean(test_seed_auc)\n",
        "  print(\"Performances for combination\", p, \"\\n\")\n",
        "  print(\"Mean val acc :\", validation_acc, \"+-\", np.std(val_seed_acc))\n",
        "  print(\"F1-score val acc :\", validation_f1, \"+-\", np.std(val_seed_f1))\n",
        "\n",
        "  print(\"Mean test acc :\", test_acc, \"+-\", np.std(test_seed_acc))\n",
        "  print(\"F1-score test acc :\", test_f1, \"+-\", np.std(test_seed_f1))\n",
        "\n",
        "  print(\"Mean val AUC:\", validation_auc, \"+-\", np.std(val_seed_auc))\n",
        "  print(\"Mean test AUC:\", test_auc, \"+-\", np.std(test_seed_auc), \"\\n\")\n",
        "\n",
        "tac = time.time()\n",
        "print(\"Grid search done after\", tac - tic)"
      ],
      "metadata": {
        "id": "bURKvqpxsLX7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6da129e-2c9f-4d26-b1d5-35b629327c81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Performances for combination 0 \n",
            "\n",
            "Mean val acc : 0.7673333333333333 +- 0.016336734339790476\n",
            "F1-score val acc : 0.7568226030987794 +- 0.01728311060612912\n",
            "Mean test acc : 0.765 +- 0.016309506430300102\n",
            "F1-score test acc : 0.750491897153062 +- 0.011696155538374973\n",
            "Mean val AUC: 0.7077846814644148 +- 0.007641971705947499\n",
            "Mean test AUC: 0.7026848016648742 +- 0.003939586463223441 \n",
            "\n",
            "Performances for combination 1 \n",
            "\n",
            "Mean val acc : 0.7000000000000001 +- 0.022759613353482054\n",
            "F1-score val acc : 0.7038190983546223 +- 0.024817617665963604\n",
            "Mean test acc : 0.6916666666666668 +- 0.024957742063113155\n",
            "F1-score test acc : 0.7064710288277379 +- 0.019314017655419426\n",
            "Mean val AUC: 0.6756504573047463 +- 0.031185920245175937\n",
            "Mean test AUC: 0.6858500289684829 +- 0.022337543956296464 \n",
            "\n",
            "Performances for combination 2 \n",
            "\n",
            "Mean val acc : 0.33066666666666666 +- 0.03308910528994232\n",
            "F1-score val acc : 0.4342316087651759 +- 0.03450807131931555\n",
            "Mean test acc : 0.33066666666666666 +- 0.028015868519267597\n",
            "F1-score test acc : 0.4255048715014427 +- 0.025203432425160176\n",
            "Mean val AUC: 0.5701700083067752 +- 0.025481589549020377\n",
            "Mean test AUC: 0.5831910056939469 +- 0.011621325520972326 \n",
            "\n",
            "Performances for combination 3 \n",
            "\n",
            "Mean val acc : 0.7616666666666667 +- 0.005436502143433369\n",
            "F1-score val acc : 0.74676205547212 +- 0.010641721914401145\n",
            "Mean test acc : 0.7603333333333334 +- 0.017782638224465536\n",
            "F1-score test acc : 0.7567338410800266 +- 0.020820022187872756\n",
            "Mean val AUC: 0.7215498864682864 +- 0.013082107227192453\n",
            "Mean test AUC: 0.6986093626566597 +- 0.015402800535868339 \n",
            "\n",
            "Performances for combination 4 \n",
            "\n",
            "Mean val acc : 0.7673333333333333 +- 0.011115554667022054\n",
            "F1-score val acc : 0.7536768314758312 +- 0.0144179252249517\n",
            "Mean test acc : 0.77 +- 0.014988884770611423\n",
            "F1-score test acc : 0.7548209359968266 +- 0.017294102002677777\n",
            "Mean val AUC: 0.7385451082232574 +- 0.016552081249954904\n",
            "Mean test AUC: 0.7025836548573879 +- 0.012354806433030168 \n",
            "\n",
            "Grid search done after 122.35183358192444\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## RandomForest"
      ],
      "metadata": {
        "id": "15UlS-ATll-8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "random_states = [0,1,2]\n",
        "\n",
        "accuracy = []\n",
        "F1_Score=[]\n",
        "AUC=[]\n",
        "\n",
        "tic = time.time()\n",
        "\n",
        "for r in random_states :\n",
        "\n",
        "  print(\"Random state : \", r, \"\\n\")\n",
        "  X, labels = dataset[0].numpy(), dataset[1].numpy()\n",
        "  X, labels = shuffle(X, labels, random_state=r)\n",
        "  np.random.seed(r)\n",
        "  random.seed(r)\n",
        "  num_examples = np.shape(X)[0]\n",
        "  num_train = int(0.80 * num_examples)\n",
        "  num_val = int((num_examples-num_train)/2)\n",
        "  X_train, y_train = X[:num_train], labels[:num_train]\n",
        "  X_test, y_test = X[(num_train+num_val) : (num_train+2*num_val)], labels[(num_train+num_val):(num_train+2*num_val)]\n",
        "\n",
        "  pipeline = Pipeline([('clf', RandomForestClassifier(random_state = r, class_weight=\"balanced\"))])\n",
        "\n",
        "  params = {'clf__n_estimators': [200, 500], 'clf__max_depth':[50, 20]}\n",
        "  rskf = StratifiedKFold(n_splits=3, random_state=r, shuffle=True)\n",
        "\n",
        "  cv = GridSearchCV(pipeline, params, cv = rskf, scoring = 'accuracy')\n",
        "  cv.fit(X_train, y_train)\n",
        "\n",
        "  preds_proba = cv.predict_proba(X_test)\n",
        "  preds = cv.predict(X_test)\n",
        "\n",
        "  accuracy.append(accuracy_score(y_test, preds))\n",
        "  F1_Score.append(f1_score(y_test, preds, average='weighted'))\n",
        "  AUC.append(roc_auc_score(y_test,preds_proba,multi_class=\"ovo\",average=\"weighted\"))\n",
        "\n",
        "print(\"Accuracy : \",np.mean(accuracy),\"with std\",np.std(accuracy))\n",
        "print(\"f1_score : \",np.mean(F1_Score),\"with std\",np.std(F1_Score))\n",
        "print(\"AUC : \",np.mean(AUC),\"with std\",np.std(AUC))\n",
        "tac = time.time()\n",
        "print('Finished in', tac-tic, 'seconds')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SyRYrybAhv6a",
        "outputId": "826c21c5-57a9-4a19-e269-25a7e7aca3d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Random state :  0 \n",
            "\n",
            "Random state :  1 \n",
            "\n",
            "Random state :  2 \n",
            "\n",
            "Accuracy :  0.7119999999999999 with std 0.0014142135623730961\n",
            "f1_score :  0.6900679124886073 with std 0.0028305514232209524\n",
            "AUC :  0.6835419164144985 with std 0.014798742444330985\n",
            "Finished in 16106.7343044281 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## LSTM (not enough RAM)"
      ],
      "metadata": {
        "id": "9ZGH6T5V6oKN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LSTM(nn.Module):\n",
        "\n",
        "    def __init__(self, params, embedding_dim=768, n_classes = 7):\n",
        "        super(LSTM, self).__init__()\n",
        "        self.hidden_dim = params['hidden_dim']\n",
        "        self.lstm = nn.LSTM(embedding_dim, params['hidden_dim'], num_layers = params['num_layers'], dropout = params['dropout'], bidirectional = params['bidirectional'])\n",
        "        self.linear = nn.Linear(params['hidden_dim'], n_classes)\n",
        "\n",
        "    def forward(self, sentence):\n",
        "        h, _ = self.lstm(sentence)\n",
        "        h = self.linear(h)\n",
        "        return h"
      ],
      "metadata": {
        "id": "ljcpOwre6qFw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(model, params, trainloader) :\n",
        "  criterion = F.cross_entropy()\n",
        "  optimizer = model['optimizer'](model.parameters(), lr=params['learning_rate'])\n",
        "\n",
        "  for epoch in range(params['n_epochs']): \n",
        "    running_loss = 0.0\n",
        "    for i, data in enumerate(trainloader, 0):\n",
        "        inputs, labels = data\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(inputs)\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        running_loss += loss.item()\n",
        "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
        "            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n",
        "            running_loss = 0.0"
      ],
      "metadata": {
        "id": "p1DII-w18mDw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(testloader, model):\n",
        "  num_correct = 0\n",
        "  num_tests = 0\n",
        "  for batched_graph, labels in testloader:\n",
        "    batched_graph = batched_graph.to(device)\n",
        "    labels = labels.to(device)\n",
        "    with torch.no_grad():\n",
        "      pred = model(batched_graph)\n",
        "    num_correct += (pred.argmax(1) == labels).sum().item()\n",
        "    num_tests += len(labels)\n",
        "\n",
        "  return num_correct / num_tests"
      ],
      "metadata": {
        "id": "LRpaaYIO8mDx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(testloader, model):\n",
        "  preds = []\n",
        "  true_labels = []\n",
        "  for batched_graph, labels in testloader:\n",
        "    true_labels.append(labels)\n",
        "    batched_graph = batched_graph.to(device)\n",
        "    labels = labels.to(device)\n",
        "    with torch.no_grad():\n",
        "      pred = model(batched_graph)\n",
        "      preds.append(preds)\n",
        "  preds = torch.cat(preds, dim=0)\n",
        "  true_labels = torch.cat(true_labels, dim=0)\n",
        "  return preds, true_labels"
      ],
      "metadata": {
        "id": "aXlUANuD8mDx"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}