pooled GRU+preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.kaggle.com/yekenot/pooled-gru"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import re\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate\n",
    "from keras.layers import GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.callbacks import Callback\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['OMP_NUM_THREADS'] = '4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = '../input/glove.6B.100d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../input/train.csv')\n",
    "test = pd.read_csv('../input/test.csv')\n",
    "submission = pd.read_csv('../input/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PREPROCESSING PART\n",
    "repl = {\n",
    "    \"&lt;3\": \" good \",\n",
    "    \":d\": \" good \",\n",
    "    \":dd\": \" good \",\n",
    "    \":p\": \" good \",\n",
    "    \"8)\": \" good \",\n",
    "    \":-)\": \" good \",\n",
    "    \":)\": \" good \",\n",
    "    \";)\": \" good \",\n",
    "    \"(-:\": \" good \",\n",
    "    \"(:\": \" good \",\n",
    "    \"yay!\": \" good \",\n",
    "    \"yay\": \" good \",\n",
    "    \"yaay\": \" good \",\n",
    "    \"yaaay\": \" good \",\n",
    "    \"yaaaay\": \" good \",\n",
    "    \"yaaaaay\": \" good \",\n",
    "    \":/\": \" bad \",\n",
    "    \":&gt;\": \" sad \",\n",
    "    \":')\": \" sad \",\n",
    "    \":-(\": \" bad \",\n",
    "    \":(\": \" bad \",\n",
    "    \":s\": \" bad \",\n",
    "    \":-s\": \" bad \",\n",
    "    \"&lt;3\": \" heart \",\n",
    "    \":d\": \" smile \",\n",
    "    \":p\": \" smile \",\n",
    "    \":dd\": \" smile \",\n",
    "    \"8)\": \" smile \",\n",
    "    \":-)\": \" smile \",\n",
    "    \":)\": \" smile \",\n",
    "    \";)\": \" smile \",\n",
    "    \"(-:\": \" smile \",\n",
    "    \"(:\": \" smile \",\n",
    "    \":/\": \" worry \",\n",
    "    \":&gt;\": \" angry \",\n",
    "    \":')\": \" sad \",\n",
    "    \":-(\": \" sad \",\n",
    "    \":(\": \" sad \",\n",
    "    \":s\": \" sad \",\n",
    "    \":-s\": \" sad \",\n",
    "    r\"\\br\\b\": \"are\",\n",
    "    r\"\\bu\\b\": \"you\",\n",
    "    r\"\\bhaha\\b\": \"ha\",\n",
    "    r\"\\bhahaha\\b\": \"ha\",\n",
    "    r\"\\bdon't\\b\": \"do not\",\n",
    "    r\"\\bdoesn't\\b\": \"does not\",\n",
    "    r\"\\bdidn't\\b\": \"did not\",\n",
    "    r\"\\bhasn't\\b\": \"has not\",\n",
    "    r\"\\bhaven't\\b\": \"have not\",\n",
    "    r\"\\bhadn't\\b\": \"had not\",\n",
    "    r\"\\bwon't\\b\": \"will not\",\n",
    "    r\"\\bwouldn't\\b\": \"would not\",\n",
    "    r\"\\bcan't\\b\": \"can not\",\n",
    "    r\"\\bcannot\\b\": \"can not\",\n",
    "    r\"\\bi'm\\b\": \"i am\",\n",
    "    \"m\": \"am\",\n",
    "    \"r\": \"are\",\n",
    "    \"u\": \"you\",\n",
    "    \"haha\": \"ha\",\n",
    "    \"hahaha\": \"ha\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"can't\": \"can not\",\n",
    "    \"cannot\": \"can not\",\n",
    "    \"i'm\": \"i am\",\n",
    "    \"m\": \"am\",\n",
    "    \"i'll\" : \"i will\",\n",
    "    \"its\" : \"it is\",\n",
    "    \"it's\" : \"it is\",\n",
    "    \"'s\" : \" is\",\n",
    "    \"that's\" : \"that is\",\n",
    "    \"weren't\" : \"were not\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [i for i in repl.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_train_data = []\n",
    "new_test_data = []\n",
    "ltr = train[\"comment_text\"].tolist()\n",
    "lte = test[\"comment_text\"].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in ltr:\n",
    "    arr = str(i).split()\n",
    "    xx = \"\"\n",
    "    for j in arr:\n",
    "        j = str(j).lower()\n",
    "        if j[:4] == 'http' or j[:3] == 'www':\n",
    "            continue\n",
    "        if j in stop_words:\n",
    "            continue\n",
    "        if j in keys:\n",
    "            # print(\"inn\")\n",
    "            j = repl[j]\n",
    "        xx += j + \" \"\n",
    "    new_train_data.append(xx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in lte:\n",
    "    arr = str(i).split()\n",
    "    xx = \"\"\n",
    "    for j in arr:\n",
    "        j = str(j).lower()\n",
    "        if j[:4] == 'http' or j[:3] == 'www':\n",
    "            continue\n",
    "        if j in stop_words:\n",
    "            continue\n",
    "        if j in keys:\n",
    "            # print(\"inn\")\n",
    "            j = repl[j]\n",
    "        xx += j + \" \"\n",
    "    new_test_data.append(xx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "crap removed\n"
     ]
    }
   ],
   "source": [
    "train[\"new_comment_text\"] = new_train_data\n",
    "test[\"new_comment_text\"] = new_test_data\n",
    "print(\"crap removed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "trate = train[\"new_comment_text\"].tolist()\n",
    "tete = test[\"new_comment_text\"].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, c in enumerate(trate):\n",
    "    trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i]).lower())\n",
    "for i, c in enumerate(tete):\n",
    "    tete[i] = re.sub('[^a-zA-Z ?!]+', '', tete[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "only alphabets\n"
     ]
    }
   ],
   "source": [
    "train[\"comment_text\"] = trate\n",
    "test[\"comment_text\"] = tete\n",
    "print('only alphabets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train[\"comment_text\"].fillna(\"fillna\").values\n",
    "y_train = train[[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]].values\n",
    "X_test = test[\"comment_text\"].fillna(\"fillna\").values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 20000\n",
    "maxlen = 100\n",
    "embed_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = text.Tokenizer(num_words=max_features)\n",
    "tokenizer.fit_on_texts(list(X_train) + list(X_test))\n",
    "X_train = tokenizer.texts_to_sequences(X_train)\n",
    "X_test = tokenizer.texts_to_sequences(X_test)\n",
    "x_train = sequence.pad_sequences(X_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(X_test, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_index = dict(get_coefs(*o.strip().split()) for o in open(EMBEDDING_FILE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_embs = np.stack(embeddings_index.values())\n",
    "emb_mean, emb_std = all_embs.mean(), all_embs.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = min(max_features, len(word_index))\n",
    "embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "for word, i in word_index.items():\n",
    "    if i >= max_features: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            print(\"\\n ROC-AUC - epoch: %d - score: %.6f \\n\" % (epoch+1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "#from keras import initializations\n",
    "from keras import initializers, regularizers, constraints\n",
    "\n",
    "\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        #self.init = initializations.get('glorot_uniform')\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    inp = Input(shape=(maxlen, ))\n",
    "    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n",
    "    x = SpatialDropout1D(0.2)(x)\n",
    "    x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "    x = SpatialDropout1D(0.5)(x)\n",
    "    x = Attention(maxlen)(x)\n",
    "#     avg_pool = GlobalAveragePooling1D()(x)\n",
    "#     max_pool = GlobalMaxPooling1D()(x)\n",
    "#     conc = concatenate([avg_pool, max_pool])\n",
    "#     outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "    outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=inp, outputs=outp)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_model()\n",
    "batch_size = 32\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "[X_tra, X_val, y_tra, y_val] = train_test_split(x_train, y_train, train_size=0.75, random_state=233)\n",
    "RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_decay = lambda init, fin, steps: (init/fin)**(1/(steps-1)) - 1\n",
    "steps = int(len(X_tra)/batch_size) * epochs\n",
    "lr_init, lr_fin = 0.001, 0.0005\n",
    "lr_decay = exp_decay(lr_init, lr_fin, steps)\n",
    "K.set_value(model.optimizer.lr, lr_init)\n",
    "K.set_value(model.optimizer.decay, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 119678 samples, validate on 39893 samples\n",
      "Epoch 1/3\n",
      "119678/119678 [==============================] - 909s 8ms/step - loss: 0.0655 - acc: 0.9779 - val_loss: 0.0484 - val_acc: 0.9823\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.977318 \n",
      "\n",
      "Epoch 2/3\n",
      "112192/119678 [===========================>..] - ETA: 7:37 - loss: 0.0486 - acc: 0.9820"
     ]
    }
   ],
   "source": [
    "hist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n",
    "                 callbacks=[RocAuc], verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### record results:\n",
    "-  gru_preprocessing_moredropouts.csv\n",
    "        x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "\n",
    "        avg_pool = GlobalAveragePooling1D()(x)\n",
    "\n",
    "        max_pool = GlobalMaxPooling1D()(x)\n",
    "\n",
    "        conc = concatenate([avg_pool, max_pool])\n",
    "\n",
    "        outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "    \n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 1120s 9ms/step - loss: 0.0580 - acc: 0.9795 - val_loss: 0.0484 - val_acc: 0.9826\n",
    "\n",
    "         ROC-AUC - epoch: 1 - score: 0.982405 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 1053s 9ms/step - loss: 0.0436 - acc: 0.9835 - val_loss: 0.0443 - val_acc: 0.9833\n",
    "\n",
    "        ROC-AUC - epoch: 2 - score: 0.984421 \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "-  gru_preprocessing_attention.csv \n",
    "        x = SpatialDropout1D(0.2)(x)\n",
    "\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "\n",
    "        x = Attention(maxlen)(x)\n",
    "\n",
    "        outp\n",
    "\n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 1002s 8ms/step - loss: 0.0394 - acc: 0.9847 - val_loss: 0.0456 - val_acc: 0.9825\n",
    "\n",
    "        ROC-AUC - epoch: 1 - score: 0.984845 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 1075s 9ms/step - loss: 0.0362 - acc: 0.9859 - val_loss: 0.0460 - val_acc: 0.9835\n",
    "\n",
    "        ROC-AUC - epoch: 2 - score: 0.984626 \n",
    "\n",
    "- gru_preprocessing_pool_attention.csv\n",
    "        x = SpatialDropout1D(0.2)(x)\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "        avg_pool = GlobalAveragePooling1D()(x)\n",
    "        max_pool = GlobalMaxPooling1D()(x)\n",
    "        conc = concatenate([avg_pool, max_pool])\n",
    "        conc = Attention(maxlen)(conc)\n",
    "        outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "        \n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 1062s 9ms/step - loss: 0.0336 - acc: 0.9868 - val_loss: 0.0488 - val_acc: 0.9830\n",
    "\n",
    "         ROC-AUC - epoch: 1 - score: 0.983471 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 940s 8ms/step - loss: 0.0310 - acc: 0.9877 - val_loss: 0.0503 - val_acc: 0.9824\n",
    "\n",
    "         ROC-AUC - epoch: 2 - score: 0.982497 \n",
    "\n",
    "- gru_preprocessing_pool_attention_concat.csv\n",
    "        x = SpatialDropout1D(0.2)(x)\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "        avg_pool = GlobalAveragePooling1D()(x)\n",
    "        max_pool = GlobalMaxPooling1D()(x)\n",
    "        atten = Attention(maxlen)(x)\n",
    "        conc = concatenate([avg_pool, max_pool, atten])\n",
    "        outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 1105s 9ms/step - loss: 0.0286 - acc: 0.9887 - val_loss: 0.0521 - val_acc: 0.9823\n",
    "\n",
    "         ROC-AUC - epoch: 1 - score: 0.982510 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 1119s 9ms/step - loss: 0.0263 - acc: 0.9897 - val_loss: 0.0561 - val_acc: 0.9820\n",
    "\n",
    "         ROC-AUC - epoch: 2 - score: 0.980674 \n",
    "         \n",
    "- 'gru_preprocessing_pool_attention_pool.csv'\n",
    "        x = SpatialDropout1D(0.2)(x)\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "        x = Attention(maxlen)(x)\n",
    "        avg_pool = GlobalAveragePooling1D()(x)\n",
    "        max_pool = GlobalMaxPooling1D()(x)\n",
    "        conc = concatenate([avg_pool, max_pool])\n",
    "        outp = Dense(6, activation=\"sigmoid\")(conc)\n",
    "        \n",
    "        Train on 119678 samples, validate on 39893 samples\n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 975s 8ms/step - loss: 0.0244 - acc: 0.9904 - val_loss: 0.0613 - val_acc: 0.9816\n",
    "\n",
    "         ROC-AUC - epoch: 1 - score: 0.979430 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 931s 8ms/step - loss: 0.0226 - acc: 0.9911 - val_loss: 0.0631 - val_acc: 0.9814\n",
    "\n",
    "         ROC-AUC - epoch: 2 - score: 0.978290\n",
    "       \n",
    "- gru_preprocessing_attention_moredropouts.csv\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "        x = Bidirectional(GRU(80, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "        x = SpatialDropout1D(0.5)(x)\n",
    "        x = Attention(maxlen)(x)\n",
    "        outp = Dense(6, activation=\"sigmoid\")(x)\n",
    "        \n",
    "        Train on 119678 samples, validate on 39893 samples\n",
    "        Epoch 1/2\n",
    "        119678/119678 [==============================] - 1016s 8ms/step - loss: 0.0211 - acc: 0.9920 - val_loss: 0.0648 - val_acc: 0.9811\n",
    "\n",
    "         ROC-AUC - epoch: 1 - score: 0.977665 \n",
    "\n",
    "        Epoch 2/2\n",
    "        119678/119678 [==============================] - 1035s 9ms/step - loss: 0.0195 - acc: 0.9925 - val_loss: 0.0683 - val_acc: 0.9810\n",
    "\n",
    "         ROC-AUC - epoch: 2 - score: 0.978057 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test, batch_size=1024)\n",
    "submission[[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]] = y_pred\n",
    "submission.to_csv('gru_preprocessing_attention_stopwords.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## analyze false postive & false negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def false_prediction(pred, real, df):\n",
    "    result = pd.DataFrame(columns=[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\", \"comment_text\"])\n",
    "    if pred.shape[0] != real.shape[0]:\n",
    "        print (\"Error in shape\")\n",
    "        return\n",
    "    for row_idx in range(pred.shape[0]):\n",
    "        pred_row = pred[row_idx, :]\n",
    "        real_row = real[row_idx, :]\n",
    "        if not np.array_equal(pred_row, real_row):\n",
    "            print (\"comment: {}\".format(df.iloc[row_idx]['comment_text']))\n",
    "            print ('t: [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]')\n",
    "            print (\"p: {}\".format(pred_row))\n",
    "            print (\"r: {}\\n\".format(real_row))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = model.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred_round = np.round(y_train_pred).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "false_prediction(y_train_pred_round, y_train, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred_round"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}