LSTM_TENFOLD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: Loading necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import sys\n",
    "import warnings\n",
    "from gensim.models.wrappers import FastText\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim import models\n",
    "from sklearn.model_selection import KFold # import KFold\n",
    "from keras.regularizers import l2\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model, Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Input, Dense, concatenate, Activation, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Conv1D, GlobalMaxPooling1D, LSTM, Conv1D, Flatten, MaxPooling1D, SpatialDropout1D\n",
    "from keras.callbacks import Callback\n",
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score, accuracy_score\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: Loading pretrained word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ug_cbow = FastText.load_fasttext_format('../cc.ne.300.bin/cc.ne.300.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 584436 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "for w in model_ug_cbow.wv.vocab.keys():\n",
    "    embeddings_index[w] = model_ug_cbow.wv[w]#np.append(model_ug_cbow.wv[w],model_ug_sg.wv[w])#\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3: Loading Trained word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ug_sg = models.Word2Vec.load('../w2v_model_ug_sg.word2vec')\n",
    "model_ug_cbow = models.Word2Vec.load('../w2v_model_ug_cbow.word2vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 42409 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "for w in model_ug_cbow.wv.vocab.keys():\n",
    "    embeddings_index[w] = model_ug_cbow.wv[w]#\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 4: Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_excel(\"NepaliEarthquakeTweets_plus_blockade.xlsx\",sheetname=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_indexes = np.where(data['bhawna']=='p')\n",
    "neg_indexes = np.where(data['bhawna']=='n')\n",
    "total_size = 600\n",
    "size = 600\n",
    "val_size = 600\n",
    "df_pos = pd.DataFrame.from_items([('text',data['tweet_text'][pos_indexes[0][0:total_size]]),('target',0)])\n",
    "df_neg = pd.DataFrame.from_items([('text',data['tweet_text'][neg_indexes[0][0:total_size]]),('target',1)])\n",
    "\n",
    "df_pos['reply'] = data['reply'][pos_indexes[0][0:total_size]]\n",
    "df_pos['retweet'] = data['retweet'][pos_indexes[0][0:total_size]]\n",
    "df_pos['likes'] = data['likes'][pos_indexes[0][0:total_size]]\n",
    "\n",
    "df_neg['reply'] = data['reply'][neg_indexes[0][0:total_size]]\n",
    "df_neg['retweet'] = data['retweet'][neg_indexes[0][0:total_size]]\n",
    "df_neg['likes'] = data['likes'][neg_indexes[0][0:total_size]]\n",
    "\n",
    "df_pos.index = range(len(df_pos.index))\n",
    "df_neg.index = range(len(df_neg.index))\n",
    "\n",
    "df_pos_f = df_pos\n",
    "df_neg_f = df_neg\n",
    "df_F = df_pos[0:size]\n",
    "df_F = df_F.append(df_neg[0:size])\n",
    "df_F.index = range(len(df_F.index))\n",
    "\n",
    "len(df_F)\n",
    "validation_df_F= df_pos[500:600]\n",
    "validation_df_F = validation_df_F.append(df_neg[500:600])\n",
    "validation_df_F.index = range(len(validation_df_F.index))\n",
    "#validation_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_pos[0:size]\n",
    "df = df.append(df_neg[0:size])\n",
    "df.index = range(len(df.index))\n",
    "validation_df= df_pos[500:600]\n",
    "validation_df = validation_df.append(df_neg[500:600])\n",
    "validation_df.index = range(len(validation_df.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 5: Data Preprocessing: Removing puncuation, english text, decimal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pat1 = r'@[A-Za-z0-9]+'\n",
    "pat2 = r'https?://[A-Za-z0-9./]+'\n",
    "pat3 = r'([@#][A-Za-z0-9]+)'\n",
    "pat4 = r'.[A-Za-z0-9./]+'\n",
    "pat5 = r'[\\,۔،۔”–’‘‘_!…।-]|(\")|(:)|(%)|(ः)|(\\u200d)|(\\xa0…)|(\\u200c\\u200c)'\n",
    "combined_pat = r'|'.join((pat1, pat2,pat3,pat4,pat5))\n",
    "df['text'] = [re.sub(combined_pat, ' ', x) for x in df['text']]\n",
    "validation_df['text'] = [re.sub(combined_pat, ' ', x) for x in validation_df['text']]\n",
    "df_pos['text'] = [re.sub(combined_pat, ' ', x) for x in df_pos['text']]\n",
    "df_neg['text'] = [re.sub(combined_pat, ' ', x) for x in df_neg['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFold(n_splits=10, random_state=None, shuffle=False)\n"
     ]
    }
   ],
   "source": [
    "kf = KFold(n_splits=10)\n",
    "print(kf) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 6: Define Necessary funtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test(df_pos,df_neg,train_index,test_index):\n",
    "    X_train = df_pos.text[train_index.tolist()]\n",
    "    X_train = X_train.append(df_neg.text[train_index.tolist()])\n",
    "    X_train.index = range(len(X_train.index))\n",
    "    \n",
    "    y_train = df_pos.target[train_index.tolist()]\n",
    "    y_train = y_train.append(df_neg.target[train_index.tolist()])\n",
    "    y_train.index = range(len(y_train.index))\n",
    "    \n",
    "    X_test = df_pos.text[test_index.tolist()]\n",
    "    X_test = X_test.append(df_neg.text[test_index.tolist()])\n",
    "    X_test.index = range(len(X_test.index))\n",
    "    \n",
    "    y_test = df_pos.target[test_index.tolist()]\n",
    "    y_test = y_test.append(df_neg.target[test_index.tolist()])\n",
    "    y_test.index = range(len(y_test.index))\n",
    "    \n",
    "    return(X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Metrics(Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.val_f1s = []\n",
    "        self.val_recalls = []\n",
    "        self.val_precisions = []\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        val_predict = (np.asarray(self.model.predict(self.model.validation_data[0]))).round()\n",
    "        val_targ = self.model.validation_data[1]\n",
    "        _val_f1 = f1_score(val_targ, val_predict)\n",
    "        _val_recall = recall_score(val_targ, val_predict)\n",
    "        _val_precision = precision_score(val_targ, val_predict)\n",
    "        self.val_f1s.append(_val_f1)\n",
    "        self.val_recalls.append(_val_recall)\n",
    "        self.val_precisions.append(_val_precision)\n",
    "        #print(\" — val_f1: %f — val_precision: %f — val_recall %f\" %(_val_f1, _val_precision, _val_recall))\n",
    "        return\n",
    "\n",
    "metrics = Metrics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 7: Constructing CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================\n",
      "\n",
      "Fold 1\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 51s 47ms/step - loss: 0.6695 - accuracy: 0.5954 - val_loss: 0.5613 - val_accuracy: 0.7450\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 48s 44ms/step - loss: 0.6074 - accuracy: 0.6546 - val_loss: 0.5700 - val_accuracy: 0.7150\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 38s 36ms/step - loss: 0.5639 - accuracy: 0.6861 - val_loss: 0.5272 - val_accuracy: 0.7500\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 40s 37ms/step - loss: 0.5582 - accuracy: 0.7130 - val_loss: 0.4729 - val_accuracy: 0.7550\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 40s 37ms/step - loss: 0.5273 - accuracy: 0.7306 - val_loss: 0.3479 - val_accuracy: 0.8400\n",
      "Validation Confustion Matrix\n",
      "[[79 21]\n",
      " [11 89]]\n",
      "Test Confustion Matrix\n",
      "[[46 14]\n",
      " [12 48]]\n",
      "=========================================\n",
      "\n",
      "Fold 2\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 39s 36ms/step - loss: 0.6612 - accuracy: 0.5981 - val_loss: 0.6426 - val_accuracy: 0.6250\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 42s 39ms/step - loss: 0.6174 - accuracy: 0.6611 - val_loss: 0.5433 - val_accuracy: 0.7300\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 48s 44ms/step - loss: 0.5718 - accuracy: 0.7074 - val_loss: 0.4826 - val_accuracy: 0.7650\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 45s 42ms/step - loss: 0.5478 - accuracy: 0.7324 - val_loss: 0.4836 - val_accuracy: 0.7500\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 44s 41ms/step - loss: 0.5150 - accuracy: 0.7398 - val_loss: 0.4109 - val_accuracy: 0.8350\n",
      "Validation Confustion Matrix\n",
      "[[84 16]\n",
      " [17 83]]\n",
      "Test Confustion Matrix\n",
      "[[34 26]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 3\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 69s 63ms/step - loss: 0.6855 - accuracy: 0.5713 - val_loss: 0.5640 - val_accuracy: 0.7450\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 91s 84ms/step - loss: 0.6179 - accuracy: 0.6519 - val_loss: 0.5073 - val_accuracy: 0.7800\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 52s 48ms/step - loss: 0.6019 - accuracy: 0.6750 - val_loss: 0.4945 - val_accuracy: 0.7500\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 49s 46ms/step - loss: 0.5623 - accuracy: 0.7148 - val_loss: 0.4071 - val_accuracy: 0.8050\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 58s 54ms/step - loss: 0.5564 - accuracy: 0.7056 - val_loss: 0.4152 - val_accuracy: 0.8250\n",
      "Validation Confustion Matrix\n",
      "[[94  6]\n",
      " [29 71]]\n",
      "Test Confustion Matrix\n",
      "[[49 11]\n",
      " [13 47]]\n",
      "=========================================\n",
      "\n",
      "Fold 4\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 60s 55ms/step - loss: 0.6779 - accuracy: 0.5593 - val_loss: 0.6293 - val_accuracy: 0.6300\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 58s 54ms/step - loss: 0.6128 - accuracy: 0.6565 - val_loss: 0.5126 - val_accuracy: 0.7500\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 49s 45ms/step - loss: 0.5792 - accuracy: 0.7037 - val_loss: 0.4635 - val_accuracy: 0.8000\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 57s 53ms/step - loss: 0.5617 - accuracy: 0.7028 - val_loss: 0.4407 - val_accuracy: 0.8000\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 60s 56ms/step - loss: 0.5162 - accuracy: 0.7417 - val_loss: 0.3666 - val_accuracy: 0.8450\n",
      "Validation Confustion Matrix\n",
      "[[80 20]\n",
      " [11 89]]\n",
      "Test Confustion Matrix\n",
      "[[44 16]\n",
      " [25 35]]\n",
      "=========================================\n",
      "\n",
      "Fold 5\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 44s 41ms/step - loss: 0.6758 - accuracy: 0.5769 - val_loss: 0.5736 - val_accuracy: 0.6800\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 37s 34ms/step - loss: 0.6085 - accuracy: 0.6750 - val_loss: 0.4953 - val_accuracy: 0.7550\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 50s 46ms/step - loss: 0.5751 - accuracy: 0.6981 - val_loss: 0.4464 - val_accuracy: 0.7800\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 39s 36ms/step - loss: 0.5474 - accuracy: 0.7176 - val_loss: 0.4184 - val_accuracy: 0.7950\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5292 - accuracy: 0.7306 - val_loss: 0.3378 - val_accuracy: 0.8500\n",
      "Validation Confustion Matrix\n",
      "[[83 17]\n",
      " [13 87]]\n",
      "Test Confustion Matrix\n",
      "[[45 15]\n",
      " [27 33]]\n",
      "=========================================\n",
      "\n",
      "Fold 6\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6740 - accuracy: 0.5741 - val_loss: 0.6028 - val_accuracy: 0.6600\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.6166 - accuracy: 0.6519 - val_loss: 0.5166 - val_accuracy: 0.7300\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5986 - accuracy: 0.6898 - val_loss: 0.4407 - val_accuracy: 0.7700\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.5712 - accuracy: 0.7037 - val_loss: 0.4062 - val_accuracy: 0.8350\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 23s 22ms/step - loss: 0.5296 - accuracy: 0.7287 - val_loss: 0.3443 - val_accuracy: 0.8700\n",
      "Validation Confustion Matrix\n",
      "[[86 14]\n",
      " [12 88]]\n",
      "Test Confustion Matrix\n",
      "[[46 14]\n",
      " [24 36]]\n",
      "=========================================\n",
      "\n",
      "Fold 7\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6828 - accuracy: 0.5704 - val_loss: 0.5811 - val_accuracy: 0.6600\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 22ms/step - loss: 0.6179 - accuracy: 0.6454 - val_loss: 0.5232 - val_accuracy: 0.7300\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6026 - accuracy: 0.6713 - val_loss: 0.4809 - val_accuracy: 0.7800\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5704 - accuracy: 0.7009 - val_loss: 0.4103 - val_accuracy: 0.8300\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5533 - accuracy: 0.7167 - val_loss: 0.3638 - val_accuracy: 0.8750\n",
      "Validation Confustion Matrix\n",
      "[[88 12]\n",
      " [13 87]]\n",
      "Test Confustion Matrix\n",
      "[[52  8]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 8\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6947 - accuracy: 0.5370 - val_loss: 0.5660 - val_accuracy: 0.7300\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6253 - accuracy: 0.6546 - val_loss: 0.5062 - val_accuracy: 0.7500\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5891 - accuracy: 0.6824 - val_loss: 0.4925 - val_accuracy: 0.7650\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5609 - accuracy: 0.7231 - val_loss: 0.4327 - val_accuracy: 0.8100\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5383 - accuracy: 0.7306 - val_loss: 0.3733 - val_accuracy: 0.8400\n",
      "Validation Confustion Matrix\n",
      "[[92  8]\n",
      " [24 76]]\n",
      "Test Confustion Matrix\n",
      "[[42 18]\n",
      " [23 37]]\n",
      "=========================================\n",
      "\n",
      "Fold 9\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6687 - accuracy: 0.5991 - val_loss: 0.6688 - val_accuracy: 0.6000\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 22ms/step - loss: 0.6047 - accuracy: 0.6806 - val_loss: 0.5971 - val_accuracy: 0.7000\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 23s 22ms/step - loss: 0.5776 - accuracy: 0.7019 - val_loss: 0.5578 - val_accuracy: 0.7150\n",
      "Epoch 4/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5648 - accuracy: 0.6954 - val_loss: 0.5079 - val_accuracy: 0.7550\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 23s 22ms/step - loss: 0.5425 - accuracy: 0.7370 - val_loss: 0.4990 - val_accuracy: 0.7600\n",
      "Validation Confustion Matrix\n",
      "[[71 29]\n",
      " [19 81]]\n",
      "Test Confustion Matrix\n",
      "[[32 28]\n",
      " [15 45]]\n",
      "=========================================\n",
      "\n",
      "Fold 10\n",
      "====================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 24s 23ms/step - loss: 0.6760 - accuracy: 0.5722 - val_loss: 0.6019 - val_accuracy: 0.6750\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.6010 - accuracy: 0.6759 - val_loss: 0.5824 - val_accuracy: 0.7100\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5719 - accuracy: 0.7019 - val_loss: 0.5662 - val_accuracy: 0.6900\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5574 - accuracy: 0.7102 - val_loss: 0.5834 - val_accuracy: 0.7000\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5247 - accuracy: 0.7352 - val_loss: 0.5096 - val_accuracy: 0.7400\n",
      "Validation Confustion Matrix\n",
      "[[60 40]\n",
      " [12 88]]\n",
      "Test Confustion Matrix\n",
      "[[30 30]\n",
      " [ 9 51]]\n",
      "##################################################################\n",
      "Test Results\n",
      "##################################################################\n",
      "Accuracy:  0.69 \tPrecision:  0.7016945100820251 \tRecall:  0.68 \tROC:  0.69\n",
      "Actual Positive:  600 \tPredictedPositive(TP):  420 \tFP:  180\n",
      "Actual Negative:  600 \tPredictedNegative(TN):  408 \tFN:  192\n",
      "F-measure: 0.6906769381713049\n"
     ]
    }
   ],
   "source": [
    "size = 300\n",
    "counter = 0\n",
    "best_test_accuracy = []\n",
    "after_best_test_accuracy = []\n",
    "df_pos = df_pos[0:600]\n",
    "df_neg = df_neg[0:600]\n",
    "predictedPositive = []\n",
    "predictedNegative = []\n",
    "accuracy = []\n",
    "precision= []\n",
    "recall = []\n",
    "roc = [] \n",
    "predictedPositive_V = []\n",
    "predictedNegative_V = []\n",
    "accuracy_V = []\n",
    "precision_V= []\n",
    "recall_V = []\n",
    "roc_V = [] \n",
    "for train_index, test_index in kf.split(df_pos):\n",
    "    counter = counter +1\n",
    "    print(\"=========================================\\n\")\n",
    "    print(\"Fold \"+str(counter)+\"\\n====================================\\n\")\n",
    "\n",
    "    x_train,y_train,x_test,y_test = get_train_test(df_pos,df_neg,train_index,test_index)\n",
    "    \n",
    "    x_validation = validation_df['text']\n",
    "    y_validation = validation_df['target']\n",
    "      \n",
    "    tokenizer = Tokenizer(num_words=100000)\n",
    "    tokenizer.fit_on_texts(x_train)\n",
    "    sequences = tokenizer.texts_to_sequences(x_train)\n",
    "\n",
    "    length = []\n",
    "    for x in x_train:\n",
    "        length.append(len(x.split()))\n",
    "    ml = max(length)+10\n",
    "    \n",
    "    x_train_seq = pad_sequences(sequences, maxlen=ml)\n",
    "    \n",
    "    sequences_val = tokenizer.texts_to_sequences(x_validation)\n",
    "    x_val_seq = pad_sequences(sequences_val, maxlen=ml)\n",
    "    \n",
    "    sequences_test = tokenizer.texts_to_sequences(x_test)\n",
    "    x_test_seq = pad_sequences(sequences_test, maxlen=ml)\n",
    "    \n",
    "    num_words = 100000\n",
    "    embedding_matrix = np.zeros((num_words, size))\n",
    "    for word, i in tokenizer.word_index.items():\n",
    "        if i >= num_words:\n",
    "            continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    \n",
    "    \n",
    "    #tweet_input = Input(shape=(ml,), dtype='int32')\n",
    "    tweet_encoder = Embedding(100000, size, weights=[embedding_matrix], input_length=ml, trainable=True)\n",
    "    \n",
    "    ######################LSTM##########\n",
    "    lstm_out = 300\n",
    "    model = Sequential()\n",
    "    model.add(tweet_encoder)\n",
    "    model.add(SpatialDropout1D(0.5))\n",
    "    model.add(LSTM(lstm_out, dropout=0.5, recurrent_dropout=0.5))\n",
    "    model.add(Dense(1,activation='sigmoid'))\n",
    "    #model.compile(loss = 'categorical_crossentropy', optimizer='adam',metrics = ['accuracy'])\n",
    "\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    #print(model.summary())\n",
    "    ####################################\n",
    "    #model.summary()\n",
    "    filepath=\"CNN_best_weights.{epoch:02d}-{val_acc:.4f}.hdf5\"\n",
    "    \n",
    "    #accuracy = []\n",
    "    #class TestCallback(Callback):\n",
    "        #def on_epoch_end(self, epoch, logs={}):\n",
    "            #accuracy.append(logs.get('val_acc'))\n",
    "    test_accuracy = []\n",
    "    test_loss = []\n",
    "    class TestCallback(Callback):\n",
    "        def __init__(self, test_data):\n",
    "            self.test_data = test_data\n",
    "\n",
    "        def on_epoch_end(self, epoch, logs={}):\n",
    "            x, y = self.test_data\n",
    "            loss, acc = self.model.evaluate(x, y, verbose=0)\n",
    "            #print('\\nacc: {}\\n'.format(acc))\n",
    "            test_accuracy.append(acc)\n",
    "            test_loss.append(loss)\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=2, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "    model.fit(x_train_seq, y_train, batch_size=32, epochs=5,\n",
    "                         validation_data=(x_val_seq, y_validation), callbacks=[TestCallback((x_test_seq, y_test))])\n",
    "    loss, acc = model.evaluate(x_test_seq, y_test, verbose=0)\n",
    "    ############################################################################\n",
    "    predictions = (np.asarray(model.predict(x_val_seq))).round()\n",
    "    roc_V.append(roc_auc_score(y_validation, predictions))\n",
    "    accuracy_V.append(accuracy_score(y_validation, predictions, normalize=True))\n",
    "    precision_V.append(precision_score(y_validation, predictions))\n",
    "    recall_V.append(recall_score(y_validation, predictions))\n",
    "    result = pd.DataFrame.from_items([(\"Actual\",y_validation),(\"Prediction\",predictions.tolist())])\n",
    "    print(\"Validation Confustion Matrix\")\n",
    "    cm = confusion_matrix(y_validation, predictions)\n",
    "    print(cm)\n",
    "    pp = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==0))]\n",
    "    nn = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==1))]\n",
    "    predictedPositive_V.append(len(pp))\n",
    "    predictedNegative_V.append(len(nn))\n",
    "    ##############################################################################\n",
    "    ############################################################################\n",
    "    predictions = (np.asarray(model.predict(x_test_seq))).round()\n",
    "    roc.append(roc_auc_score(y_test, predictions))\n",
    "    accuracy.append(accuracy_score(y_test, predictions, normalize=True))\n",
    "    precision.append(precision_score(y_test, predictions))\n",
    "    recall.append(recall_score(y_test, predictions))\n",
    "    result = pd.DataFrame.from_items([(\"Actual\",y_test),(\"Prediction\",predictions.tolist())])\n",
    "    print(\"Test Confustion Matrix\")\n",
    "    cm = confusion_matrix(y_test, predictions)\n",
    "    print(cm)\n",
    "    pp = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==0))]\n",
    "    nn = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==1))]\n",
    "    predictedPositive.append(len(pp))\n",
    "    predictedNegative.append(len(nn))\n",
    "    ##############################################################################\n",
    "    #print('\\nAfter 5 epoch Testing loss: {}, acc: {}\\n'.format(loss, acc))\n",
    "    after_best_test_accuracy.append(acc)\n",
    "    best_test_accuracy.append(max(test_accuracy))    \n",
    "    \n",
    "\n",
    "#######################################################################\n",
    "'''print(\"##################################################################\")\n",
    "print(\"Validation Results\")\n",
    "print(\"##################################################################\")\n",
    "a = sum(accuracy_V)/10\n",
    "p = sum(precision_V)/10\n",
    "r = sum(recall_V)/10\n",
    "ro = sum(roc_V)/10\n",
    "print(\"Accuracy: \",a,\"\\tPrecision: \",p,\"\\tRecall: \",r,\"\\tROC: \",ro)\n",
    "TP = sum(predictedPositive_V)/10\n",
    "FP = 100-TP\n",
    "TN = sum(predictedNegative_V)/10\n",
    "FN = 100-TN\n",
    "print(\"Actual Positive: \",len(df_pos),\"\\tPredictedPositive(TP): \",TP,\"\\tFP: \",FP)\n",
    "print(\"Actual Negative: \",len(df_neg),\"\\tPredictedNegative(TN): \",TN,\"\\tFN: \",FN)\n",
    "F_measure = (2 * p * r)/(p + r)\n",
    "print(\"F-measure: \"+str(F_measure))'''\n",
    "##############################################################################\n",
    "#######################################################################\n",
    "print(\"##################################################################\")\n",
    "print(\"Test Results\")\n",
    "print(\"##################################################################\")\n",
    "a = sum(accuracy)/10\n",
    "p = sum(precision)/10\n",
    "r = sum(recall)/10\n",
    "ro = sum(roc)/10\n",
    "print(\"Accuracy: \",a,\"\\tPrecision: \",p,\"\\tRecall: \",r,\"\\tROC: \",ro)\n",
    "TP = sum(predictedPositive)\n",
    "FP = (len(df_pos)-sum(predictedPositive))\n",
    "TN = sum(predictedNegative)\n",
    "FN = (len(df_neg)-sum(predictedNegative))\n",
    "print(\"Actual Positive: \",len(df_pos),\"\\tPredictedPositive(TP): \",TP,\"\\tFP: \",FP)\n",
    "print(\"Actual Negative: \",len(df_neg),\"\\tPredictedNegative(TN): \",TN,\"\\tFN: \",FN)\n",
    "F_measure = (2 * p * r)/(p + r)\n",
    "print(\"F-measure: \"+str(F_measure))\n",
    "##############################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results of LSTM_PRETRAINED WORD2VEC MODEL"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========================================\n",
    "\n",
    "Fold 1\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 28s 26ms/step - loss: 0.6881 - accuracy: 0.5463 - val_loss: 0.6542 - val_accuracy: 0.7500\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 29s 26ms/step - loss: 0.5962 - accuracy: 0.6991 - val_loss: 0.4629 - val_accuracy: 0.9000\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 32s 30ms/step - loss: 0.4497 - accuracy: 0.8269 - val_loss: 0.2599 - val_accuracy: 0.9450\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 31s 28ms/step - loss: 0.2965 - accuracy: 0.8972 - val_loss: 0.1809 - val_accuracy: 0.9650\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.1989 - accuracy: 0.9343 - val_loss: 0.0946 - val_accuracy: 0.9850\n",
    "Validation Confustion Matrix\n",
    "[[98  2]\n",
    " [ 1 99]]\n",
    "Test Confustion Matrix\n",
    "[[40 20]\n",
    " [19 41]]\n",
    "=========================================\n",
    "\n",
    "Fold 2\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 27s 25ms/step - loss: 0.6752 - accuracy: 0.5833 - val_loss: 0.6320 - val_accuracy: 0.5900\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5850 - accuracy: 0.7296 - val_loss: 0.4772 - val_accuracy: 0.8650\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.4221 - accuracy: 0.8241 - val_loss: 0.2163 - val_accuracy: 0.9500\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2712 - accuracy: 0.8954 - val_loss: 0.1310 - val_accuracy: 0.9600\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1792 - accuracy: 0.9398 - val_loss: 0.0893 - val_accuracy: 0.9700\n",
    "Validation Confustion Matrix\n",
    "[[95  5]\n",
    " [ 1 99]]\n",
    "Test Confustion Matrix\n",
    "[[31 29]\n",
    " [12 48]]\n",
    "=========================================\n",
    "\n",
    "Fold 3\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 25s 24ms/step - loss: 0.6884 - accuracy: 0.5435 - val_loss: 0.6566 - val_accuracy: 0.6950\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6287 - accuracy: 0.6778 - val_loss: 0.4613 - val_accuracy: 0.8550\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.4689 - accuracy: 0.8148 - val_loss: 0.3560 - val_accuracy: 0.9150\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.3195 - accuracy: 0.8898 - val_loss: 0.1896 - val_accuracy: 0.9650\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2060 - accuracy: 0.9380 - val_loss: 0.0853 - val_accuracy: 0.9800\n",
    "Validation Confustion Matrix\n",
    "[[98  2]\n",
    " [ 2 98]]\n",
    "Test Confustion Matrix\n",
    "[[48 12]\n",
    " [ 8 52]]\n",
    "=========================================\n",
    "\n",
    "Fold 4\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6822 - accuracy: 0.5519 - val_loss: 0.6341 - val_accuracy: 0.6900\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6113 - accuracy: 0.6778 - val_loss: 0.4684 - val_accuracy: 0.7550\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.4848 - accuracy: 0.7981 - val_loss: 0.2654 - val_accuracy: 0.9250\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2882 - accuracy: 0.8870 - val_loss: 0.1711 - val_accuracy: 0.9550\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1903 - accuracy: 0.9435 - val_loss: 0.0960 - val_accuracy: 0.9800\n",
    "Validation Confustion Matrix\n",
    "[[98  2]\n",
    " [ 2 98]]\n",
    "Test Confustion Matrix\n",
    "[[47 13]\n",
    " [19 41]]\n",
    "=========================================\n",
    "\n",
    "Fold 5\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6838 - accuracy: 0.5333 - val_loss: 0.6584 - val_accuracy: 0.6850\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6024 - accuracy: 0.7046 - val_loss: 0.5303 - val_accuracy: 0.7700\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.4613 - accuracy: 0.8120 - val_loss: 0.2933 - val_accuracy: 0.8950\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.3022 - accuracy: 0.9009 - val_loss: 0.1807 - val_accuracy: 0.9800\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1761 - accuracy: 0.9509 - val_loss: 0.0992 - val_accuracy: 0.9800\n",
    "Validation Confustion Matrix\n",
    "[[97  3]\n",
    " [ 1 99]]\n",
    "Test Confustion Matrix\n",
    "[[45 15]\n",
    " [22 38]]\n",
    "=========================================\n",
    "\n",
    "Fold 6\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6884 - accuracy: 0.5389 - val_loss: 0.6590 - val_accuracy: 0.7200\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6209 - accuracy: 0.6870 - val_loss: 0.4780 - val_accuracy: 0.8400\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 23ms/step - loss: 0.4609 - accuracy: 0.8093 - val_loss: 0.2757 - val_accuracy: 0.9250\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.3023 - accuracy: 0.8870 - val_loss: 0.1257 - val_accuracy: 0.9700\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1905 - accuracy: 0.9426 - val_loss: 0.1934 - val_accuracy: 0.9450\n",
    "Validation Confustion Matrix\n",
    "[[97  3]\n",
    " [ 8 92]]\n",
    "Test Confustion Matrix\n",
    "[[50 10]\n",
    " [25 35]]\n",
    "=========================================\n",
    "\n",
    "Fold 7\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6815 - accuracy: 0.5796 - val_loss: 0.6453 - val_accuracy: 0.6100\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6063 - accuracy: 0.6731 - val_loss: 0.4504 - val_accuracy: 0.8350\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.4784 - accuracy: 0.8102 - val_loss: 0.3173 - val_accuracy: 0.9000\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2971 - accuracy: 0.8889 - val_loss: 0.2295 - val_accuracy: 0.9500\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.1987 - accuracy: 0.9426 - val_loss: 0.1016 - val_accuracy: 0.9750\n",
    "Validation Confustion Matrix\n",
    "[[ 95   5]\n",
    " [  0 100]]\n",
    "Test Confustion Matrix\n",
    "[[40 20]\n",
    " [20 40]]\n",
    "=========================================\n",
    "\n",
    "Fold 8\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6866 - accuracy: 0.5444 - val_loss: 0.6434 - val_accuracy: 0.7600\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 23ms/step - loss: 0.6156 - accuracy: 0.6954 - val_loss: 0.4894 - val_accuracy: 0.8750\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.4499 - accuracy: 0.8250 - val_loss: 0.3809 - val_accuracy: 0.9000\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.3024 - accuracy: 0.8991 - val_loss: 0.1318 - val_accuracy: 0.9650\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1717 - accuracy: 0.9444 - val_loss: 0.1047 - val_accuracy: 0.9800\n",
    "Validation Confustion Matrix\n",
    "[[99  1]\n",
    " [ 3 97]]\n",
    "Test Confustion Matrix\n",
    "[[45 15]\n",
    " [27 33]]\n",
    "=========================================\n",
    "\n",
    "Fold 9\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6868 - accuracy: 0.5389 - val_loss: 0.6527 - val_accuracy: 0.6600\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6156 - accuracy: 0.6889 - val_loss: 0.5650 - val_accuracy: 0.7550\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.4574 - accuracy: 0.8111 - val_loss: 0.4454 - val_accuracy: 0.8350\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2855 - accuracy: 0.8963 - val_loss: 0.3460 - val_accuracy: 0.8400\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1759 - accuracy: 0.9435 - val_loss: 0.3370 - val_accuracy: 0.8550\n",
    "Validation Confustion Matrix\n",
    "[[80 20]\n",
    " [ 9 91]]\n",
    "Test Confustion Matrix\n",
    "[[33 27]\n",
    " [14 46]]\n",
    "=========================================\n",
    "\n",
    "Fold 10\n",
    "====================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 28s 26ms/step - loss: 0.6844 - accuracy: 0.5565 - val_loss: 0.6567 - val_accuracy: 0.6350\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6080 - accuracy: 0.7065 - val_loss: 0.5946 - val_accuracy: 0.7000\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.4565 - accuracy: 0.8361 - val_loss: 0.5583 - val_accuracy: 0.7200\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.3016 - accuracy: 0.8852 - val_loss: 0.5411 - val_accuracy: 0.7650\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.1693 - accuracy: 0.9463 - val_loss: 0.5147 - val_accuracy: 0.7750\n",
    "Validation Confustion Matrix\n",
    "[[80 20]\n",
    " [25 75]]\n",
    "Test Confustion Matrix\n",
    "[[40 20]\n",
    " [25 35]]\n",
    "##################################################################\n",
    "Test Results\n",
    "##################################################################\n",
    "Accuracy:  0.6900000000000001 \tPrecision:  0.6982693229361789 \tRecall:  0.6816666666666666 \tROC:  0.6900000000000002\n",
    "Actual Positive:  600 \tPredictedPositive(TP):  419 \tFP:  181\n",
    "Actual Negative:  600 \tPredictedNegative(TN):  409 \tFN:  191\n",
    "F-measure: 0.6898681176341916"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}