CNN_LSTM_TENFOLD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1: Loading necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import sys\n",
    "import warnings\n",
    "from gensim.models.wrappers import FastText\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim import models\n",
    "from sklearn.model_selection import KFold # import KFold\n",
    "from keras.regularizers import l2\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model, Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Input, Dense, concatenate, Activation, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Conv1D, GlobalMaxPooling1D, LSTM, Conv1D, Flatten, MaxPooling1D\n",
    "from keras.callbacks import Callback\n",
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score, accuracy_score\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: Loading pretrained word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ug_cbow = FastText.load_fasttext_format('../cc.ne.300.bin/cc.ne.300.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 584436 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "for w in model_ug_cbow.wv.vocab.keys():\n",
    "    embeddings_index[w] = model_ug_cbow.wv[w]#np.append(model_ug_cbow.wv[w],model_ug_sg.wv[w])#\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3: Loading Trained word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ug_sg = models.Word2Vec.load('../w2v_model_ug_sg.word2vec')\n",
    "model_ug_cbow = models.Word2Vec.load('../w2v_model_ug_cbow.word2vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 42409 word vectors.\n",
      "Found 42409 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "for w in model_ug_cbow.wv.vocab.keys():\n",
    "    embeddings_index[w] = model_ug_cbow.wv[w]#\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 4: Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_excel(\"NepaliEarthquakeTweets_plus_blockade.xlsx\",sheetname=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_indexes = np.where(data['bhawna']=='p')\n",
    "neg_indexes = np.where(data['bhawna']=='n')\n",
    "total_size = 600\n",
    "size = 600\n",
    "val_size = 600\n",
    "df_pos = pd.DataFrame.from_items([('text',data['tweet_text'][pos_indexes[0][0:total_size]]),('target',0)])\n",
    "df_neg = pd.DataFrame.from_items([('text',data['tweet_text'][neg_indexes[0][0:total_size]]),('target',1)])\n",
    "\n",
    "df_pos['reply'] = data['reply'][pos_indexes[0][0:total_size]]\n",
    "df_pos['retweet'] = data['retweet'][pos_indexes[0][0:total_size]]\n",
    "df_pos['likes'] = data['likes'][pos_indexes[0][0:total_size]]\n",
    "\n",
    "df_neg['reply'] = data['reply'][neg_indexes[0][0:total_size]]\n",
    "df_neg['retweet'] = data['retweet'][neg_indexes[0][0:total_size]]\n",
    "df_neg['likes'] = data['likes'][neg_indexes[0][0:total_size]]\n",
    "\n",
    "df_pos.index = range(len(df_pos.index))\n",
    "df_neg.index = range(len(df_neg.index))\n",
    "\n",
    "df_pos_f = df_pos\n",
    "df_neg_f = df_neg\n",
    "df_F = df_pos[0:size]\n",
    "df_F = df_F.append(df_neg[0:size])\n",
    "df_F.index = range(len(df_F.index))\n",
    "\n",
    "len(df_F)\n",
    "validation_df_F= df_pos[500:600]\n",
    "validation_df_F = validation_df_F.append(df_neg[500:600])\n",
    "validation_df_F.index = range(len(validation_df_F.index))\n",
    "#validation_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_pos[0:size]\n",
    "df = df.append(df_neg[0:size])\n",
    "df.index = range(len(df.index))\n",
    "validation_df= df_pos[500:600]\n",
    "validation_df = validation_df.append(df_neg[500:600])\n",
    "validation_df.index = range(len(validation_df.index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 5: Data Preprocessing: Removing puncuation, english text, decimal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pat1 = r'@[A-Za-z0-9]+'\n",
    "pat2 = r'https?://[A-Za-z0-9./]+'\n",
    "pat3 = r'([@#][A-Za-z0-9]+)'\n",
    "pat4 = r'.[A-Za-z0-9./]+'\n",
    "pat5 = r'[\\,۔،۔”–’‘‘_!…।-]|(\")|(:)|(%)|(ः)|(\\u200d)|(\\xa0…)|(\\u200c\\u200c)'\n",
    "combined_pat = r'|'.join((pat1, pat2,pat3,pat4,pat5))\n",
    "df['text'] = [re.sub(combined_pat, ' ', x) for x in df['text']]\n",
    "validation_df['text'] = [re.sub(combined_pat, ' ', x) for x in validation_df['text']]\n",
    "df_pos['text'] = [re.sub(combined_pat, ' ', x) for x in df_pos['text']]\n",
    "df_neg['text'] = [re.sub(combined_pat, ' ', x) for x in df_neg['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFold(n_splits=10, random_state=None, shuffle=False)\n",
      "KFold(n_splits=10, random_state=None, shuffle=False)\n"
     ]
    }
   ],
   "source": [
    "kf = KFold(n_splits=10)\n",
    "print(kf) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 6: Define Necessary funtions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test(df_pos,df_neg,train_index,test_index):\n",
    "    X_train = df_pos.text[train_index.tolist()]\n",
    "    X_train = X_train.append(df_neg.text[train_index.tolist()])\n",
    "    X_train.index = range(len(X_train.index))\n",
    "    \n",
    "    y_train = df_pos.target[train_index.tolist()]\n",
    "    y_train = y_train.append(df_neg.target[train_index.tolist()])\n",
    "    y_train.index = range(len(y_train.index))\n",
    "    \n",
    "    X_test = df_pos.text[test_index.tolist()]\n",
    "    X_test = X_test.append(df_neg.text[test_index.tolist()])\n",
    "    X_test.index = range(len(X_test.index))\n",
    "    \n",
    "    y_test = df_pos.target[test_index.tolist()]\n",
    "    y_test = y_test.append(df_neg.target[test_index.tolist()])\n",
    "    y_test.index = range(len(y_test.index))\n",
    "    \n",
    "    return(X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Metrics(Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        self.val_f1s = []\n",
    "        self.val_recalls = []\n",
    "        self.val_precisions = []\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        val_predict = (np.asarray(self.model.predict(self.model.validation_data[0]))).round()\n",
    "        val_targ = self.model.validation_data[1]\n",
    "        _val_f1 = f1_score(val_targ, val_predict)\n",
    "        _val_recall = recall_score(val_targ, val_predict)\n",
    "        _val_precision = precision_score(val_targ, val_predict)\n",
    "        self.val_f1s.append(_val_f1)\n",
    "        self.val_recalls.append(_val_recall)\n",
    "        self.val_precisions.append(_val_precision)\n",
    "        #print(\" — val_f1: %f — val_precision: %f — val_recall %f\" %(_val_f1, _val_precision, _val_recall))\n",
    "        return\n",
    "\n",
    "metrics = Metrics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 7: Constructing CNN-LSTM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================\n",
      "\n",
      "Fold 1\n",
      "=========================================\n",
      "\n",
      "=========================================\n",
      "\n",
      "Fold 1\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6623 - accuracy: 0.5889 - val_loss: 0.5581 - val_accuracy: 0.7200\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6623 - accuracy: 0.5889 - val_loss: 0.5581 - val_accuracy: 0.7200\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.5145 - accuracy: 0.7657 - val_loss: 0.4299 - val_accuracy: 0.8100\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.5145 - accuracy: 0.7657 - val_loss: 0.4299 - val_accuracy: 0.8100\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.3563 - accuracy: 0.8537 - val_loss: 0.2190 - val_accuracy: 0.9200\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.3563 - accuracy: 0.8537 - val_loss: 0.2190 - val_accuracy: 0.9200\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.2705 - accuracy: 0.8880 - val_loss: 0.1383 - val_accuracy: 0.9600\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.2705 - accuracy: 0.8880 - val_loss: 0.1383 - val_accuracy: 0.9600\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1956 - accuracy: 0.9259 - val_loss: 0.0713 - val_accuracy: 0.9850\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1956 - accuracy: 0.9259 - val_loss: 0.0713 - val_accuracy: 0.9850\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[30 30]\n",
      " [ 9 51]]\n",
      "=========================================\n",
      "\n",
      "Fold 2\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[30 30]\n",
      " [ 9 51]]\n",
      "=========================================\n",
      "\n",
      "Fold 2\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6728 - accuracy: 0.5907 - val_loss: 0.5633 - val_accuracy: 0.7150\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6728 - accuracy: 0.5907 - val_loss: 0.5633 - val_accuracy: 0.7150\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.5379 - accuracy: 0.7278 - val_loss: 0.3790 - val_accuracy: 0.8350\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.5379 - accuracy: 0.7278 - val_loss: 0.3790 - val_accuracy: 0.8350\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 19s 18ms/step - loss: 0.3880 - accuracy: 0.8324 - val_loss: 0.2164 - val_accuracy: 0.9300\n",
      "1080/1080 [==============================] - 19s 18ms/step - loss: 0.3880 - accuracy: 0.8324 - val_loss: 0.2164 - val_accuracy: 0.9300\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.3036 - accuracy: 0.8806 - val_loss: 0.1311 - val_accuracy: 0.9650\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.3036 - accuracy: 0.8806 - val_loss: 0.1311 - val_accuracy: 0.9650\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1894 - accuracy: 0.9259 - val_loss: 0.0492 - val_accuracy: 0.9950\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1894 - accuracy: 0.9259 - val_loss: 0.0492 - val_accuracy: 0.9950\n",
      "Validation Confustion Matrix\n",
      "[[ 99   1]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[36 24]\n",
      " [21 39]]\n",
      "=========================================\n",
      "\n",
      "Fold 3\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 99   1]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[36 24]\n",
      " [21 39]]\n",
      "=========================================\n",
      "\n",
      "Fold 3\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6801 - accuracy: 0.5491 - val_loss: 0.5486 - val_accuracy: 0.7600\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6801 - accuracy: 0.5491 - val_loss: 0.5486 - val_accuracy: 0.7600\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.5730 - accuracy: 0.6880 - val_loss: 0.4224 - val_accuracy: 0.8200\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.5730 - accuracy: 0.6880 - val_loss: 0.4224 - val_accuracy: 0.8200\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.4384 - accuracy: 0.8046 - val_loss: 0.2617 - val_accuracy: 0.9000\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.4384 - accuracy: 0.8046 - val_loss: 0.2617 - val_accuracy: 0.9000\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.3508 - accuracy: 0.8620 - val_loss: 0.1373 - val_accuracy: 0.9750\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.3508 - accuracy: 0.8620 - val_loss: 0.1373 - val_accuracy: 0.9750\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.2045 - accuracy: 0.9250 - val_loss: 0.0671 - val_accuracy: 0.9850\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.2045 - accuracy: 0.9250 - val_loss: 0.0671 - val_accuracy: 0.9850\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[43 17]\n",
      " [ 9 51]]\n",
      "=========================================\n",
      "\n",
      "Fold 4\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[43 17]\n",
      " [ 9 51]]\n",
      "=========================================\n",
      "\n",
      "Fold 4\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.6913 - accuracy: 0.5398 - val_loss: 0.5961 - val_accuracy: 0.6700\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.6913 - accuracy: 0.5398 - val_loss: 0.5961 - val_accuracy: 0.6700\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.5718 - accuracy: 0.7056 - val_loss: 0.3998 - val_accuracy: 0.8350\n",
      "1080/1080 [==============================] - 20s 18ms/step - loss: 0.5718 - accuracy: 0.7056 - val_loss: 0.3998 - val_accuracy: 0.8350\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 19s 18ms/step - loss: 0.3970 - accuracy: 0.8241 - val_loss: 0.3420 - val_accuracy: 0.8400\n",
      "1080/1080 [==============================] - 19s 18ms/step - loss: 0.3970 - accuracy: 0.8241 - val_loss: 0.3420 - val_accuracy: 0.8400\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.3004 - accuracy: 0.8880 - val_loss: 0.1341 - val_accuracy: 0.9700\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.3004 - accuracy: 0.8880 - val_loss: 0.1341 - val_accuracy: 0.9700\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.2123 - accuracy: 0.9269 - val_loss: 0.0909 - val_accuracy: 0.9850\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.2123 - accuracy: 0.9269 - val_loss: 0.0909 - val_accuracy: 0.9850\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[46 14]\n",
      " [21 39]]\n",
      "=========================================\n",
      "\n",
      "Fold 5\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 97   3]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[46 14]\n",
      " [21 39]]\n",
      "=========================================\n",
      "\n",
      "Fold 5\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.6901 - accuracy: 0.5574 - val_loss: 0.5914 - val_accuracy: 0.7350\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.6901 - accuracy: 0.5574 - val_loss: 0.5914 - val_accuracy: 0.7350\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5573 - accuracy: 0.7093 - val_loss: 0.3778 - val_accuracy: 0.8400\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5573 - accuracy: 0.7093 - val_loss: 0.3778 - val_accuracy: 0.8400\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.4296 - accuracy: 0.7954 - val_loss: 0.2976 - val_accuracy: 0.8950\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.4296 - accuracy: 0.7954 - val_loss: 0.2976 - val_accuracy: 0.8950\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.3178 - accuracy: 0.8750 - val_loss: 0.1091 - val_accuracy: 0.9750\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.3178 - accuracy: 0.8750 - val_loss: 0.1091 - val_accuracy: 0.9750\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.1960 - accuracy: 0.9259 - val_loss: 0.1018 - val_accuracy: 0.9750\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.1960 - accuracy: 0.9259 - val_loss: 0.1018 - val_accuracy: 0.9750\n",
      "Validation Confustion Matrix\n",
      "[[ 95   5]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[39 21]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 6\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 95   5]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[39 21]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 6\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.6718 - accuracy: 0.5806 - val_loss: 0.5606 - val_accuracy: 0.7250\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.6718 - accuracy: 0.5806 - val_loss: 0.5606 - val_accuracy: 0.7250\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5424 - accuracy: 0.7250 - val_loss: 0.4243 - val_accuracy: 0.8200\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5424 - accuracy: 0.7250 - val_loss: 0.4243 - val_accuracy: 0.8200\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.4412 - accuracy: 0.8028 - val_loss: 0.3061 - val_accuracy: 0.8950\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.4412 - accuracy: 0.8028 - val_loss: 0.3061 - val_accuracy: 0.8950\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 27s 25ms/step - loss: 0.3138 - accuracy: 0.8722 - val_loss: 0.1633 - val_accuracy: 0.9600\n",
      "1080/1080 [==============================] - 27s 25ms/step - loss: 0.3138 - accuracy: 0.8722 - val_loss: 0.1633 - val_accuracy: 0.9600\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.2097 - accuracy: 0.9185 - val_loss: 0.0905 - val_accuracy: 0.9850\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.2097 - accuracy: 0.9185 - val_loss: 0.0905 - val_accuracy: 0.9850\n",
      "Validation Confustion Matrix\n",
      "[[99  1]\n",
      " [ 2 98]]\n",
      "Test Confustion Matrix\n",
      "[[53  7]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 7\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[99  1]\n",
      " [ 2 98]]\n",
      "Test Confustion Matrix\n",
      "[[53  7]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 7\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 27s 25ms/step - loss: 0.6623 - accuracy: 0.5935 - val_loss: 0.5503 - val_accuracy: 0.7350\n",
      "1080/1080 [==============================] - 27s 25ms/step - loss: 0.6623 - accuracy: 0.5935 - val_loss: 0.5503 - val_accuracy: 0.7350\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5526 - accuracy: 0.7139 - val_loss: 0.4801 - val_accuracy: 0.7650\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5526 - accuracy: 0.7139 - val_loss: 0.4801 - val_accuracy: 0.7650\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.4211 - accuracy: 0.8065 - val_loss: 0.2574 - val_accuracy: 0.8900\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.4211 - accuracy: 0.8065 - val_loss: 0.2574 - val_accuracy: 0.8900\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.2849 - accuracy: 0.8861 - val_loss: 0.1075 - val_accuracy: 0.9750\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.2849 - accuracy: 0.8861 - val_loss: 0.1075 - val_accuracy: 0.9750\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.2097 - accuracy: 0.9250 - val_loss: 0.0584 - val_accuracy: 0.9900\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.2097 - accuracy: 0.9250 - val_loss: 0.0584 - val_accuracy: 0.9900\n",
      "Validation Confustion Matrix\n",
      "[[ 98   2]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[45 15]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 8\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 98   2]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[45 15]\n",
      " [22 38]]\n",
      "=========================================\n",
      "\n",
      "Fold 8\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6844 - accuracy: 0.5574 - val_loss: 0.6100 - val_accuracy: 0.6750\n",
      "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6844 - accuracy: 0.5574 - val_loss: 0.6100 - val_accuracy: 0.6750\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5763 - accuracy: 0.6954 - val_loss: 0.4094 - val_accuracy: 0.8150\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5763 - accuracy: 0.6954 - val_loss: 0.4094 - val_accuracy: 0.8150\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.4097 - accuracy: 0.8157 - val_loss: 0.2669 - val_accuracy: 0.8650\n",
      "1080/1080 [==============================] - 22s 20ms/step - loss: 0.4097 - accuracy: 0.8157 - val_loss: 0.2669 - val_accuracy: 0.8650\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.3014 - accuracy: 0.8704 - val_loss: 0.1656 - val_accuracy: 0.9500\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.3014 - accuracy: 0.8704 - val_loss: 0.1656 - val_accuracy: 0.9500\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.1983 - accuracy: 0.9250 - val_loss: 0.0511 - val_accuracy: 0.9950\n",
      "1080/1080 [==============================] - 21s 20ms/step - loss: 0.1983 - accuracy: 0.9250 - val_loss: 0.0511 - val_accuracy: 0.9950\n",
      "Validation Confustion Matrix\n",
      "[[ 99   1]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[36 24]\n",
      " [19 41]]\n",
      "=========================================\n",
      "\n",
      "Fold 9\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[ 99   1]\n",
      " [  0 100]]\n",
      "Test Confustion Matrix\n",
      "[[36 24]\n",
      " [19 41]]\n",
      "=========================================\n",
      "\n",
      "Fold 9\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6951 - accuracy: 0.5583 - val_loss: 0.6007 - val_accuracy: 0.6750\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6951 - accuracy: 0.5583 - val_loss: 0.6007 - val_accuracy: 0.6750\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5680 - accuracy: 0.7102 - val_loss: 0.5443 - val_accuracy: 0.7050\n",
      "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5680 - accuracy: 0.7102 - val_loss: 0.5443 - val_accuracy: 0.7050\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.4273 - accuracy: 0.8083 - val_loss: 0.6566 - val_accuracy: 0.6900\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.4273 - accuracy: 0.8083 - val_loss: 0.6566 - val_accuracy: 0.6900\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.2950 - accuracy: 0.8796 - val_loss: 0.3616 - val_accuracy: 0.8400\n",
      "1080/1080 [==============================] - 21s 19ms/step - loss: 0.2950 - accuracy: 0.8796 - val_loss: 0.3616 - val_accuracy: 0.8400\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.2105 - accuracy: 0.9167 - val_loss: 0.3464 - val_accuracy: 0.8450\n",
      "1080/1080 [==============================] - 20s 19ms/step - loss: 0.2105 - accuracy: 0.9167 - val_loss: 0.3464 - val_accuracy: 0.8450\n",
      "Validation Confustion Matrix\n",
      "[[82 18]\n",
      " [13 87]]\n",
      "Test Confustion Matrix\n",
      "[[34 26]\n",
      " [16 44]]\n",
      "=========================================\n",
      "\n",
      "Fold 10\n",
      "=========================================\n",
      "\n",
      "Validation Confustion Matrix\n",
      "[[82 18]\n",
      " [13 87]]\n",
      "Test Confustion Matrix\n",
      "[[34 26]\n",
      " [16 44]]\n",
      "=========================================\n",
      "\n",
      "Fold 10\n",
      "=========================================\n",
      "\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "Train on 1080 samples, validate on 200 samples\n",
      "Epoch 1/5\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.6841 - accuracy: 0.5611 - val_loss: 0.6179 - val_accuracy: 0.6450\n",
      "1080/1080 [==============================] - 23s 21ms/step - loss: 0.6841 - accuracy: 0.5611 - val_loss: 0.6179 - val_accuracy: 0.6450\n",
      "Epoch 2/5\n",
      "Epoch 2/5\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5346 - accuracy: 0.7296 - val_loss: 0.6406 - val_accuracy: 0.6850\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5346 - accuracy: 0.7296 - val_loss: 0.6406 - val_accuracy: 0.6850\n",
      "Epoch 3/5\n",
      "Epoch 3/5\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.4129 - accuracy: 0.8194 - val_loss: 0.6660 - val_accuracy: 0.6750\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.4129 - accuracy: 0.8194 - val_loss: 0.6660 - val_accuracy: 0.6750\n",
      "Epoch 4/5\n",
      "Epoch 4/5\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.2883 - accuracy: 0.8815 - val_loss: 0.6159 - val_accuracy: 0.7600\n",
      "1080/1080 [==============================] - 22s 21ms/step - loss: 0.2883 - accuracy: 0.8815 - val_loss: 0.6159 - val_accuracy: 0.7600\n",
      "Epoch 5/5\n",
      "Epoch 5/5\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.1723 - accuracy: 0.9398 - val_loss: 0.7031 - val_accuracy: 0.7750\n",
      "1080/1080 [==============================] - 25s 23ms/step - loss: 0.1723 - accuracy: 0.9398 - val_loss: 0.7031 - val_accuracy: 0.7750\n",
      "Validation Confustion Matrix\n",
      "[[65 35]\n",
      " [10 90]]\n",
      "Test Confustion Matrix\n",
      "[[25 35]\n",
      " [10 50]]\n",
      "##################################################################\n",
      "Test Results\n",
      "##################################################################\n",
      "Accuracy:  0.6799999999999999 \tPrecision:  0.6787595631869413 \tRecall:  0.7150000000000001 \tROC:  0.6799999999999999\n",
      "Actual Positive:  600 \tPredictedPositive(TP):  387 \tFP:  213\n",
      "Actual Negative:  600 \tPredictedNegative(TN):  429 \tFN:  171\n",
      "F-measure: 0.696408620966096\n",
      "Validation Confustion Matrix\n",
      "[[65 35]\n",
      " [10 90]]\n",
      "Test Confustion Matrix\n",
      "[[25 35]\n",
      " [10 50]]\n",
      "##################################################################\n",
      "Test Results\n",
      "##################################################################\n",
      "Accuracy:  0.6799999999999999 \tPrecision:  0.6787595631869413 \tRecall:  0.7150000000000001 \tROC:  0.6799999999999999\n",
      "Actual Positive:  600 \tPredictedPositive(TP):  387 \tFP:  213\n",
      "Actual Negative:  600 \tPredictedNegative(TN):  429 \tFN:  171\n",
      "F-measure: 0.696408620966096\n"
     ]
    }
   ],
   "source": [
    "size = 300\n",
    "counter = 0\n",
    "best_test_accuracy = []\n",
    "after_best_test_accuracy = []\n",
    "df_pos = df_pos[0:600]\n",
    "df_neg = df_neg[0:600]\n",
    "predictedPositive = []\n",
    "predictedNegative = []\n",
    "accuracy = []\n",
    "precision= []\n",
    "recall = []\n",
    "roc = [] \n",
    "predictedPositive_V = []\n",
    "predictedNegative_V = []\n",
    "accuracy_V = []\n",
    "precision_V= []\n",
    "recall_V = []\n",
    "roc_V = [] \n",
    "for train_index, test_index in kf.split(df_pos):\n",
    "    counter = counter +1\n",
    "    print(\"=========================================\\n\")\n",
    "    print(\"Fold \"+str(counter)+\"\\n=========================================\\n\")\n",
    "\n",
    "    x_train,y_train,x_test,y_test = get_train_test(df_pos,df_neg,train_index,test_index)\n",
    "    \n",
    "    x_validation = validation_df['text']\n",
    "    y_validation = validation_df['target']\n",
    "      \n",
    "    tokenizer = Tokenizer(num_words=100000)\n",
    "    tokenizer.fit_on_texts(x_train)\n",
    "    sequences = tokenizer.texts_to_sequences(x_train)\n",
    "\n",
    "    length = []\n",
    "    for x in x_train:\n",
    "        length.append(len(x.split()))\n",
    "    ml = max(length)+10\n",
    "    \n",
    "    x_train_seq = pad_sequences(sequences, maxlen=ml)\n",
    "    \n",
    "    sequences_val = tokenizer.texts_to_sequences(x_validation)\n",
    "    x_val_seq = pad_sequences(sequences_val, maxlen=ml)\n",
    "    \n",
    "    sequences_test = tokenizer.texts_to_sequences(x_test)\n",
    "    x_test_seq = pad_sequences(sequences_test, maxlen=ml)\n",
    "    \n",
    "    num_words = 100000\n",
    "    embedding_matrix = np.zeros((num_words, size))\n",
    "    for word, i in tokenizer.word_index.items():\n",
    "        if i >= num_words:\n",
    "            continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    \n",
    "    \n",
    "    #tweet_input = Input(shape=(ml,), dtype='int32')\n",
    "    tweet_encoder = Embedding(100000, size, weights=[embedding_matrix], input_length=ml, trainable=True)\n",
    "    \n",
    "    ######################LSTM##########\n",
    "    lstm_out = 300\n",
    "    model = Sequential()\n",
    "    model.add(tweet_encoder)\n",
    "    model.add(Conv1D(filters=32, kernel_size=4, activation='relu', padding='causal'))\n",
    "    model.add(MaxPooling1D(pool_size=2))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(LSTM(units=lstm_out))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    #print(model.summary())\n",
    "    ####################################\n",
    "    #model.summary()\n",
    "    filepath=\"CNN_best_weights.{epoch:02d}-{val_acc:.4f}.hdf5\"\n",
    "    \n",
    "    #accuracy = []\n",
    "    #class TestCallback(Callback):\n",
    "        #def on_epoch_end(self, epoch, logs={}):\n",
    "            #accuracy.append(logs.get('val_acc'))\n",
    "    test_accuracy = []\n",
    "    test_loss = []\n",
    "    class TestCallback(Callback):\n",
    "        def __init__(self, test_data):\n",
    "            self.test_data = test_data\n",
    "\n",
    "        def on_epoch_end(self, epoch, logs={}):\n",
    "            x, y = self.test_data\n",
    "            loss, acc = self.model.evaluate(x, y, verbose=0)\n",
    "            #print('\\nacc: {}\\n'.format(acc))\n",
    "            test_accuracy.append(acc)\n",
    "            test_loss.append(loss)\n",
    "    checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "    callbacks_list = [checkpoint]\n",
    "    model.fit(x_train_seq, y_train, batch_size=32, epochs=5,\n",
    "                         validation_data=(x_val_seq, y_validation), callbacks=[TestCallback((x_test_seq, y_test))])\n",
    "    loss, acc = model.evaluate(x_test_seq, y_test, verbose=0)\n",
    "    ############################################################################\n",
    "    predictions = (np.asarray(model.predict(x_val_seq))).round()\n",
    "    roc_V.append(roc_auc_score(y_validation, predictions))\n",
    "    accuracy_V.append(accuracy_score(y_validation, predictions, normalize=True))\n",
    "    precision_V.append(precision_score(y_validation, predictions))\n",
    "    recall_V.append(recall_score(y_validation, predictions))\n",
    "    result = pd.DataFrame.from_items([(\"Actual\",y_validation),(\"Prediction\",predictions.tolist())])\n",
    "    print(\"Validation Confustion Matrix\")\n",
    "    cm = confusion_matrix(y_validation, predictions)\n",
    "    print(cm)\n",
    "    pp = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==0))]\n",
    "    nn = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==1))]\n",
    "    predictedPositive_V.append(len(pp))\n",
    "    predictedNegative_V.append(len(nn))\n",
    "    ##############################################################################\n",
    "    ############################################################################\n",
    "    predictions = (np.asarray(model.predict(x_test_seq))).round()\n",
    "    roc.append(roc_auc_score(y_test, predictions))\n",
    "    accuracy.append(accuracy_score(y_test, predictions, normalize=True))\n",
    "    precision.append(precision_score(y_test, predictions))\n",
    "    recall.append(recall_score(y_test, predictions))\n",
    "    result = pd.DataFrame.from_items([(\"Actual\",y_test),(\"Prediction\",predictions.tolist())])\n",
    "    print(\"Test Confustion Matrix\")\n",
    "    cm = confusion_matrix(y_test, predictions)\n",
    "    print(cm)\n",
    "    pp = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==0))]\n",
    "    nn = [result['Actual'][x] for x in range(0,len(result)) if ((result['Actual'][x] ==result['Prediction'][x]) and (result['Actual'][x]==1))]\n",
    "    predictedPositive.append(len(pp))\n",
    "    predictedNegative.append(len(nn))\n",
    "    ##############################################################################\n",
    "    #print('\\nAfter 5 epoch Testing loss: {}, acc: {}\\n'.format(loss, acc))\n",
    "    after_best_test_accuracy.append(acc)\n",
    "    best_test_accuracy.append(max(test_accuracy))    \n",
    "    \n",
    "\n",
    "#######################################################################\n",
    "'''print(\"##################################################################\")\n",
    "print(\"Validation Results\")\n",
    "print(\"##################################################################\")\n",
    "a = sum(accuracy_V)/10\n",
    "p = sum(precision_V)/10\n",
    "r = sum(recall_V)/10\n",
    "ro = sum(roc_V)/10\n",
    "print(\"Accuracy: \",a,\"\\tPrecision: \",p,\"\\tRecall: \",r,\"\\tROC: \",ro)\n",
    "TP = sum(predictedPositive_V)/10\n",
    "FP = 100-TP\n",
    "TN = sum(predictedNegative_V)/10\n",
    "FN = 100-TN\n",
    "print(\"Actual Positive: \",len(df_pos),\"\\tPredictedPositive(TP): \",TP,\"\\tFP: \",FP)\n",
    "print(\"Actual Negative: \",len(df_neg),\"\\tPredictedNegative(TN): \",TN,\"\\tFN: \",FN)\n",
    "F_measure = (2 * p * r)/(p + r)\n",
    "print(\"F-measure: \"+str(F_measure))'''\n",
    "##############################################################################\n",
    "#######################################################################\n",
    "print(\"##################################################################\")\n",
    "print(\"Test Results\")\n",
    "print(\"##################################################################\")\n",
    "a = sum(accuracy)/10\n",
    "p = sum(precision)/10\n",
    "r = sum(recall)/10\n",
    "ro = sum(roc)/10\n",
    "print(\"Accuracy: \",a,\"\\tPrecision: \",p,\"\\tRecall: \",r,\"\\tROC: \",ro)\n",
    "TP = sum(predictedPositive)\n",
    "FP = (len(df_pos)-sum(predictedPositive))\n",
    "TN = sum(predictedNegative)\n",
    "FN = (len(df_neg)-sum(predictedNegative))\n",
    "print(\"Actual Positive: \",len(df_pos),\"\\tPredictedPositive(TP): \",TP,\"\\tFP: \",FP)\n",
    "print(\"Actual Negative: \",len(df_neg),\"\\tPredictedNegative(TN): \",TN,\"\\tFN: \",FN)\n",
    "F_measure = (2 * p * r)/(p + r)\n",
    "print(\"F-measure: \"+str(F_measure))\n",
    "##############################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result of CNN-LSTM + Trained Word2Vec"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "###############################\n",
    "Test Results\n",
    "###############################\n",
    "Accuracy:  0.6799999999999999 \tPrecision:  0.6787595631869413 \tRecall:  0.7150000000000001 \t\n",
    "ROC:  0.6799999999999999\n",
    "Actual Positive:  600 \tPredictedPositive(TP):  387 \tFP:  213\n",
    "Actual Negative:  600 \tPredictedNegative(TN):  429 \tFN:  171\n",
    "F-measure: 0.696408620966096"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result of CNN-LSTM + Pretrained word2vec"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "=========================================\n",
    "Fold 1\n",
    "=========================================\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6878 - accuracy: 0.5463 - \n",
    "val_loss: 0.6417 - val_accuracy: 0.7600\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.4730 - accuracy: 0.8194 - \n",
    "val_loss: 0.1404 - val_accuracy: 0.9650\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.1274 - accuracy: 0.9722 - \n",
    "val_loss: 0.0737 - val_accuracy: 0.9850\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 21s 19ms/step - loss: 0.0431 - accuracy: 0.9926 - \n",
    "val_loss: 0.0201 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 21s 20ms/step - loss: 0.0121 - accuracy: 0.9981 - \n",
    "val_loss: 0.0280 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[41 19]\n",
    " [18 42]]\n",
    "\n",
    "=========================================\n",
    "Fold 2\n",
    "=========================================\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.6875 - accuracy: 0.5593 - \n",
    "val_loss: 0.6504 - val_accuracy: 0.7600\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 21s 20ms/step - loss: 0.5238 - accuracy: 0.7963 - \n",
    "val_loss: 0.1839 - val_accuracy: 0.9400\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.1335 - accuracy: 0.9620 - \n",
    "val_loss: 0.1368 - val_accuracy: 0.9750\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.0433 - accuracy: 0.9944 - \n",
    "val_loss: 0.0379 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.0138 - accuracy: 0.9972 -\n",
    "val_loss: 0.0395 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[41 19]\n",
    " [26 34]]\n",
    "\n",
    "=========================================\n",
    "Fold 3\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 23s 22ms/step - loss: 0.6909 - accuracy: 0.5213 - \n",
    "val_loss: 0.6592 - val_accuracy: 0.6300\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5584 - accuracy: 0.7426 - \n",
    "val_loss: 0.2233 - val_accuracy: 0.9550\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 21s 20ms/step - loss: 0.1491 - accuracy: 0.9620 - \n",
    "val_loss: 0.0702 - val_accuracy: 0.9850\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 21s 20ms/step - loss: 0.0539 - accuracy: 0.9889 - \n",
    "val_loss: 0.0500 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.0150 - accuracy: 0.9981 - \n",
    "val_loss: 0.0275 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[46 14]\n",
    " [22 38]]\n",
    "\n",
    "=========================================\n",
    "Fold 4\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.6908 - accuracy: 0.5389 - \n",
    "val_loss: 0.6577 - val_accuracy: 0.7200\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5653 - accuracy: 0.7463 - \n",
    "val_loss: 0.2650 - val_accuracy: 0.9450\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1737 - accuracy: 0.9556 - \n",
    "val_loss: 0.0742 - val_accuracy: 0.9850\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.0438 - accuracy: 0.9926 - \n",
    "val_loss: 0.0468 - val_accuracy: 0.9900\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 21s 19ms/step - loss: 0.0239 - accuracy: 0.9963 - \n",
    "val_loss: 0.0244 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[50 10]\n",
    " [33 27]]\n",
    "\n",
    "=========================================\n",
    "Fold 5\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6904 - accuracy: 0.5370 - \n",
    "val_loss: 0.6660 - val_accuracy: 0.7400\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.5593 - accuracy: 0.7676 - \n",
    "val_loss: 0.2105 - val_accuracy: 0.9500\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1517 - accuracy: 0.9583 - \n",
    "val_loss: 0.0505 - val_accuracy: 0.9850\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.0410 - accuracy: 0.9880 - \n",
    "val_loss: 0.0431 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.0184 - accuracy: 0.9981 - \n",
    "val_loss: 0.0303 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[50 10]\n",
    " [25 35]]\n",
    "\n",
    "=========================================\n",
    "Fold 6\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.6906 - accuracy: 0.5352 - \n",
    "val_loss: 0.6578 - val_accuracy: 0.7050\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.5249 - accuracy: 0.7731 - \n",
    "val_loss: 0.2566 - val_accuracy: 0.9050\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.1615 - accuracy: 0.9528 - \n",
    "val_loss: 0.1026 - val_accuracy: 0.9850\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.0461 - accuracy: 0.9926 - \n",
    "val_loss: 0.0387 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.0221 - accuracy: 0.9963 - \n",
    "val_loss: 0.0186 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[43 17]\n",
    " [21 39]]\n",
    " \n",
    "=========================================\n",
    "Fold 7\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 26s 24ms/step - loss: 0.6962 - accuracy: 0.4954 - \n",
    "val_loss: 0.6818 - val_accuracy: 0.5150\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.6196 - accuracy: 0.7000 - \n",
    "val_loss: 0.2886 - val_accuracy: 0.9100\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.2305 - accuracy: 0.9278 - \n",
    "val_loss: 0.0692 - val_accuracy: 0.9900\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.0475 - accuracy: 0.9917 - \n",
    "val_loss: 0.0468 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.0220 - accuracy: 0.9991 - \n",
    "val_loss: 0.0186 - val_accuracy: 0.9950\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[ 99   1]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[45 15]\n",
    " [26 34]]\n",
    " \n",
    "=========================================\n",
    "Fold 8\n",
    "=========================================\n",
    "\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 27s 25ms/step - loss: 0.6858 - accuracy: 0.5500 - \n",
    "val_loss: 0.6107 - val_accuracy: 0.6300\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.4679 - accuracy: 0.8028 - \n",
    "val_loss: 0.1440 - val_accuracy: 0.9550\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 21s 19ms/step - loss: 0.1296 - accuracy: 0.9685 - \n",
    "val_loss: 0.0539 - val_accuracy: 0.9950\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 21s 19ms/step - loss: 0.0478 - accuracy: 0.9917 - \n",
    "val_loss: 0.0092 - val_accuracy: 1.0000\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.0233 - accuracy: 0.9963 - \n",
    "val_loss: 0.0073 - val_accuracy: 1.0000\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[100   0]\n",
    " [  0 100]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[50 10]\n",
    " [28 32]]\n",
    "\n",
    "=========================================\n",
    "Fold 9\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.6918 - accuracy: 0.5204 - \n",
    "val_loss: 0.6729 - val_accuracy: 0.6700\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 20s 19ms/step - loss: 0.5757 - accuracy: 0.7546 - \n",
    "val_loss: 0.4744 - val_accuracy: 0.7500\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.1968 - accuracy: 0.9417 - \n",
    "val_loss: 0.3881 - val_accuracy: 0.8300\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 22s 20ms/step - loss: 0.0487 - accuracy: 0.9926 - \n",
    "val_loss: 0.3987 - val_accuracy: 0.8250\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 21s 20ms/step - loss: 0.0314 - accuracy: 0.9972 - \n",
    "val_loss: 0.4341 - val_accuracy: 0.8250\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[82 18]\n",
    " [17 83]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[33 27]\n",
    " [22 38]]\n",
    " \n",
    "=========================================\n",
    "Fold 10\n",
    "=========================================\n",
    "\n",
    "Train on 1080 samples, validate on 200 samples\n",
    "\n",
    "Epoch 1/5\n",
    "1080/1080 [==============================] - 25s 23ms/step - loss: 0.6913 - accuracy: 0.5231 - \n",
    "val_loss: 0.6797 - val_accuracy: 0.5100\n",
    "\n",
    "Epoch 2/5\n",
    "1080/1080 [==============================] - 22s 21ms/step - loss: 0.5404 - accuracy: 0.7704 - \n",
    "val_loss: 0.4630 - val_accuracy: 0.7750\n",
    "\n",
    "Epoch 3/5\n",
    "1080/1080 [==============================] - 24s 22ms/step - loss: 0.1618 - accuracy: 0.9667 - \n",
    "val_loss: 0.5004 - val_accuracy: 0.7950\n",
    "\n",
    "Epoch 4/5\n",
    "1080/1080 [==============================] - 21s 19ms/step - loss: 0.0390 - accuracy: 0.9907 - \n",
    "val_loss: 0.6386 - val_accuracy: 0.7950\n",
    "\n",
    "Epoch 5/5\n",
    "1080/1080 [==============================] - 23s 21ms/step - loss: 0.0113 - accuracy: 0.9991 - \n",
    "val_loss: 0.7965 - val_accuracy: 0.7800\n",
    "\n",
    "Validation Confustion Matrix\n",
    "[[73 27]\n",
    " [17 83]]\n",
    "\n",
    "Test Confustion Matrix\n",
    "[[33 27]\n",
    " [17 43]]\n",
    "\n",
    "#################################\n",
    "Test Results\n",
    "#################################\n",
    "Accuracy:  0.6616666666666667 \tPrecision:  0.6919422745657776 \tRecall:  0.6033333333333333 \t\n",
    "ROC:  0.6616666666666667\n",
    "Actual Positive:  600 \tPredictedPositive(TP):  432 \tFP:  168\n",
    "Actual Negative:  600 \tPredictedNegative(TN):  362 \tFN:  238\n",
    "F-measure: 0.6446069646368822"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}