06downstreamkeras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Beyond counting words: Working with word embeddings\n",
    "\n",
    "Workshop by Damian Trilling\n",
    "\n",
    "This notebook illustrates how we can use embeddings in Machine Learning tasks.\n",
    "\n",
    "As always, we first import neccesary modules. We also get our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install embeddingvectorizer    # you need to install this module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Supervised text classification\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Embedding, LSTM, GlobalMaxPooling1D\n",
    "from keras.metrics import Precision, Recall\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "# general\n",
    "import numpy as np\n",
    "import re\n",
    "# word embedding stuff\n",
    "import gensim\n",
    "import gensim.downloader as api\n",
    "from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex\n",
    "from gensim.corpora import Dictionary\n",
    "\n",
    "# data\n",
    "from courseutils import get_review_data\n",
    "\n",
    "# lets get more output\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cached file reviewdata.pickle.bz2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-04-13 13:18:40,570 : INFO : loading projection weights from /home/damian/gensim-data/glove-wiki-gigaword-300/glove-wiki-gigaword-300.gz\n",
      "2021-04-13 13:21:54,742 : INFO : KeyedVectors lifecycle event {'msg': 'loaded (400000, 300) matrix of type float32 from /home/damian/gensim-data/glove-wiki-gigaword-300/glove-wiki-gigaword-300.gz', 'binary': False, 'encoding': 'utf8', 'datetime': '2021-04-13T13:21:54.729984', 'gensim': '4.0.1', 'python': '3.8.5 (default, Jan 27 2021, 15:41:15) \\n[GCC 9.3.0]', 'platform': 'Linux-5.4.0-70-generic-x86_64-with-glibc2.29', 'event': 'load_word2vec_format'}\n"
     ]
    }
   ],
   "source": [
    "# get data\n",
    "reviews_train, reviews_test, y_train, y_test = get_review_data()\n",
    "\n",
    "reviews_train, y_train = shuffle(reviews_train, y_train, random_state=42)\n",
    "reviews_test, y_test = shuffle(reviews_test, y_test, random_state=42)\n",
    "\n",
    "# get word embedding model\n",
    "\n",
    "# pretrained:\n",
    "# wv = api.load('word2vec-google-news-300')\n",
    "wv = api.load(\"glove-wiki-gigaword-300\")\n",
    "\n",
    "# or our own:\n",
    "#wv = gensim.models.Word2Vec.load(\"mymodel\").wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fasttext-wiki-news-subwords-300',\n",
       " 'conceptnet-numberbatch-17-06-300',\n",
       " 'word2vec-ruscorpora-300',\n",
       " 'word2vec-google-news-300',\n",
       " 'glove-wiki-gigaword-50',\n",
       " 'glove-wiki-gigaword-100',\n",
       " 'glove-wiki-gigaword-200',\n",
       " 'glove-wiki-gigaword-300',\n",
       " 'glove-twitter-25',\n",
       " 'glove-twitter-50',\n",
       " 'glove-twitter-100',\n",
       " 'glove-twitter-200',\n",
       " '__testing_word2vec-matrix-synopsis']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api.info(name_only=True)['models']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(reviews_train) == len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(reviews_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Dumb is as dumb does, in this thoroughly uninteresting, supposed black comedy. Essentially what starts out as Chris Klein trying to maintain a low profile, eventually morphs into an uninspired version of \"The Three Amigos\", only without any laughs. In order for black comedy to work, it must be outrageous, which \"Play Dead\" is not. In order for black comedy to work, it cannot be mean spirited, which \"Play Dead\" is. What \"Play Dead\" really is, is a town full of nut jobs. Fred Dunst does however do a pretty fair imitation of Billy Bob Thornton\\'s character from \"A Simple Plan\", while Jake Busey does a pretty fair imitation of, well, Jake Busey. - MERK',\n",
       " \"I dug out from my garage some old musicals and this is another one of my favorites. It was written by Jay Alan Lerner and directed by Vincent Minelli. It won two Academy Awards for Best Picture of 1951 and Best Screenplay. The story of an American painter in Paris who tries to make it big. Nina Foch is a sophisticated lady of means and is very interested in helping him, but soon finds she loves the guy. Meanwhile Gene Kelly falls for lovely damsel, Leslie Caron. His main dancing partner, and I must say they are fantastic together on the floor and otherwise. Famous French singer Georges Guietary sings, too. So if you like good smooth dancing and fun filled scenes filled with Oscar Levant's nimble piano fingers, the songs of George Gershwyn will live on forever in this colorful gem. 8/10\",\n",
       " 'After watching this movie I was honestly disappointed - not because of the actors, story or directing - I was disappointed by this film advertisements.<br /><br />The trailers were suggesting that the battalion \"have chosen the third way out\" other than surrender or die (Polish infos were even misguiding that they had the choice between being killed by own artillery or German guns, they even translated the title wrong as \"misplaced battalion\"). This have tickled the right spot and I bought the movie.<br /><br />The disappointment started when I realized that the third way is to just sit down and count dead bodies followed by sitting down and counting dead bodies... Then I began to think \"hey, this story can\\'t be that simple... I bet this clever officer will find some cunning way to save what left of his troops\". Well, he didn\\'t, they were just sitting and waiting for something to happen. And so was I.<br /><br />The story was based on real events of World War I, so the writers couldn\\'t make much use of their imagination, but even thought I found this movie really unchallenging and even a little bit boring. And as I wrote in the first place - it isn\\'t fault of actors, writers or director - their marketing people have raised my expectations high above the level that this movie could cope with.']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_train[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['neg',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'pos',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'neg',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'neg',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'neg',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'pos',\n",
       " 'pos',\n",
       " 'pos',\n",
       " 'neg',\n",
       " 'neg']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras\n",
    "\n",
    "## A simple neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALIDATION_SIZE = 2500\n",
    "np.random.seed(666)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encodeY(Y):\n",
    "    '''create one-hot (dummies) for output, see also https://machinelearningmastery.com/multi-class-classification-tutorial-keras-deep-learning-library/\n",
    "    encode class values as integers\n",
    "    '''\n",
    "    encoder = LabelEncoder()\n",
    "    encoder.fit(Y)\n",
    "    encoded_Y = encoder.transform(Y)\n",
    "    dummy_y = np_utils.to_categorical(encoded_Y)\n",
    "    return dummy_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encodeY(['aa','bb','aa','cc','aa','cc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = CountVectorizer(stop_words='english')\n",
    "X_train = vectorizer.fit_transform(reviews_train)\n",
    "X_test = vectorizer.transform(reviews_test)\n",
    "X_test.sort_indices()\n",
    "X_train.sort_indices()\n",
    "\n",
    "input_dim = X_train.shape[1]  # Number of features\n",
    "\n",
    "y_train_int = encodeY(y_train)[:,0]\n",
    "y_test_int = encodeY(y_test)[:,0]\n",
    "\n",
    "numberoflabels = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74538"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 300)               22361700  \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 22,362,001\n",
      "Trainable params: 22,362,001\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(300, input_dim=input_dim, activation='relu'))\n",
    "#model.add(layers.Dense(100, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', \n",
    "           optimizer='adam', \n",
    "            metrics=['accuracy', Precision(), Recall()])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "704/704 [==============================] - 92s 129ms/step - loss: 0.3858 - accuracy: 0.8296 - precision: 0.8395 - recall: 0.8086 - val_loss: 0.2726 - val_accuracy: 0.8856 - val_precision: 0.8884 - val_recall: 0.8813\n",
      "Epoch 2/5\n",
      "704/704 [==============================] - 91s 129ms/step - loss: 0.0814 - accuracy: 0.9720 - precision: 0.9724 - recall: 0.9716 - val_loss: 0.3747 - val_accuracy: 0.8844 - val_precision: 0.8863 - val_recall: 0.8813\n",
      "Epoch 3/5\n",
      "704/704 [==============================] - 89s 126ms/step - loss: 0.0185 - accuracy: 0.9946 - precision: 0.9938 - recall: 0.9955 - val_loss: 0.4996 - val_accuracy: 0.8816 - val_precision: 0.8795 - val_recall: 0.8837\n",
      "Epoch 4/5\n",
      "704/704 [==============================] - 90s 127ms/step - loss: 0.0041 - accuracy: 0.9994 - precision: 0.9989 - recall: 0.9998 - val_loss: 0.6214 - val_accuracy: 0.8824 - val_precision: 0.8922 - val_recall: 0.8693\n",
      "Epoch 5/5\n",
      "704/704 [==============================] - 90s 128ms/step - loss: 0.0011 - accuracy: 0.9999 - precision: 0.9998 - recall: 0.9999 - val_loss: 0.6985 - val_accuracy: 0.8824 - val_precision: 0.8922 - val_recall: 0.8693\n",
      "782/782 [==============================] - 3s 4ms/step - loss: 0.9158 - accuracy: 0.8507 - precision: 0.8458 - recall: 0.8578\n",
      "Accuracy: 0.85, Precision: 0.85, Recall: 0.86\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE],\n",
    "                     epochs=5,\n",
    "                     verbose=True,\n",
    "                     validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))\n",
    "\n",
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A model with a second layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 300)               22361700  \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 22,542,601\n",
      "Trainable params: 22,542,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/5\n",
      "350/704 [=============>................] - ETA: 39s - loss: 0.4396 - accuracy: 0.8007 - precision_1: 0.8065 - recall_1: 0.7907"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(300, input_dim=input_dim, activation='relu'))\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', \n",
    "           optimizer='adam', \n",
    "            metrics=['accuracy', Precision(), Recall()])\n",
    "model.summary()\n",
    "\n",
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE],\n",
    "                     epochs=5,\n",
    "                     verbose=True,\n",
    "                     validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))\n",
    "\n",
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_5 (Dense)              (None, 300)               22361700  \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 22,723,201\n",
      "Trainable params: 22,723,201\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/5\n",
      "704/704 [==============================] - 59s 82ms/step - loss: 0.4063 - accuracy: 0.8162 - precision_2: 0.8132 - recall_2: 0.8358 - val_loss: 0.3184 - val_accuracy: 0.8670 - val_precision_2: 0.8560 - val_recall_2: 0.8812\n",
      "Epoch 2/5\n",
      "704/704 [==============================] - 58s 83ms/step - loss: 0.0801 - accuracy: 0.9717 - precision_2: 0.9717 - recall_2: 0.9719 - val_loss: 0.6739 - val_accuracy: 0.8450 - val_precision_2: 0.8476 - val_recall_2: 0.8397\n",
      "Epoch 3/5\n",
      "704/704 [==============================] - 59s 83ms/step - loss: 0.0122 - accuracy: 0.9957 - precision_2: 0.9967 - recall_2: 0.9947 - val_loss: 1.4569 - val_accuracy: 0.8528 - val_precision_2: 0.8499 - val_recall_2: 0.8555\n",
      "Epoch 4/5\n",
      "704/704 [==============================] - 69s 98ms/step - loss: 0.0097 - accuracy: 0.9976 - precision_2: 0.9975 - recall_2: 0.9978 - val_loss: 1.0837 - val_accuracy: 0.8484 - val_precision_2: 0.8556 - val_recall_2: 0.8367\n",
      "Epoch 5/5\n",
      "704/704 [==============================] - 75s 107ms/step - loss: 0.0043 - accuracy: 0.9985 - precision_2: 0.9984 - recall_2: 0.9987 - val_loss: 1.2904 - val_accuracy: 0.8466 - val_precision_2: 0.8281 - val_recall_2: 0.8731\n",
      "782/782 [==============================] - 2s 3ms/step - loss: 1.2862 - accuracy: 0.8464 - precision_2: 0.8290 - recall_2: 0.8730\n",
      "Accuracy: 0.85, Precision: 0.83, Recall: 0.87\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(300, input_dim=input_dim, activation='relu'))\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(300, activation='relu'))\n",
    "\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', \n",
    "           optimizer='adam', \n",
    "            metrics=['accuracy', Precision(), Recall()])\n",
    "model.summary()\n",
    "\n",
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE],\n",
    "                     epochs=5,\n",
    "                     verbose=True,\n",
    "                     validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))\n",
    "\n",
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolutional Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "embedding_dim = 300\n",
    "\n",
    "# Tokenize words\n",
    "tokenizer = Tokenizer(num_words=5000)\n",
    "tokenizer.fit_on_texts(reviews_train)\n",
    "X_train = tokenizer.texts_to_sequences(reviews_train)\n",
    "X_test = tokenizer.texts_to_sequences(reviews_test)\n",
    "\n",
    "# Adding 1 because of reserved 0 index\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "# Pad sequences with zeros\n",
    "maxlen = len(max(X_train, key=len)) # never truncate -- alternatively, set max length to lower value \n",
    "X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\n",
    "X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 1973, 300)         26574900  \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, 1969, 300)         450300    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 27,115,801\n",
      "Trainable params: 27,115,801\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=vocab_size, \n",
    "                           output_dim=embedding_dim, \n",
    "                           input_length=maxlen))\n",
    "model.add(Conv1D(embedding_dim, 5, activation='relu'))\n",
    "model.add(GlobalMaxPooling1D())\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',  Precision(), Recall()])\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "704/704 [==============================] - 1399s 2s/step - loss: 0.4571 - accuracy: 0.7607 - precision_4: 0.7591 - recall_4: 0.7744 - val_loss: 0.2258 - val_accuracy: 0.9084 - val_precision_4: 0.8970 - val_recall_4: 0.9222\n",
      "Epoch 2/5\n",
      "704/704 [==============================] - 1346s 2s/step - loss: 0.1325 - accuracy: 0.9535 - precision_4: 0.9529 - recall_4: 0.9538 - val_loss: 0.2415 - val_accuracy: 0.9044 - val_precision_4: 0.8883 - val_recall_4: 0.9246\n",
      "Epoch 3/5\n",
      "704/704 [==============================] - 1310s 2s/step - loss: 0.0292 - accuracy: 0.9927 - precision_4: 0.9932 - recall_4: 0.9921 - val_loss: 0.3663 - val_accuracy: 0.8940 - val_precision_4: 0.8642 - val_recall_4: 0.9342\n",
      "Epoch 4/5\n",
      "704/704 [==============================] - 1562s 2s/step - loss: 0.0227 - accuracy: 0.9922 - precision_4: 0.9925 - recall_4: 0.9919 - val_loss: 0.5236 - val_accuracy: 0.8820 - val_precision_4: 0.9577 - val_recall_4: 0.7987\n",
      "Epoch 5/5\n",
      "704/704 [==============================] - 1323s 2s/step - loss: 0.0259 - accuracy: 0.9902 - precision_4: 0.9912 - recall_4: 0.9889 - val_loss: 0.4639 - val_accuracy: 0.8832 - val_precision_4: 0.8375 - val_recall_4: 0.9503\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE], \n",
    "          epochs=3, verbose=True,\n",
    "          validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [==============================] - 387s 496ms/step - loss: 0.5126 - accuracy: 0.8749 - precision_4: 0.8284 - recall_4: 0.9458\n",
      "Accuracy: 0.87, Precision: 0.83, Recall: 0.95\n"
     ]
    }
   ],
   "source": [
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pretrained embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = wv.get_keras_embedding(train_embeddings=False)\n",
    "input_dim = (len(X_train[:-VALIDATION_SIZE]), 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, None, 300)         120000000 \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, None, 300)         450300    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 1)                 301       \n",
      "=================================================================\n",
      "Total params: 120,540,901\n",
      "Trainable params: 540,901\n",
      "Non-trainable params: 120,000,000\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Conv1D(embedding_dim, 5, activation='relu'))\n",
    "model.add(GlobalMaxPooling1D())\n",
    "model.add(Dense(300, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',  Precision(), Recall()])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "704/704 [==============================] - 785s 1s/step - loss: 0.5776 - accuracy: 0.6828 - precision_5: 0.6877 - recall_5: 0.6828 - val_loss: 0.4122 - val_accuracy: 0.8104 - val_precision_5: 0.7463 - val_recall_5: 0.9391\n",
      "Epoch 2/5\n",
      "704/704 [==============================] - 746s 1s/step - loss: 0.2846 - accuracy: 0.8827 - precision_5: 0.8869 - recall_5: 0.8795 - val_loss: 0.3580 - val_accuracy: 0.8380 - val_precision_5: 0.9177 - val_recall_5: 0.7418\n",
      "Epoch 3/5\n",
      "704/704 [==============================] - 737s 1s/step - loss: 0.1318 - accuracy: 0.9535 - precision_5: 0.9555 - recall_5: 0.9510 - val_loss: 0.4007 - val_accuracy: 0.8456 - val_precision_5: 0.8840 - val_recall_5: 0.7947\n",
      "Epoch 4/5\n",
      "704/704 [==============================] - 735s 1s/step - loss: 0.0537 - accuracy: 0.9838 - precision_5: 0.9834 - recall_5: 0.9842 - val_loss: 0.4870 - val_accuracy: 0.8404 - val_precision_5: 0.8649 - val_recall_5: 0.8059\n",
      "Epoch 5/5\n",
      "704/704 [==============================] - 735s 1s/step - loss: 0.0380 - accuracy: 0.9876 - precision_5: 0.9885 - recall_5: 0.9870 - val_loss: 0.7288 - val_accuracy: 0.8244 - val_precision_5: 0.9097 - val_recall_5: 0.7193\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE], \n",
    "          epochs=5, verbose=True,\n",
    "          validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [==============================] - 326s 417ms/step - loss: 0.7356 - accuracy: 0.8215 - precision_5: 0.9051 - recall_5: 0.7184\n",
      "Accuracy: 0.82, Precision: 0.91, Recall: 0.72\n"
     ]
    }
   ],
   "source": [
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# too much memory requirements below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Conv1D(128, 5, activation='relu'))\n",
    "model.add(MaxPooling1D(5))\n",
    "model.add(Conv1D(128, 5, activation='relu'))\n",
    "model.add(MaxPooling1D(5))\n",
    "model.add(Conv1D(128, 5, activation='relu'))\n",
    "model.add(MaxPooling1D(35))\n",
    "#model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',  Precision(), Recall()])\n",
    "print(model.summary())\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM\n",
    "\n",
    "More examples: https://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=vocab_size, \n",
    "                           output_dim=embedding_dim, \n",
    "                           input_length=maxlen))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(units=1, activation='sigmoid'))\n",
    " \n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', Recall(), Precision()])\n",
    "print(model.summary())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE], \n",
    "          epochs=3, batch_size=128, verbose=1, \n",
    "          validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, acc, precision, recall = model.evaluate(X_test, y_test_int)\n",
    "print(f\"Accuracy: {acc:.2f}, Precision: {precision:.2f}, Recall: {recall:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "history = model.fit(X_train[:-VALIDATION_SIZE], y_train_int[:-VALIDATION_SIZE], \n",
    "          epochs=5, verbose=True,\n",
    "          validation_data=(X_train[-VALIDATION_SIZE:], y_train_int[-VALIDATION_SIZE:]))\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}