IMDB movie reviews sentiment classification - CNN Model - TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import imdb\n",
    "from keras.preprocessing import sequence\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the dataset\n",
    "top_words = 5000\n",
    "(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_one_hot = to_categorical(y_test)\n",
    "y_train_one_hot = to_categorical(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((25000, 2), (25000, 2))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_one_hot.shape, y_train_one_hot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing sestences\n",
    "max_review_len = 500\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=max_review_len)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=max_review_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the model\n",
    "Pipeline\n",
    "- Define X and y as tf.placeholders\n",
    "- Helper functions for weights and biases as tf.Variables\n",
    "- Define Hyper parameters\n",
    "    - Learning Rate\n",
    "    - Batch size\n",
    "    - Number of epochs\n",
    "    - Embedding vector size\n",
    "    - Keep Probability\n",
    "- Build the model\n",
    "    - Embedding layer (None, sequence_length, embedding_vector_size, 1) + Dropout\n",
    "    - CONV_1d_2 (2, embedding_vector_size, 32) with 32 kernels + Leaky Relu + MaxPool_1d_2 + Dropout\n",
    "    - CONV_1d_2 (2, 32, 32) with 32 kernels + Leaky Relu + MaxPool_1d_2 + Dropout\n",
    "    - Dense Layer (??, 256) with relu activation + Dropout\n",
    "    - Dense Layer (256, 1) with sigmoid activation\n",
    "- Define the loss operation - Binary crossentropy\n",
    "- Define the optimizer operation - ADAM\n",
    "- Run the training loop\n",
    "- Capture summaries for visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdir = 'log/'\n",
    "\n",
    "VOCAB_SIZE = 5000\n",
    "EMBEDDING_SIZE = 32\n",
    "EPOCHS = 3\n",
    "BATCH_SIZE = 64\n",
    "KEEP_PROB = 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(dtype=tf.int32)\n",
    "y = tf.placeholder(dtype=tf.float32)\n",
    "keep_prob = tf.placeholder(dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight_variable(shape, name='weights'):\n",
    "    init = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(init, name=name)\n",
    "\n",
    "def bias_variable(shape, name='bias'):\n",
    "    init = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(init, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building the model\n",
    "with tf.name_scope('embedding'):\n",
    "    W = weight_variable([VOCAB_SIZE, EMBEDDING_SIZE])\n",
    "    embedded_chars = tf.nn.embedding_lookup(W, X)\n",
    "\n",
    "with tf.name_scope('embedding_dropout'):\n",
    "    embedded_drop = tf.nn.dropout(embedded_chars, keep_prob)\n",
    "    \n",
    "with tf.name_scope('conv_maxpool_1'):\n",
    "    W = weight_variable([2, EMBEDDING_SIZE, 32])\n",
    "    b = bias_variable([32])\n",
    "    h_conv1 = tf.nn.leaky_relu(tf.nn.conv1d(embedded_drop, W, 1, padding='VALID') + b, alpha=0.1)\n",
    "    h_pool1 = tf.layers.max_pooling1d(h_conv1, 2, 2, padding='VALID')\n",
    "\n",
    "with tf.name_scope('conv_maxpool_1_dropout'):\n",
    "    h_pool1_drop = tf.nn.dropout(h_pool1, keep_prob)\n",
    "\n",
    "with tf.name_scope('conv_maxpool_2'):\n",
    "    W = weight_variable([2, 32, 32])\n",
    "    b = bias_variable([32])\n",
    "    h_conv2 = tf.nn.leaky_relu(tf.nn.conv1d(h_pool1_drop, W, 1, padding='VALID') + b, alpha=0.1)\n",
    "    h_pool2 = tf.layers.max_pooling1d(h_conv2, 2, 2, padding='VALID')\n",
    "\n",
    "with tf.name_scope('fully_connected'):\n",
    "    W = weight_variable([3968, 256])\n",
    "    b = bias_variable([256])\n",
    "    \n",
    "    pool_flatten = tf.reshape(h_pool2, [-1, 3968])\n",
    "    \n",
    "    pool_flatten_drop = tf.nn.dropout(pool_flatten, keep_prob)\n",
    "    \n",
    "    fc1 = tf.nn.relu(tf.matmul(pool_flatten_drop, W) + b)\n",
    "\n",
    "with tf.name_scope('fc1_dropout'):\n",
    "    fc1_drop = tf.nn.dropout(fc1, keep_prob)\n",
    "\n",
    "with tf.name_scope('output'):\n",
    "    W = weight_variable([256, 2])\n",
    "    b = bias_variable([2])\n",
    "    \n",
    "    y_ = tf.matmul(fc1_drop, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-10-8c880c3947cf>:2: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Defining the loss op and trainop\n",
    "loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_))\n",
    "tf.summary.scalar('loss', loss_op)\n",
    "\n",
    "train_op = tf.train.AdamOptimizer().minimize(loss_op)\n",
    "\n",
    "# Evaluations\n",
    "correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "# Summaries\n",
    "merged = tf.summary.merge_all()\n",
    "\n",
    "# Init op\n",
    "init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 391/391 [00:50<00:00,  7.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.509440004825592%\n",
      "Test Accuracy: 0.5073599815368652%\n",
      "Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 391/391 [00:57<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.6567599773406982%\n",
      "Test Accuracy: 0.6414399743080139%\n",
      "Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 391/391 [01:02<00:00,  6.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8992400169372559%\n",
      "Test Accuracy: 0.8668000102043152%\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "with tf.Session() as sess:\n",
    "    \n",
    "    # Summary Writer\n",
    "    summary_writer = tf.summary.FileWriter(logdir, sess.graph)\n",
    "    \n",
    "    sess.run(init_op)\n",
    "    \n",
    "    for epoch in range(EPOCHS):\n",
    "        print('Epoch {}'.format(epoch + 1))\n",
    "        \n",
    "        for index, offset in tqdm(list(enumerate(range(0, X_train.shape[0], BATCH_SIZE))), ncols=100):\n",
    "            xs, ys = X_train[offset: offset + BATCH_SIZE], y_train_one_hot[offset: offset + BATCH_SIZE]\n",
    "            summary, _ = sess.run([merged, train_op], feed_dict={\n",
    "                X: xs,\n",
    "                y: ys,\n",
    "                keep_prob: KEEP_PROB\n",
    "            })\n",
    "            \n",
    "            summary_writer.add_summary(summary, epoch)\n",
    "            \n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            X: X_train,\n",
    "            y: y_train_one_hot,\n",
    "            keep_prob: KEEP_PROB\n",
    "        })\n",
    "        \n",
    "        test_accuracy = accuracy.eval(feed_dict={\n",
    "            X: X_test,\n",
    "            y: y_test_one_hot,\n",
    "            keep_prob:KEEP_PROB\n",
    "        })\n",
    "\n",
    "        print('Training Accuracy: {}%\\nTest Accuracy: {}%'.format(train_accuracy, test_accuracy))\n",
    "    \n",
    "    summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}