Fashion - MNIST classification - CNN Model - TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Tensorflow and other utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import fashion_mnist\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.data import Dataset\n",
    "\n",
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Download and Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 10) (10000, 10)\n",
      "(60000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "# Loading Dataset\n",
    "(train_X, train_y), (test_X, test_y) = fashion_mnist.load_data()\n",
    "\n",
    "# Preprocess and save the data\n",
    "# Preparing data for CNN model\n",
    "\n",
    "# Normalization\n",
    "train_X = train_X.astype('float32')\n",
    "test_X = test_X.astype('float32')\n",
    "train_X = train_X / 255\n",
    "test_X = test_X / 255\n",
    "\n",
    "# Converting train_Y and test_Y to 1-hot vectors\n",
    "train_Y_one_hot = to_categorical(train_y)\n",
    "test_Y_one_hot = to_categorical(test_y)\n",
    "print(train_Y_one_hot.shape, test_Y_one_hot.shape)\n",
    "\n",
    "# Image Shapes - converting to 4-D (for 4-D tensors in CNN) \n",
    "train_X_processed = train_X.reshape([-1, 28, 28, 1])\n",
    "print(train_X_processed.shape)\n",
    "\n",
    "# Validation Data\n",
    "train_X_split, valid_X_split, train_Y_split, valid_Y_split = train_test_split(train_X_processed, train_Y_one_hot, test_size=0.2)\n",
    "\n",
    "# Saving Data\n",
    "np.save('data/train_X.npy', train_X_split)\n",
    "np.save('data/train_y.npy', train_Y_split)\n",
    "np.save('data/valid_X.npy', valid_X_split)\n",
    "np.save('data/valid_y.npy', valid_Y_split)\n",
    "np.save('data/test_X.npy', test_X)\n",
    "np.save('data/test_y.npy', test_Y_one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading data (if data already preprocessed and saved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((48000, 28, 28, 1), (48000, 10))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Saved Data\n",
    "train_X, train_y = np.load('data/train_X.npy'), np.load('data/train_y.npy')\n",
    "test_X, test_y = np.load('data/test_X.npy'), np.load('data/test_y.npy')\n",
    "valid_X, valid_y = np.load('data/valid_X.npy'), np.load('data/valid_y.npy')\n",
    "train_X.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Convolutional Neural Network for classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline\n",
    "1. Define Constants (hyper parameters)\n",
    "    - Learning rate\n",
    "    - Batch size\n",
    "    - Number of epochs\n",
    "    - Image size\n",
    "    - Number of classes\n",
    "2. Define X and y as tf.placeholders\n",
    "3. Define helper functions for generating weights and biases as tf.Variables\n",
    "4. Define and build architecture as follows\n",
    "    - (CNN_8_3x3 + LeakyRelu + MAX Pool_2x2_2)\n",
    "    - (CNN_16_3x3 + LeakyRelu + MAX Pool_2x2_2)\n",
    "    - (CNN_32_3x3 + LeakyRelu + MAX Pool_2x2_2)\n",
    "    - (Flatten)\n",
    "    - (Dense_128)\n",
    "    - (Dense_10)\n",
    "5. Define accuracy operation\n",
    "6. Define loss operation - softmax + crossentropy\n",
    "7. Define optimizer - Adam\n",
    "8. Run Training loop in a session\n",
    "9. Capture scalar summaries for visualization on TensorBoard\n",
    "    - Learning rate\n",
    "    - Loss\n",
    "    - Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LOG_DIR = 'tmp/log'\n",
    "\n",
    "# Constants - Hyper Parameters\n",
    "LEARNING_RATE = 1e-4\n",
    "BATCH_SIZE = 64\n",
    "EPOCHS = 20\n",
    "IMAGE_SIZE = 28\n",
    "NUM_CLASS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    weights = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(weights)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Defining inputs to model\n",
    "X = tf.placeholder(dtype=tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1])\n",
    "y = tf.placeholder(dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Building the model\n",
    "# Conv 2d + Leaky ReLU + MaxPool\n",
    "with tf.name_scope(name=\"CONV_1\"):\n",
    "    W_1 = weight_variable([3, 3, 1, 8])\n",
    "    b_1 = bias_variable([8])\n",
    "    conv_layer_1 = tf.nn.conv2d(X, W_1, [1, 1, 1, 1], padding=\"SAME\") + b_1 # (W*X + b)\n",
    "    leaky_relu_1 = tf.nn.leaky_relu(conv_layer_1, alpha=0.1)\n",
    "    max_pool_1 = tf.nn.max_pool(leaky_relu_1, [1, 2, 2, 1], [1, 2, 2, 1], padding=\"SAME\")\n",
    "    \n",
    "# Conv 2d + Leaky ReLU + MaxPool\n",
    "with tf.name_scope(name=\"CONV_2\"):\n",
    "    W_2 = weight_variable([3, 3, 8, 16])\n",
    "    b_2 = bias_variable([16])\n",
    "    conv_layer_2 = tf.nn.conv2d(max_pool_1, W_2, [1, 1, 1, 1], padding=\"SAME\") + b_2 # (W*max_pool1 + b)\n",
    "    leaky_relu_2 = tf.nn.leaky_relu(conv_layer_2, alpha=0.1)\n",
    "    max_pool_2 = tf.nn.max_pool(leaky_relu_2, [1, 2, 2, 1], [1, 2, 2, 1], padding=\"SAME\")\n",
    "    \n",
    "# Conv 2d + Leaky ReLU + MaxPool\n",
    "with tf.name_scope(name=\"CONV_3\"):\n",
    "    W_3 = weight_variable([3, 3, 16, 32])\n",
    "    b_3 = bias_variable([32])\n",
    "    conv_layer_3 = tf.nn.conv2d(max_pool_2, W_3, [1, 1, 1, 1], padding=\"SAME\") + b_3 # (W*max_pool2 + b)\n",
    "    leaky_relu_3 = tf.nn.leaky_relu(conv_layer_3, alpha=0.1)\n",
    "    max_pool_3 = tf.nn.max_pool(leaky_relu_3, [1, 2, 2, 1], [1, 2, 2, 1], padding=\"SAME\")\n",
    "\n",
    "# FLATTEN + Fully connected Layer\n",
    "with tf.name_scope(name=\"DENSE\"):\n",
    "    W_fc1 = weight_variable([4 * 4 * 32, 128])\n",
    "    b_fc1 = bias_variable([128])\n",
    "    \n",
    "    pool_flatten = tf.reshape(max_pool_3, [-1, 4 * 4 * 32])\n",
    "    \n",
    "    fc_layer_1 = tf.matmul(pool_flatten, W_fc1) + b_fc1\n",
    "    fc_activated = tf.nn.relu(fc_layer_1)\n",
    "\n",
    "# Output Layer - Dense\n",
    "with tf.name_scope(name=\"Output\"):\n",
    "    W_op = weight_variable([128, NUM_CLASS])\n",
    "    b_op = bias_variable([NUM_CLASS])\n",
    "    \n",
    "    y_ = tf.matmul(fc_activated, W_op) + b_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Defining Loss op + optimizer\n",
    "loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_))\n",
    "train_op = tf.train.AdamOptimizer().minimize(loss_op)\n",
    "\n",
    "# Evaluations\n",
    "correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "# Init op\n",
    "init_op = tf.global_variables_initializer()\n",
    "\n",
    "# Summary Merge op\n",
    "merged = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:45<00:00, 16.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8488749861717224%\n",
      "Validation Accuracy: 0.8460000157356262%\n",
      "\n",
      "Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:45<00:00, 16.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8747291564941406%\n",
      "Validation Accuracy: 0.8696666955947876%\n",
      "\n",
      "Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8855000138282776%\n",
      "Validation Accuracy: 0.8801666498184204%\n",
      "\n",
      "Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:42<00:00, 17.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.895229160785675%\n",
      "Validation Accuracy: 0.8891666531562805%\n",
      "\n",
      "Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:44<00:00, 16.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9007499814033508%\n",
      "Validation Accuracy: 0.8955000042915344%\n",
      "\n",
      "Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:42<00:00, 17.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9033958315849304%\n",
      "Validation Accuracy: 0.8963333368301392%\n",
      "\n",
      "Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:39<00:00, 19.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9067708253860474%\n",
      "Validation Accuracy: 0.9003333449363708%\n",
      "\n",
      "Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9100833535194397%\n",
      "Validation Accuracy: 0.9022499918937683%\n",
      "\n",
      "Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.914312481880188%\n",
      "Validation Accuracy: 0.9039999842643738%\n",
      "\n",
      "Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:44<00:00, 16.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9171249866485596%\n",
      "Validation Accuracy: 0.9027500152587891%\n",
      "\n",
      "Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:42<00:00, 17.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.921500027179718%\n",
      "Validation Accuracy: 0.9050833582878113%\n",
      "\n",
      "Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:40<00:00, 18.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9241041541099548%\n",
      "Validation Accuracy: 0.9042500257492065%\n",
      "\n",
      "Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:42<00:00, 17.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9267916679382324%\n",
      "Validation Accuracy: 0.9045833349227905%\n",
      "\n",
      "Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9263125061988831%\n",
      "Validation Accuracy: 0.9038333296775818%\n",
      "\n",
      "Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9322291612625122%\n",
      "Validation Accuracy: 0.9072499871253967%\n",
      "\n",
      "Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:41<00:00, 18.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9352916479110718%\n",
      "Validation Accuracy: 0.9076666831970215%\n",
      "\n",
      "Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9367499947547913%\n",
      "Validation Accuracy: 0.9081666469573975%\n",
      "\n",
      "Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9388958215713501%\n",
      "Validation Accuracy: 0.9079999923706055%\n",
      "\n",
      "Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:43<00:00, 17.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.941979169845581%\n",
      "Validation Accuracy: 0.9080833196640015%\n",
      "\n",
      "Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████| 750/750 [00:41<00:00, 18.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.942270815372467%\n",
      "Validation Accuracy: 0.9051666855812073%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "with tf.Session() as sess:\n",
    "    summary_writer = tf.summary.FileWriter(LOG_DIR, sess.graph)\n",
    "    sess.run(init_op)\n",
    "    for epoch in range(EPOCHS):\n",
    "        print('Epoch {}'.format(epoch + 1))\n",
    "        \n",
    "        for index, offset in tqdm(list(enumerate(range(0, train_X.shape[0], BATCH_SIZE))), ncols=100):\n",
    "            xs, ys = train_X[offset: offset + BATCH_SIZE], train_y[offset: offset + BATCH_SIZE]\n",
    "            sess.run(train_op, feed_dict={\n",
    "                X: xs,\n",
    "                y: ys\n",
    "            })\n",
    "        \n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            X: train_X,\n",
    "            y: train_y\n",
    "        })\n",
    "        validation_accuracy = accuracy.eval(feed_dict={\n",
    "            X: valid_X,\n",
    "            y: valid_y\n",
    "        })\n",
    "        print('Training Accuracy: {}%\\nValidation Accuracy: {}%\\n'.format(train_accuracy, validation_accuracy))\n",
    "    summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}