Tensorflow_07_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(55000, 784)\n",
      "(55000, 10)\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "num_features = mnist.train.images.shape[1]\n",
    "num_labels = mnist.train.labels.shape[1]\n",
    "img_size = int(np.sqrt(num_features))\n",
    "print(img_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()\n",
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input and output (Placeholder)\n",
    "x = tf.placeholder(tf.float32, [None, num_features])\n",
    "y = tf.placeholder(tf.float32, [None, num_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Variables\n",
    "def weights(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "def bias(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layers\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding = 'SAME')\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First convolutional layer\n",
    "W_conv1 = weights([5, 5, 1, 32]) # Filter kernel: 5x5, 1 input channel (grayscale), 32 output feature maps\n",
    "b_conv1 = bias([32]) # 32 outputs\n",
    "\n",
    "# convert images to tensors\n",
    "x_image = tf.reshape(x, [-1,28, 28, 1])\n",
    "\n",
    "# First layer\n",
    "convolve1 = conv2d(x_image, W_conv1) + b_conv1\n",
    "# Apply RELU (nonlinear activation function mostly used for classification problems)\n",
    "h_conv1 = tf.nn.relu(convolve1)\n",
    "# Apply max pooling\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "layer1 = h_pool1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Second convolutional layer\n",
    "W_conv2 = weights([5, 5, 32, 64]) # Filter kernel: 5x5, 32 input channels (first layer), 64 output feature maps\n",
    "b_conv2 = bias([64]) # 64 outputs\n",
    "\n",
    "convolve2 = conv2d(layer1, W_conv2) + b_conv2\n",
    "h_conv2 = tf.nn.relu(convolve2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "layer2 = h_pool2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Third layer = Fully connected layer\n",
    "W_fc1 = weights([7*7*64, 1024])\n",
    "b_fc1 = bias([1024])\n",
    "\n",
    "#Flattening\n",
    "layer2_matrix = tf.reshape(layer2, [-1, 7*7*64])\n",
    "# Apply weights\n",
    "matmul_fc1 = tf.matmul(layer2_matrix, W_fc1) + b_fc1\n",
    "# Apply RELU\n",
    "h_fc1 = tf.nn.relu(matmul_fc1)\n",
    "\n",
    "layer3 = h_fc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional dropout to reduce overfitting\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "layer3_drop = tf.nn.dropout(layer3, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final layer: Softmax, Fully connected layer\n",
    "W_fc2 = weights([1024, 10]) # 10 classes for digist 0,1,2,3,4,5,6,7,8,9\n",
    "b_fc2 = bias([10])\n",
    "matmul_fc2 = tf.matmul(layer3_drop, W_fc2) + b_fc2\n",
    "# Apply softmax activation function\n",
    "y_conv = tf.nn.softmax(matmul_fc2)\n",
    "layer4 = y_conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Loss and Optimizer\n",
    "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(layer4), axis=1))\n",
    "train = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(layer4, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training acc 0.03999999910593033\n",
      "step 100, training acc 0.8799999952316284\n",
      "step 200, training acc 0.9599999785423279\n",
      "step 300, training acc 0.9200000166893005\n",
      "step 400, training acc 0.9599999785423279\n",
      "step 500, training acc 1.0\n",
      "step 600, training acc 0.9399999976158142\n",
      "step 700, training acc 0.9800000190734863\n",
      "step 800, training acc 0.9599999785423279\n",
      "step 900, training acc 1.0\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.global_variables_initializer())\n",
    "epochs = 1000\n",
    "for i in range(epochs):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    if i % 100 == 0:\n",
    "        train_acc = accuracy.eval(feed_dict={x: batch[0], y: batch[1], keep_prob: 1.0})\n",
    "        print(\"step {0}, training acc {1}\".format(i, train_acc))\n",
    "    train.run(feed_dict={x: batch[0], y: batch[1], keep_prob: 0.5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy: 0.9637000003457069\n"
     ]
    }
   ],
   "source": [
    "# Evaluat model, evaluate in batches to avoid out of memory issues\n",
    "n_batches = mnist.test.images.shape[0] // 50\n",
    "cumulative_acc = 0.0\n",
    "for i in range(n_batches):\n",
    "    batch = mnist.test.next_batch(50)\n",
    "    cumulative_acc += accuracy.eval(feed_dict={x: batch[0], y: batch[1], keep_prob: 1.0})\n",
    "print(\"test accuracy: {0}\".format(cumulative_acc / n_batches))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (ai_env)",
   "language": "python",
   "name": "ai_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}