02-03-classification-solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST\n",
    "\n",
    "MNIST is a database of handwritten digits. The images are 28x28 greyscale images (encoded as 784-dimensional vectors in row-major order). There are 60,000 images in the training set, and there are 10,000 images in the test set.\n",
    "\n",
    "Why is there a train-test split? We care about how our function generalizes, and so we want to benchmark its performance on a set of data that it hasn't seen before. Otherwise, a \"perfect\" learning algorithm could just memorize all the data points, but this algorithm wouldn't generalize well.\n",
    "\n",
    "Let's see what one of the MNIST images looks like."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x123598860>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADO5JREFUeJzt3V2IXfW5x/Hf76QpiOlFYjUMNpqeogerSKKjCMYS9VhyYiEWg9SLkkLJ9CJKCyVU7EVzWaQv1JvAlIbGkmMrpNUoYmNjMQ1qcSJqEmNiElIzMW9lhCaCtNGnF7Nsp3H2f+/st7XH5/uBYfZez3p52Mxv1lp77bX/jggByOe/6m4AQD0IP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpD7Vz43Z5uOEQI9FhFuZr6M9v+1ltvfZPmD7gU7WBaC/3O5n+23PkrRf0h2SxiW9LOneiHijsAx7fqDH+rHnv1HSgYg4FBF/l/RrSSs6WB+APuok/JdKOjLl+Xg17T/YHrE9Znusg20B6LKev+EXEaOSRiUO+4FB0sme/6ikBVOef66aBmAG6CT8L0u6wvbnbX9a0tckbelOWwB6re3D/og4a/s+Sb+XNEvShojY07XOAPRU25f62toY5/xAz/XlQz4AZi7CDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmp7iG5Jsn1Y0mlJH0g6GxHD3WgKQO91FP7KrRHx1y6sB0AfcdgPJNVp+EPSVts7bY90oyEA/dHpYf+SiDhq+xJJz9p+MyK2T52h+qfAPwZgwDgiurMie52kMxHxo8I83dkYgIYiwq3M1/Zhv+0LbX/mo8eSvixpd7vrA9BfnRz2z5f0O9sfref/I+KZrnQFoOe6dtjf0sY47Ad6rueH/QBmNsIPJEX4gaQIP5AU4QeSIvxAUt24qy+FlStXNqytXr26uOw777xTrL///vvF+qZNm4r148ePN6wdOHCguCzyYs8PJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0lxS2+LDh061LC2cOHC/jUyjdOnTzes7dmzp4+dDJbx8fGGtYceeqi47NjYWLfb6Rtu6QVQRPiBpAg/kBThB5Ii/EBShB9IivADSXE/f4tK9+xfe+21xWX37t1brF911VXF+nXXXVesL126tGHtpptuKi575MiRYn3BggXFeifOnj1brJ86dapYHxoaanvbb7/9drE+k6/zt4o9P5AU4QeSIvxAUoQfSIrwA0kRfiApwg8k1fR+ftsbJH1F0smIuKaaNk/SbyQtlHRY0j0R8W7Tjc3g+/kH2dy5cxvWFi1aVFx2586dxfoNN9zQVk+taDZewf79+4v1Zp+fmDdvXsPamjVrisuuX7++WB9k3byf/5eSlp0z7QFJ2yLiCknbqucAZpCm4Y+I7ZImzpm8QtLG6vFGSXd1uS8APdbuOf/8iDhWPT4uaX6X+gHQJx1/tj8ionQub3tE0kin2wHQXe3u+U/YHpKk6vfJRjNGxGhEDEfEcJvbAtAD7YZ/i6RV1eNVkp7oTjsA+qVp+G0/KulFSf9je9z2NyX9UNIdtt+S9L/VcwAzCN/bj4F19913F+uPPfZYsb579+6GtVtvvbW47MTEuRe4Zg6+tx9AEeEHkiL8QFKEH0iK8ANJEX4gKS71oTaXXHJJsb5r166Oll+5cmXD2ubNm4vLzmRc6gNQRPiBpAg/kBThB5Ii/EBShB9IivADSTFEN2rT7OuzL7744mL93XfL3xa/b9++8+4pE/b8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU9/Ojp26++eaGteeee6647OzZs4v1pUuXFuvbt28v1j+puJ8fQBHhB5Ii/EBShB9IivADSRF+ICnCDyTV9H5+2xskfUXSyYi4ppq2TtJqSaeq2R6MiKd71SRmruXLlzesNbuOv23btmL9xRdfbKsnTGplz/9LScummf7TiFhU/RB8YIZpGv6I2C5pog+9AOijTs7577P9uu0Ntud2rSMAfdFu+NdL+oKkRZKOSfpxoxltj9gesz3W5rYA9EBb4Y+IExHxQUR8KOnnkm4szDsaEcMRMdxukwC6r63w2x6a8vSrknZ3px0A/dLKpb5HJS2V9Fnb45J+IGmp7UWSQtJhSd/qYY8AeoD7+dGRCy64oFjfsWNHw9rVV19dXPa2224r1l944YViPSvu5wdQRPiBpAg/kBThB5Ii/EBShB9IiiG60ZG1a9cW64sXL25Ye+aZZ4rLcimvt9jzA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBS3NKLojvvvLNYf/zxx4v19957r2Ft2bLpvhT631566aViHdPjll4ARYQfSIrwA0kRfiApwg8kRfiBpAg/kBT38yd30UUXFesPP/xwsT5r1qxi/emnGw/gzHX8erHnB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkmt7Pb3uBpEckzZcUkkYj4me250n6jaSFkg5Luici3m2yLu7n77Nm1+GbXWu//vrri/WDBw8W66V79psti/Z0837+s5K+GxFflHSTpDW2vyjpAUnbIuIKSduq5wBmiKbhj4hjEfFK9fi0pL2SLpW0QtLGaraNku7qVZMAuu+8zvltL5S0WNKfJc2PiGNV6bgmTwsAzBAtf7bf9hxJmyV9JyL+Zv/7tCIiotH5vO0RSSOdNgqgu1ra89uercngb4qI31aTT9gequpDkk5Ot2xEjEbEcEQMd6NhAN3RNPye3MX/QtLeiPjJlNIWSauqx6skPdH99gD0SiuX+pZI+pOkXZI+rCY/qMnz/sckXSbpL5q81DfRZF1c6uuzK6+8slh/8803O1r/ihUrivUnn3yyo/Xj/LV6qa/pOX9E7JDUaGW3n09TAAYHn/ADkiL8QFKEH0iK8ANJEX4gKcIPJMVXd38CXH755Q1rW7du7Wjda9euLdafeuqpjtaP+rDnB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkuM7/CTAy0vhb0i677LKO1v38888X682+DwKDiz0/kBThB5Ii/EBShB9IivADSRF+ICnCDyTFdf4ZYMmSJcX6/fff36dO8EnCnh9IivADSRF+ICnCDyRF+IGkCD+QFOEHkmp6nd/2AkmPSJovKSSNRsTPbK+TtFrSqWrWByPi6V41mtktt9xSrM+ZM6ftdR88eLBYP3PmTNvrxmBr5UM+ZyV9NyJesf0ZSTttP1vVfhoRP+pdewB6pWn4I+KYpGPV49O290q6tNeNAeit8zrnt71Q0mJJf64m3Wf7ddsbbM9tsMyI7THbYx11CqCrWg6/7TmSNkv6TkT8TdJ6SV+QtEiTRwY/nm65iBiNiOGIGO5CvwC6pKXw256tyeBviojfSlJEnIiIDyLiQ0k/l3Rj79oE0G1Nw2/bkn4haW9E/GTK9KEps31V0u7utwegV1p5t/9mSV+XtMv2q9W0ByXda3uRJi//HZb0rZ50iI689tprxfrtt99erE9MTHSzHQyQVt7t3yHJ05S4pg/MYHzCD0iK8ANJEX4gKcIPJEX4gaQIP5CU+znEsm3GcwZ6LCKmuzT/Mez5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpfg/R/VdJf5ny/LPVtEE0qL0Nal8SvbWrm71d3uqMff2Qz8c2bo8N6nf7DWpvg9qXRG/tqqs3DvuBpAg/kFTd4R+tefslg9rboPYl0Vu7aumt1nN+APWpe88PoCa1hN/2Mtv7bB+w/UAdPTRi+7DtXbZfrXuIsWoYtJO2d0+ZNs/2s7bfqn5PO0xaTb2ts320eu1etb28pt4W2P6j7Tds77H97Wp6ra9doa9aXre+H/bbniVpv6Q7JI1LelnSvRHxRl8bacD2YUnDEVH7NWHbX5J0RtIjEXFNNe0hSRMR8cPqH+fciPjegPS2TtKZukdurgaUGZo6srSkuyR9QzW+doW+7lENr1sde/4bJR2IiEMR8XdJv5a0ooY+Bl5EbJd07qgZKyRtrB5v1OQfT9816G0gRMSxiHilenxa0kcjS9f62hX6qkUd4b9U0pEpz8c1WEN+h6SttnfaHqm7mWnMr4ZNl6TjkubX2cw0mo7c3E/njCw9MK9dOyNedxtv+H3ckoi4TtL/SVpTHd4OpJg8ZxukyzUtjdzcL9OMLP0vdb527Y543W11hP+opAVTnn+umjYQIuJo9fukpN9p8EYfPvHRIKnV75M19/MvgzRy83QjS2sAXrtBGvG6jvC/LOkK25+3/WlJX5O0pYY+Psb2hdUbMbJ9oaQva/BGH94iaVX1eJWkJ2rs5T8MysjNjUaWVs2v3cCNeB0Rff+RtFyT7/gflPT9Onpo0Nd/S3qt+tlTd2+SHtXkYeA/NPneyDclXSRpm6S3JP1B0rwB6u1XknZJel2TQRuqqbclmjykf13Sq9XP8rpfu0JftbxufMIPSIo3/ICkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJPVP82g/p9/JjhUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12037b6a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(mnist.test.images[0].reshape(28, 28), cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check the label of this image. The MNIST labels are encoded in one-hot format. There are 10 possible labels, and the vector with label $i$ is the $i$-dimensional vector that has the entry $1$ in the $i$th position and $0$s elsewhere."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 1., 0., 0.])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(mnist.test.labels[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fully-connected neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's design a fully-connected neural network to classify MNIST digits. It should take a 784-dimensional input and give a 10-dimensional one-hot encoded probability distribution as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, (None, 28*28)) # batch of inputs\n",
    "y_ = tf.placeholder(tf.float32, (None, 10)) # batch of corresponding labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this corresponds with the model of a neuron in [ 02-01-notes ]\n",
    "# except this is describing an entire layer, not a single neuron\n",
    "# and we're not including the activation function inside here\n",
    "\n",
    "def fully_connected(x, input_dimension, output_dimension):\n",
    "    w = tf.Variable(tf.random_normal((input_dimension, output_dimension)))\n",
    "    b = tf.Variable(tf.random_normal((output_dimension,)))\n",
    "    \n",
    "    return tf.matmul(x, w) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with a really simple neural network with only one fully-connected layer (the output layer) with 10 neurons.\n",
    "\n",
    "See [ 02-04-notes ] for an architecture diagram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = fully_connected(x, 784, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above, $y$ is a 10-dimensional vector, but it's not a probability distribution. We can fix that by applying the softmax function to the logits $y$:\n",
    "\n",
    "$$\\sigma(y)_i = \\frac{e^{y_i}}{\\sum_j e^{y_j}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can define loss as the cross entropy between the true probability distribution (the labels) $p$ and the predicted probability distribution $q$:\n",
    "\n",
    "$$H(p, q) = - \\sum_i p(x) \\log q(x)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In TensorFlow, we can do both of these in a single step (also needed for numerical stability):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 60000 training points, so we'll be doing minibatch stochastic gradient descent to train our network (instead of computing gradients over all 60000 data points)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 100\n",
    "ITERATIONS = 1000\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(ITERATIONS):\n",
    "    x_batch, y_batch = mnist.train.next_batch(BATCH_SIZE)\n",
    "    sess.run(optimizer, {x: x_batch, y_: y_batch})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's evaluate the accuracy of our network over the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "    return np.mean(np.argmax(predictions, 1) == np.argmax(labels, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8699"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = y.eval({x: mnist.test.images})\n",
    "accuracy(predictions, mnist.test.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep fully-connected network\n",
    "\n",
    "Will adding a ton of parameters help us find a better solution? Let's use a deep fully-connected network using layers with 2000, 1000, and 100 neurons in the hidden layers and then 10 neurons in the output layer. Let's use ReLU activation for all the hidden layers. See [ 02-05-notes ] for an architecture diagram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc1 = tf.nn.relu(fully_connected(x, 784, 2000))\n",
    "fc2 = tf.nn.relu(fully_connected(fc1, 2000, 1000))\n",
    "fc3 = tf.nn.relu(fully_connected(fc2, 1000, 100))\n",
    "fc4 = fully_connected(fc3, 100, 10)\n",
    "\n",
    "y = fc4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our initial network had ~8,000 parameters. The above network has ~3.5 million parameters, which is over 400x the capacity of the first one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use a fancier optimizer this time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 100, batch loss 1524.092163\n",
      "iteration 200, batch loss 2593.736328\n",
      "iteration 300, batch loss 1095.994141\n",
      "iteration 400, batch loss 982.541748\n",
      "iteration 500, batch loss 986.342651\n",
      "iteration 600, batch loss 732.789795\n",
      "iteration 700, batch loss 1450.782837\n",
      "iteration 800, batch loss 689.906555\n",
      "iteration 900, batch loss 398.776398\n",
      "iteration 1000, batch loss 335.374756\n",
      "iteration 1100, batch loss 593.934448\n",
      "iteration 1200, batch loss 1487.456299\n",
      "iteration 1300, batch loss 2010.209351\n",
      "iteration 1400, batch loss 170.993164\n",
      "iteration 1500, batch loss 330.138214\n",
      "iteration 1600, batch loss 0.000000\n",
      "iteration 1700, batch loss 310.644623\n",
      "iteration 1800, batch loss 39.443127\n",
      "iteration 1900, batch loss 906.453918\n",
      "iteration 2000, batch loss 122.963959\n",
      "iteration 2100, batch loss 414.156403\n",
      "iteration 2200, batch loss 225.709030\n",
      "iteration 2300, batch loss 162.094055\n",
      "iteration 2400, batch loss 185.161530\n",
      "iteration 2500, batch loss 94.067307\n",
      "iteration 2600, batch loss 66.464867\n",
      "iteration 2700, batch loss 217.196075\n",
      "iteration 2800, batch loss 37.685604\n",
      "iteration 2900, batch loss 173.089432\n",
      "iteration 3000, batch loss 230.676468\n",
      "iteration 3100, batch loss 137.282837\n",
      "iteration 3200, batch loss 22.259512\n",
      "iteration 3300, batch loss 0.000000\n",
      "iteration 3400, batch loss 221.934433\n",
      "iteration 3500, batch loss 18.538671\n",
      "iteration 3600, batch loss 81.654099\n",
      "iteration 3700, batch loss 133.341293\n",
      "iteration 3800, batch loss 194.342484\n",
      "iteration 3900, batch loss 243.813461\n",
      "iteration 4000, batch loss 34.092228\n",
      "iteration 4100, batch loss 161.006149\n",
      "iteration 4200, batch loss 77.275589\n",
      "iteration 4300, batch loss 0.000000\n",
      "iteration 4400, batch loss 824.152649\n",
      "iteration 4500, batch loss 86.573669\n",
      "iteration 4600, batch loss 71.565506\n",
      "iteration 4700, batch loss 47.667850\n",
      "iteration 4800, batch loss 0.000000\n",
      "iteration 4900, batch loss 41.683495\n",
      "iteration 5000, batch loss 58.690819\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 50\n",
    "ITERATIONS = 5000 # this takes ~ 2 minutes on my laptop\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(ITERATIONS):\n",
    "    x_batch, y_batch = mnist.train.next_batch(BATCH_SIZE)\n",
    "    l, _ = sess.run([loss, optimizer], {x: x_batch, y_: y_batch})\n",
    "    if (i+1) % 100 == 0:\n",
    "        print('iteration %d, batch loss %f' % (i+1, np.mean(l)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9529"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = y.eval({x: mnist.test.images})\n",
    "accuracy(predictions, mnist.test.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional neural network\n",
    "\n",
    "Let's design a convolutional neural network to classify MNIST digits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_image = tf.reshape(x, (-1, 28, 28, 1)) # turn our 784-dimensional vector into a 28x28 image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a convolutional layer\n",
    "\n",
    "def convolve(x, kernel_height, kernel_width, input_channels, output_channels):\n",
    "    w = tf.Variable(tf.random_normal((kernel_height, kernel_width, input_channels, output_channels)))\n",
    "    b = tf.Variable(tf.random_normal((output_channels,)))\n",
    "    \n",
    "    return tf.nn.conv2d(x, w, strides=(1, 1, 1, 1), padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a 2x2 max pooling layer\n",
    "\n",
    "def pool(x):\n",
    "    return tf.nn.max_pool(x, (1, 2, 2, 1), (1, 2, 2, 1), padding='SAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network architecture\n",
    "\n",
    "Let's design an architecture with the following layers:\n",
    "\n",
    "* convolution layer with 25 3x3 filters, relu activation\n",
    "* 2x2 max pooling layer\n",
    "* convolution layer with 50 3x3 filters, relu activation\n",
    "* 2x2 max pooling layer\n",
    "* fully-connected layer with 1000 neurons, relu activation\n",
    "* fully-connected output layer (10 neurons)\n",
    "\n",
    "See [ 02-06-notes ] for an architecture diagram."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv1 = tf.nn.relu(convolve(x_image, 3, 3, 1, 25))\n",
    "pool1 = pool(conv1)\n",
    "conv2 = tf.nn.relu(convolve(pool1, 3, 3, 25, 50))\n",
    "pool2 = pool(conv2)\n",
    "pool2_flat = tf.reshape(pool2, (-1, 7 * 7 * 50))\n",
    "fc1 = tf.nn.relu(fully_connected(pool2_flat, 7 * 7 * 50, 1000))\n",
    "fc2 = fully_connected(fc1, 1000, 10)\n",
    "\n",
    "y = fc2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This network has approximately 2.5 million parameters. Note that this is about 1 million parameters _fewer_ than the deep fully-connected network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 100, batch loss 904.771484\n",
      "iteration 200, batch loss 276.587616\n",
      "iteration 300, batch loss 453.983093\n",
      "iteration 400, batch loss 406.946472\n",
      "iteration 500, batch loss 100.544708\n",
      "iteration 600, batch loss 114.825882\n",
      "iteration 700, batch loss 133.377121\n",
      "iteration 800, batch loss 126.084885\n",
      "iteration 900, batch loss 430.765533\n",
      "iteration 1000, batch loss 46.572891\n",
      "iteration 1100, batch loss 343.002045\n",
      "iteration 1200, batch loss 85.059456\n",
      "iteration 1300, batch loss 42.467697\n",
      "iteration 1400, batch loss 165.551407\n",
      "iteration 1500, batch loss 129.811783\n",
      "iteration 1600, batch loss 0.000000\n",
      "iteration 1700, batch loss 43.886387\n",
      "iteration 1800, batch loss 341.299652\n",
      "iteration 1900, batch loss 76.959167\n",
      "iteration 2000, batch loss 84.741837\n",
      "iteration 2100, batch loss 125.253632\n",
      "iteration 2200, batch loss 110.654861\n",
      "iteration 2300, batch loss 73.772011\n",
      "iteration 2400, batch loss 0.000000\n",
      "iteration 2500, batch loss 0.000000\n",
      "iteration 2600, batch loss 270.552063\n",
      "iteration 2700, batch loss 48.319942\n",
      "iteration 2800, batch loss 54.076015\n",
      "iteration 2900, batch loss 0.000000\n",
      "iteration 3000, batch loss 151.410980\n",
      "iteration 3100, batch loss 11.557969\n",
      "iteration 3200, batch loss 11.288750\n",
      "iteration 3300, batch loss 59.667030\n",
      "iteration 3400, batch loss 0.000000\n",
      "iteration 3500, batch loss 153.702972\n",
      "iteration 3600, batch loss 15.906386\n",
      "iteration 3700, batch loss 0.000000\n",
      "iteration 3800, batch loss 0.000000\n",
      "iteration 3900, batch loss 0.000000\n",
      "iteration 4000, batch loss 20.755957\n",
      "iteration 4100, batch loss 0.000000\n",
      "iteration 4200, batch loss 73.231575\n",
      "iteration 4300, batch loss 44.342968\n",
      "iteration 4400, batch loss 0.000000\n",
      "iteration 4500, batch loss 119.391495\n",
      "iteration 4600, batch loss 0.000000\n",
      "iteration 4700, batch loss 11.547559\n",
      "iteration 4800, batch loss 0.000000\n",
      "iteration 4900, batch loss 23.342422\n",
      "iteration 5000, batch loss 0.000000\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 50\n",
    "ITERATIONS = 5000 # this takes ~ 3.5 minutes on my laptop\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(ITERATIONS):\n",
    "    x_batch, y_batch = mnist.train.next_batch(BATCH_SIZE)\n",
    "    l, _ = sess.run([loss, optimizer], {x: x_batch, y_: y_batch})\n",
    "    if (i+1) % 100 == 0:\n",
    "        print('iteration %d, batch loss %f' % (i+1, np.mean(l)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9756"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = y.eval({x: mnist.test.images})\n",
    "accuracy(predictions, mnist.test.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what some intermediate activations look like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv1_, conv2_ = sess.run([conv1, conv2], {x: mnist.test.images[0:1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1221e6320>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAD8CAYAAADub8g7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAFpBJREFUeJzt3WuMXWX1x/HfopRaWsFecJy2U1q1oANJRQsBahDE1gJqvRBtE2qjTRoRCBiiVEh8YaKSPwnRF8RYA2kxBDQpSRuNqaXhItE0tYiVtkJb7GXotKUXSinpff1fzHazn81czpzLPvvM8/0kk1nPeWbOXjCrK3s/Z1/M3QUAMTmn2QkAQNFofACiQ+MDEB0aH4Do0PgARIfGByA6ND4A0amp8ZnZHDN71cy2mdmSeiUFNBu1PbRZtScwm9kwSa9JmiWpS9J6SfPdfXP90gOKR20PfefW8LtXSdrm7q9Lkpk9JWmupD6Lw8y4TKQ8Drj7Rc1OoqQGVdvUdalUVNe1HOpOlLQ7M+5KXkNr2NnsBEqM2m5dFdV1LXt8FTGzxZIWN3o7QJGo69ZWS+N7Q1JHZjwpeS3g7kslLZU4JEDLGLC2qevWVsuh7npJ08xsqpmdJ2mepFX1SQtoKmp7iKt6j8/dT5vZnZJWSxom6TF331S3zIAmobaHvqpPZ6lqYxwSlMkGd5/R7CSGAuq6VCqqa67cABAdGh+A6ND4AESHxgcgOjQ+ANGh8QGIDo0PQHRofACiQ+MDEB0aH4Do0PgARIfGByA6ND4A0aHxAYhOw28932iXXnppML7hhhvSeMeOHcHcyJEj0/j555/v8z0PHTpUn+SAQfjEJz6RxqNHjw7m9uzZk8bTp08P5jZs2JDGEyZMCOY2b37v+UgjRowI5o4ePVp9si2OPT4A0aHxAYgOjQ9AdFr+1vP5NY0pU6akcX6t7vrrr0/jEydOBHMf+chH0njr1q3BXHZtsF7OPTdcXj1+/Hga/+lPfwrm3n777bpvX9x6vm7qVdff+9730vipp54K5ubNm5fG48ePD+aytfTOO+8Ec/v27Uvj888/P5g7ePBg9cn2IV+rGzduTOMPfehDwdyBAwd6jWvErecBoDc0PgDRafnTWbIf80vSsGHD0nj37t3B3H/+858+3+eiiy5K41OnTgVzb731Vq8/J0mnT5+uONfDhw+ncfbUBUm69dZb0/iKK64I5vo79QZDx7PPPpvGs2fPDubOO++8NP70pz8dzL366qtpPGrUqGBu+PDhaZw/DL3yyivTeO/evcFc/n3OnDmTxtllGSn8N5FfXsrmnd/G2LFj07iOh7oVYY8PQHRofACiQ+MDEJ2WXOO78MIL0/i6664L5trb29P4pZdeCuaya2zbt28P5t588800njRpUjCXXeM7cuRIMHfy5Mk+88xfdpR1zTXXBOPsmkp+DQVxyNZ1Npakiy++OI137twZzGXXtfOXpX30ox9N4/wa9yc/+ck0zq9Vt7W1BePsfH6t8Ic//GEa/+pXv+ozt+x/gyT985//VLOwxwcgOgM2PjN7zMz2m9krmdfGmtkaM9uafB/T2DSB+qO24zXglRtmdp2kdyQ97u6XJ6/9n6RD7v6gmS2RNMbd7xtwY3U6wz37MfjUqVODuezH7vnTUiZPnpzG+bPIs4esnZ2dwdyLL76YxtkrQ6T3H5Zmz6L/wAc+EMz99re/TePf/e53wVz2v+lrX/tan7nVUfRXbtSrtutV12PGvNdj84ea2brKX4GRreWJEycGcx/+8IfTOL/0cs457+33mFkwl6/dbJ+YNm1aMJf9d7Vu3bpgLlu7+auVfvGLX6Txrl27VCf1uXLD3V+QlL9P01xJy5N4uaSvDjo9oMmo7XhV++FGm7t3J/FeSW19/aCZLZa0uMrtAEWrqLap69ZW86e67u797eq7+1JJS6XG3KQAaJT+apu6bm3VNr59Ztbu7t1m1i5pfz2TGkj2spjB3C1506ZNaZxfbxg3blwa//Wvfw3msmuF+cvg8qcBZH/2jTfeCOaylxpdcsklwdzq1avTuEFreqhM02o7e7pVNh6MfF1n1//ytZpdK8z/XvZSMylc8/vyl78czD3xxBNpnL87dPa0rWXLlgVzzbzTebWns6yStDCJF0paWZ90gKajtiNQyeksT0r6u6RLzazLzBZJelDSLDPbKukLyRhoKdR2vAY81HX3+X1M3VjnXAqV3+3P3rCxP9UegkjSF7/4xTTO34Fl4cKF+R9Hgw3F2s7XdX93PclekTSQ7BVSL7/8cjB37bXXpnH+NLHs3ZP+9a9/BXPHjh2rePv1xpUbAKJD4wMQHRofgOi05N1ZWsVNN90UjK+66qo0vv3224O5/u4ODRSto6MjGGfvrNLfXccvv/zyYPzII4+k8bZt24K5Ih90lsceH4Do0PgARIdD3QaaNWtWMJ45c2Ya33XXXUWnA1Qsf9PQ7F2IPv7xjwdz2auQ8qfPZG82WqYb7LLHByA6ND4A0aHxAYgOa3x1duON713tNGfOnGDugQceSOOurq7CcgIGkr87y2WXXRaMs6ewZB9SLknjx49P44cffjiY27FjR6/v0Wzs8QGIDo0PQHRofACiwxpfnd1yyy1p3N9T1oAyufrqq4Nx/qHh2XW8CRMmBHOvv/56Gm/YsCGYK+vdxNnjAxAdGh+A6HCoW6PPfe5zwfjKK69M44ceeqjodICKZS89yz/A/N133w3G2QeT5x9anr0Dy8GDB+uZYsOwxwcgOjQ+ANGh8QGIDmt8gzRy5MhgnH+4cvYUll//+teF5ARUIl+72dukbdq0KZj79re/HYwvvPDCNM6eviKFt546depUzXkWgT0+ANGh8QGIDoe6g7Ro0aJgnH8w+M9+9rMi0wH6ZWZpPG/evGDub3/7WxrfeeedwdyIESOCcfZ0lvwVSIcOHao5z6KxxwcgOgM2PjPrMLNnzWyzmW0ys7uT18ea2Roz25p8H9P4dIH6obbjVcke32lJ97p7p6SrJd1hZp2Slkha6+7TJK1NxkArobYjNeAan7t3S+pO4qNmtkXSRElzJV2f/NhySc9Juq8hWTbZd77znTT+/Oc/H8ytW7cuGK9ataqQnFC7GGp7zJj3dlbPOSfcz+ns7Ezj/MO9s5deStLSpUvTeMuWLcFcq5zCkjWoNT4zmyLpCknrJLUlhSNJeyW19fFrQOlR23Gp+FNdMxstaYWke9z97eynRe7uZuZ9/N5iSYtrTRRolGpqm7pubRU1PjMbrp7CeMLdn05e3mdm7e7ebWbtkvb39rvuvlTS0uR9em2OZZO96aIUHt4eOXIkmPvNb34TjM+ePdu4xFB31dZ2Wes6e4WFFF5ZtGfPnmDu2muvTeP8jUiff/75YPyXv/wljd96662a82y2Sj7VNUmPStri7tlHKK2StDCJF0paWf/0gMahtuNVyR7fTEkLJP3bzF5OXrtf0oOS/mBmiyTtlPTNxqQINAy1HalKPtV9UZL1MX1jH68DpUdtx4tL1nrxjW98Ixjfdtttafz1r389mNu5c2chOQGV+NSnPhWMt2/fnsY333xzMJc91eX48ePB3B//+Mdg3NXVlcZnzpypOc9m45I1ANGh8QGIDoe6idmzZ6fxxz72sWAu+4zRlSv5gA/lMnny5DTOP/N2+PDhaTx//vxgbsWKFWmcvZmoJO3YsSMYnzhxotY0S4U9PgDRofEBiA6ND0B0WONL3HDDDWmcf5jyxo0b05hL0lA27e3taTxs2LBgLnsp2ujRo4O506dPp3F2HVuSjh07Vs8US4c9PgDRofEBiE60h7rZWw9J0qRJk9I4/1H+kiXcgBfllT2Eveyyy4K57HOen3vuuWAue7PRn/70p8Hc4cOH65hh+bDHByA6ND4A0aHxAYhOtGt8+Yer/PKXv0zj/B1Y9u3bV0hOQDVeeeWVND7//PODuZkzZ6bxueeG/9yXLVuWxv/973+DuVZ8gNBgsMcHIDo0PgDRiepQd8qUKWl8yy23BHMXXHBBGj/55JPB3LZt2xqaF1CLkSNHpvGoUaOCuewDhlavXh3Mbd68OY337w+fp5RfChpq2OMDEB0aH4Do0PgARMeKPJY3szfV87i+8ZIOFLbh/sWay8XuflFB2xrSSlrXUrnyKSqXiuq60MaXbtTsH+4+o/AN94JcUC9l+/uVKZ8y5SJxqAsgQjQ+ANFpVuNb2qTt9oZcUC9l+/uVKZ8y5dKcNT4AaCYOdQFEh8YHIDqFNj4zm2Nmr5rZNjMr/H7uZvaYme03s1cyr401szVmtjX5PqagXDrM7Fkz22xmm8zs7mbmg9o0s7ap68ErrPGZ2TBJj0i6SVKnpPlm1lnU9hPLJM3JvbZE0lp3nyZpbTIuwmlJ97p7p6SrJd2R/P9oVj6oUglqe5mo60Epco/vKknb3P11dz8p6SlJcwvcvtz9BUmHci/PlbQ8iZdL+mpBuXS7+0tJfFTSFkkTm5UPatLU2qauB6/IxjdR0u7MuCt5rdna3L07ifdKais6ATObIukKSevKkA8GrYy13fQ6KnNd8+FGhvec21Po+T1mNlrSCkn3uHvwOPtm5IOhh7p+vyIb3xuSOjLjSclrzbbPzNolKfm+f4CfrxszG66e4njC3Z9udj6oWhlrm7ruR5GNb72kaWY21czOkzRP0qoCt9+XVZIWJvFCSSuL2Kj1PNH8UUlb3P3hZueDmpSxtqnr/rh7YV+Sbpb0mqTtkh4octvJ9p+U1C3plHrWYRZJGqeeT5m2SnpG0tiCcvmsenb3N0p6Ofm6uVn58FXz37NptU1dD/6LS9YARIcPNwBEp6bG1+wrMYBGobaHtqoPdZOz1V+TNEs96wrrJc139839/iJQctT20FfLc3XTs9Ulycz+d7Z6n8VhZiwolscB55kbfRlUbVPXpVJRXddyqFvGs9VRuZ3NTqDEqO3WVVFd17LHVxEzWyxpcaO3AxSJum5ttTS+is5Wd/elSm47zSEBWsSAtU1dt7ZaDnXLeLY6UA/U9hBX9R6fu582szslrZY0TNJj7r6pbpkBTUJtD32FXrnBIUGpbPASPeC5lVHXpVJRXXPlBoDo0PgARIfGByA6ND4A0aHxAYgOjQ9AdGh8AKJD4wMQHRofgOjQ+ABEh8YHIDo0PgDRofEBiA6ND0B0Gn7reQCVmTJlShrfddddwdzUqVPTePXq1cHcrl270vjPf/5zMGdmaVzkLejKjj0+ANGh8QGIDo0PQHS49Xy8uPV8ndSrrhcsWJDGjz/+eDD3ox/9KI0PHDjQ53tMmzYtGJ85cyaN33333WBuwoQJVeXZn/w21qxZk8bPPPNM3bfXC249DwC9ofEBiA6HuiXx4x//OI1//vOfB3OTJ09O4927d9drkxzq1kmZ6jp7SowkTZ8+vc+f3bdvXxrnD5GPHj0ajM+ePZvGx44dC+ZOnDiRxh0dHcHchg0b0vi1117rM5c64lAXAHpD4wMQHRofgOiwxhcv1vjqhLp+z7e+9a1g/Pvf/77oFOqzxmdmj5nZfjN7JfPaWDNbY2Zbk+9jas0WKBq1Ha9KDnWXSZqTe22JpLXuPk3S2mQMtJplorajNODdWdz9BTObknt5rqTrk3i5pOck3VfHvICGo7brY/bs2Wnc1dXVxEwqV+2HG23u3p3EeyW11SkfoNmo7QjUfD8+d/f+FnfNbLGkxbVuByhaf7VNXbe2avf49plZuyQl3/f39YPuvtTdZ/AJIlpERbVNXbe2avf4VklaKOnB5PvKumUENBe1PYAf/OAHwfiSSy5J49tvv73i9/ngBz+YxvlL5BqtktNZnpT0d0mXmlmXmS1ST1HMMrOtkr6QjIGWQm3Hq5JPdef3MXVjnXMBCkVtx4uHDQEY0IwZ7y1lXnPNNcHc97///areM3+XlyJxrS6A6ND4AESHxgcgOqzxAXiftrbwgpXbbrstjdevXx/M9ffwo6zhw4cH41OnTlWZXe3Y4wMQHRofgOhwqAvgfaZOnRqMs8/Lfeihhyp+n+wh8+HDh2tPrE7Y4wMQHRofgOjQ+ABEhzU+ABo2bFgwHjVqVDDetGlTVe975MiRND558mRV79EI7PEBiA6ND0B0aHwAosMaHwBNnz49GM+dOzcY33///RW9z7hx44LxwYMHa0usQdjjAxAdGh+A6HCoC0Sqo6MjjfMPENq8eXMwfueddyp6z+ylbWXGHh+A6ND4AESHxgcgOqzxAZEYMWJEMP7KV76Sxtn1PklasGBBxe+bPYWlrKev5LHHByA6ND4A0eFQFxjCzCyNv/SlLwVzP/nJT9I4/3ChwWiVU1iy2OMDEJ0BG5+ZdZjZs2a22cw2mdndyetjzWyNmW1Nvo9pfLpA/VDb8apkj++0pHvdvVPS1ZLuMLNOSUskrXX3aZLWJmOglVDbkRpwjc/duyV1J/FRM9siaaKkuZKuT35suaTnJN3XkCyBBoihti+44II0/sxnPhPMZdf4BmP8+PHBuNIHipfJoD7cMLMpkq6QtE5SW1I4krRXUq+ro2a2WNLi6lMEGm+wtU1dt7aKP9wws9GSVki6x93fzs65u0vy3n7P3Ze6+wx3n1FTpkCDVFPb1HVrq2iPz8yGq6cwnnD3p5OX95lZu7t3m1m7pP2NShJolKFW26NHjw7G3/3ud9M4/5DwSm8uKoWHt9kHCLWqSj7VNUmPStri7g9nplZJWpjECyWtrH96QONQ2/GqZI9vpqQFkv5tZi8nr90v6UFJfzCzRZJ2SvpmY1IEGobajlQln+q+KMn6mL6xvukAxaG248Ula8AQ0tnZGYyzd2AZzB1X8o4fP57Gp06dqvp9yoJL1gBEh8YHIDoc6gItLntnlVtvvTWYW7VqVRp3dXVVvY1KHzbUKtjjAxAdGh+A6ND4AESHNT6gxU2aNCmNDx8+HMwtX7686HRaAnt8AKJD4wMQHQ51gRa3Z8+eNF6xYkUwd+jQoareM/uQIknquTvX0MEeH4Do0PgARIfGByA6rPEBLa67u7vXeDAmT54cjPfu3RuMT548WdX7lhV7fACiQ+MDEB0OdYFItLe3B+MRI0ak8a5du4K5s2fPFpJTs7DHByA6ND4A0aHxAYiOFXkpipm9qZ7H9Y2XdKCwDfcv1lwudveLCtrWkFbSupbKlU9RuVRU14U2vnSjZv9w9xmFb7gX5IJ6Kdvfr0z5lCkXiUNdABGi8QGITrMa39Imbbc35IJ6Kdvfr0z5lCmX5qzxAUAzcagLIDqFNj4zm2Nmr5rZNjNbUuS2k+0/Zmb7zeyVzGtjzWyNmW1Nvo8pKJcOM3vWzDab2SYzu7uZ+aA2zaxt6nrwCmt8ZjZM0iOSbpLUKWm+mXUWtf3EMklzcq8tkbTW3adJWpuMi3Ba0r3u3inpakl3JP8/mpUPqlSC2l4m6npQitzju0rSNnd/3d1PSnpK0twCty93f0FS/iEEcyX97xl8yyV9taBcut39pSQ+KmmLpInNygc1aWptU9eDV2Tjmyhpd2bclbzWbG3u/r+7N+6V1FZ0AmY2RdIVktaVIR8MWhlru+l1VOa65sONDO/5iLvQj7nNbLSkFZLucfe3m50Phh7q+v2KbHxvSOrIjCclrzXbPjNrl6Tk+/6iNmxmw9VTHE+4+9PNzgdVK2NtU9f9KLLxrZc0zcymmtl5kuZJWlXg9vuyStLCJF4oaWURG7WeB5c+KmmLuz/c7HxQkzLWNnXdH3cv7EvSzZJek7Rd0gNFbjvZ/pOSuiWdUs86zCJJ49TzKdNWSc9IGltQLp9Vz+7+RkkvJ183Nysfvmr+ezattqnrwX9x5QaA6PDhBoDo0PgARIfGByA6ND4A0aHxAYgOjQ9AdGh8AKJD4wMQnf8HtNe8jukH4xsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1220dbc88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(2, 2)\n",
    "\n",
    "ax[0, 0].imshow(conv1_[0,:,:,0], cmap='gray')\n",
    "ax[0, 1].imshow(conv1_[0,:,:,1], cmap='gray')\n",
    "ax[1, 0].imshow(conv1_[0,:,:,2], cmap='gray')\n",
    "ax[1, 1].imshow(conv1_[0,:,:,3], cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x112e54f60>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAD8CAYAAADub8g7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAEixJREFUeJzt3VuoVXW7x/Hf4+qgqaF5TuvdlZbo7mR2QBKMt4OCZFFERSEiLAK76CLY3kQQVF5U8EJRrF5NN3SEcGsQmogV5U5dgqGZpzywlqkr7Whhpj37wtFmtfA/1lzzNP5z/r+fm3n4rTXHgz4+jjnnf4xh7i4ASEm/ogsAgHpj8AFIDoMPQHIYfACSw+ADkBwGH4DkMPgAJIfBByA5DD4AyTmnkl82s5mS/iWpRdK/3X1RLz/PYSLxOOruI4ouIlZ96W36Oiol9XXZe3xm1iLpFUmzJE2S9JCZTSr39VB3B4ouIFb0dkMrqa8reat7k6Q97r7X3U9KekfSnApeD4gFvd3kKhl8YyV1dHvcmT0HNDp6u8lV9BlfKcysVVJrrbcD1BN93dgqGXwHJV3S7fG47Lm/cfc2SW0SHwKjYfTa2/R1Y6vkre4mSRPM7DIzO0/Sg5JWVqcsoFD0dpMre4/P3U+Z2eOSVuvMV/5L3P2rqlUGFITebn5WzzMw85YgKpvdfWrRRTQD+joqJfU1R24ASA6DD0ByGHwAksPgA5AcBh+A5NT8yA0A6Ro3blwwy1tR8t133wWzkydPVlSTxB4fgAQx+AAkh8EHIDkMPgDJYfABSA6DD0ByolnO0q9feAb/+eefdawEKN60adOC2W+//RbMDh8+HMzylohcfPHFufV0dXUFswkTJgSzYcOGBbOpU8PnEliyZEkwYzkLAJSBwQcgOQw+AMlh8AFIDoMPQHIYfACSE81ylsWLFwezzZs3B7NVq1YFsz179lRUE1CpWbNmBbPZs2cHswsvvDCYXXPNNcFs165dweznn38OZldccUUwk6SOjo5gtnr16mB29913B7O8s7N88MEHweyHH34IZqVijw9Achh8AJLD4AOQHAYfgOQw+AAkh8EHIDnRLGdZsWJFMHvuueeC2cCBA4PZ2rVrg9mhQ4eC2dixY4PZxo0bg1klLrjggmCWdzYOxG3fvn3BbOfOncEs7+wsb7zxRjD78ccfg1lnZ2cwO378eDCT8mvNk3fmlg8//DCYnTp1qqztlaqiwWdm+yX9Ium0pFPuHj7PDNBA6O3mVo09vtvc/WgVXgeIDb3dpPiMD0ByKh18LukjM9tsZq1n+wEzazWzdjNrr3BbQD3l9jZ93dgqfat7q7sfNLORktaY2Q53/7T7D7h7m6Q2STKz8MF5QFxye5u+bmwV7fG5+8HstkvSckk3VaMooGj0dnOzvDMk5P6i2UBJ/dz9l+z+GknPuHvwdCl5/zPmXexk8uTJwax///7B7IYbbghmLS0twez9998PZlu2bAlmDWYz31SeXV97O6+vBw8eHNzOiBEjgtnIkSODWd6/2Z9++imY7dixI5hV4sUXXwxmL7zwQlmvmbfcrBcl9XUlb3VHSVpuZn+9zlt5Qw9oIPR2kyt78Ln7XknXVrEWIAr0dvNjOQuA5DD4ACSHwQcgOQw+AMkpezlLWRtLfKHnkCFDglneWTVqhOUsVRJTX2ffRJ9VJf/Wp04Nt8rVV18dzLZv3x7MNmzYUHY9OUrqa/b4ACSHwQcgOQw+AMlh8AFIDoMPQHIYfACSw+ADkJxorrLWLPJOr/Xtt9/WsRKkqNy1eoMGDcrN33zzzWA2b968YHbs2LGy6qk19vgAJIfBByA5DD4AyWHwAUgOgw9Achh8AJLDcpYqY8kKGtF9992Xm7/77rvB7PDhw8Fs7969ZddUS+zxAUgOgw9Achh8AJLD4AOQHAYfgOQw+AAkp9flLGa2RNJsSV3u/p/ZcxdJelfSf0jaL+kBd/+hdmXGY9q0abn5+vXr61QJKhVzb9fqamkhl19+eW7e2dkZzDo6OqpdTs2Vsse3VNLMHs8tlLTW3SdIWps9BhrNUtHbSep18Ln7p5K+7/H0HEnLsvvLJN1T5bqAmqO301XukRuj3P1Qdv+wpFGhHzSzVkmtZW4HqLeSepu+bmwVH7Lm7p53JXl3b5PUJsV1xXmgN3m9TV83tnK/1T1iZmMkKbvtql5JQKHo7QSUO/hWSpqb3Z8raUV1ygEKR28nwHr7atzM3pY0Q9JwSUckPS3pfyS9J+lSSQd05iv/nh8Sn+21eEsQj83uPrXoIopUrd5ulL5+5plngtnOnTtzf3f37t3BbOPGjWXXVAMl9XWvn/G5+0OB6J99LgmICL2dLo7cAJAcBh+A5DD4ACSHwQcgOQw+AMlp+IsNXXvttcHsyy+/rGMlQPUMGTIkmA0fPjyYTZ8+PZg99dRTweyWW27JrSfvbDGNiD0+AMlh8AFIDoMPQHIYfACSw+ADkBwGH4DkNPxyliuvvDKYbdmyJZi9/vrrway1lRProljjx48PZgsWLAhmjz76aDBrtiUplWCPD0ByGHwAksPgA5AcBh+A5DD4ACSHwQcgOb1ebKiqGzP7Tmcu4CKducDL0bptvHcx1VOPWv7h7iNqvI0k9OhrKb1e6ota11NSX9d18P1tw2btMV3lK6Z6YqoFfRfT319MtUjx1MNbXQDJYfABSE6Rg6+twG2fTUz1xFQL+i6mv7+YapEiqaewz/gAoCi81QWQHAYfgOQUMvjMbKaZ7TSzPWa2sIgautWy38y2mtkWM2svYPtLzKzLzLZ1e+4iM1tjZruz26H1rgt9F1NfZ/UU1tux93XdB5+ZtUh6RdIsSZMkPWRmk+pdRw+3uft1Ba0vWippZo/nFkpa6+4TJK3NHiNikfa1VFxvL1XEfV3EHt9Nkva4+153PynpHUlzCqgjCu7+qaTvezw9R9Ky7P4ySffUtSiUg77uJva+LmLwjZXU0e1xZ/ZcUVzSR2a22cxiOfXyKHc/lN0/LGlUkcWgJLH1tRRfb0fT1w1/6vkquNXdD5rZSElrzGxH9r9VFNzdzYw1RyhHtL1ddF8Xscd3UNIl3R6Py54rhLsfzG67JC3XmbcsRTtiZmMkKbvtKrge9C6qvpai7O1o+rqIwbdJ0gQzu8zMzpP0oKSVBdQhMxtoZoP/ui/pTknb8n+rLlZKmpvdnytpRYG1oDTR9LUUbW9H09d1f6vr7qfM7HFJqyW1SFri7l/Vu47MKEnLs6tPnSPpLXdfVc8CzOxtSTMkDTezTklPS1ok6T0zm68zpzt6oJ41oe8i62up4N6Ova85ZA1AcjhyA0ByGHwAksPgA5Ccir7cMLOZkv6lMx/m/tvdF/Xy83ygGI+jXHMjrC+9TV9HpaS+LnuPL+JjE1GaA73/SJro7YZWUl9X8laXYxPRrOjtJlfJ4Cvp2EQzazWz9iJO+QSUqdfepq8bW80XMLt7m7Lz7PNZCJoFfd3YKtnji+7YRKBK6O0mV8ngi+rYRKCK6O0mV/Zb3QiPTQSqgt5ufnU9VpfPQqKyuaBT7Tcd+joqJfU1R24ASA6DD0ByGHwAksPgA5AcBh+A5DTEVdamTJkSzH7//fdgduBA+Hjl888/P5idOHEimJ0+fTqYSdKAAQOCWf/+/YPZoUOHghnQjCZOnBjMOjo6gtmvv/5a8bbZ4wOQHAYfgOQw+AAkh8EHIDkMPgDJYfABSE40y1nyloFs3bo1mA0dOjSYHT9+PJjlLS3Jrj5/Vueck/9HlrfN6667LpixnAWpufnmm4PZ6NGjg9nHH39c8bbZ4wOQHAYfgOQw+AAkh8EHIDkMPgDJYfABSE5DLGe58cYbg9m4ceOC2S+//BLM8pbBdHZ2BrO8OiXp/vvvD2bt7eFrT69bty6YtbS0BLM///wzmNXzeiqorrxlU6dOnSrrNcePHx/M9uzZU9ZrStKwYcOC2UsvvRTMpk4NXxrjySefLLueUrDHByA5DD4AyWHwAUgOgw9Achh8AJLD4AOQnIqWs5jZfkm/SDot6ZS7h7+f7kXeV/TnnntuMJs0aVIwu/7664PZjz/+GMy2bdsWzC688MJgJkl33HFHMNu0aVMwmz17djBbv359MMs7k8yxY8eCGfJVs7fLUe6SlTyVLFnJk9dn27dvD2aXX355MBs0aFBFNfWmGuv4bnP3o1V4HSA29HaT4q0ugORUOvhc0kdmttnMWqtREBAJeruJVfpW91Z3P2hmIyWtMbMd7v5p9x/ImobGQaPJ7W36urFVtMfn7gez2y5JyyXddJafaXP3qfX+cBioRG+9TV83trIHn5kNNLPBf92XdKek8NehQIOgt5tfJW91R0lani2nOEfSW+6+qtwX+/3334PZ119/HcyOHg1/6bZr165gdsUVVwSzvK/SFyxYEMyk/LNqTJkyJZgdOXIkmE2cODGYbdiwIbcelKWqvZ2yvN599tlng9nnn39ei3L+X9mDz933Srq2irUAUaC3mx/LWQAkh8EHIDkMPgDJYfABSA6DD0ByornYUN5ylm+++aas7IsvviirltGjRwezvOUqkvTaa68Fs7wzt+zduzeYnThxIpidPn06tx6g1ubPnx/Mbr/99mD2/PPPB7O8C4VVA3t8AJLD4AOQHAYfgOQw+AAkh8EHIDkMPgDJYfABSE406/hicvfddwezTz75JPd377rrrmC2ZcuWYJZ39batW7fmbhMo0qWXXhrMXn755WCWd9q4WmOPD0ByGHwAksPgA5AcBh+A5DD4ACSHwQcgOckuZ7n33nuD2YEDB8rKpPzTaw0cODCY7d+/P5jV+hQ9QG/a29uDWd6VDh977LFalFMx9vgAJIfBByA5DD4AyWHwAUgOgw9AcnodfGa2xMy6zGxbt+cuMrM1ZrY7ux1a2zKB6qO301XKcpalkl6W9N/dnlsoaa27LzKzhdnj/6p+eZUZMGBAMMu7ktr69euD2YwZM3K3+dlnnwWzsWPHBrMiz1SRsKVq0N6utokTJ+bmV111VTB79dVXg1neMq0i9brH5+6fSvq+x9NzJC3L7i+TdE+V6wJqjt5OV7mf8Y1y90PZ/cOSRlWpHqBo9HYCKj5yw93dzDyUm1mrpNZKtwPUW15v09eNrdw9viNmNkaSstuu0A+6e5u7T3X3qWVuC6inknqbvm5s5Q6+lZLmZvfnSlpRnXKAwtHbCShlOcvbkv5X0lVm1mlm8yUtknSHme2WdHv2GGgo9Ha6ev2Mz90fCkT/rHItVTd58uRg9sQTTwSzMWPGBLOdO3fmbrOlpaWs3z158mTu66L6Grm3q2369Om5ed4Sr8WLF1e7nJrjyA0AyWHwAUgOgw9Achh8AJLD4AOQHAYfgOQ0/MWG8i7gs2/fvmC2cuXKYDZ8+PBgtn379tx6+vfvH8y2bdsWzIBae+SRR4JZW1tb7u/Omzev2uUUij0+AMlh8AFIDoMPQHIYfACSw+ADkBwGH4DkNPxylj/++COYPfzww8GsX7/wzF+3bl0wO3r0aG49x44dC2anT5/O/V2glvL6L295l9R8S7HY4wOQHAYfgOQw+AAkh8EHIDkMPgDJYfABSI65B68FXv2NmX0n6UD2cLik/LUh9RVTPfWo5R/uPqLG20hCj76W0uulvqh1PSX1dV0H3982bNYe08WYY6onplrQdzH9/cVUixRPPbzVBZAcBh+A5BQ5+PJP+Vp/MdUTUy3ou5j+/mKqRYqknsI+4wOAovBWF0ByChl8ZjbTzHaa2R4zW1hEDd1q2W9mW81si5m1F7D9JWbWZWbbuj13kZmtMbPd2e3QeteFvoupr7N6Cuvt2Pu67oPPzFokvSJplqRJkh4ys0n1rqOH29z9uoK+Zl8qaWaP5xZKWuvuEyStzR4jYpH2tVRcby9VxH1dxB7fTZL2uPtedz8p6R1JcwqoIwru/qmk73s8PUfSsuz+Mkn31LUolIO+7ib2vi5i8I2V1NHtcWf2XFFc0kdmttnMWguso7tR7n4ou39Y0qgii0FJYutrKb7ejqavG/4MzFVwq7sfNLORktaY2Y7sf6souLubGV+9oxzR9nbRfV3EHt9BSZd0ezwue64Q7n4wu+2StFxn3rIU7YiZjZGk7Lar4HrQu6j6Woqyt6Pp6yIG3yZJE8zsMjM7T9KDkvJP+F8jZjbQzAb/dV/SnZJiuLjASklzs/tzJa0osBaUJpq+lqLt7Wj6uu5vdd39lJk9Lmm1pBZJS9z9q3rXkRklabmZSWf+LN5y91X1LMDM3pY0Q9JwM+uU9LSkRZLeM7P5OnPWjwfqWRP6LrK+lgru7dj7miM3ACSHIzcAJIfBByA5DD4AyWHwAUgOgw9Achh8AJLD4AOQHAYfgOT8H5Jkeld+yMR0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1220dbb00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(2, 2)\n",
    "\n",
    "ax[0, 0].imshow(conv2_[0,:,:,0], cmap='gray')\n",
    "ax[0, 1].imshow(conv2_[0,:,:,1], cmap='gray')\n",
    "ax[1, 0].imshow(conv2_[0,:,:,2], cmap='gray')\n",
    "ax[1, 1].imshow(conv2_[0,:,:,3], cmap='gray')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}