NEW_PRAC_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data as mnist_data\n",
    "mnist = mnist_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZgAAADBCAYAAABCFNVcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeYVdXZ/vH7ERSVZldQBAuioK8iauzoawka1JiYaDSK\nFWOJMa+a2Hv4xRKNFcUSMLG3SIIxlthbFGNDIyCCqKgoSBELZf3+2HvkrLUPc85Zc+rM93Ndc8Gz\nZ+911plzc2ZmuX2WOecEAAAAAAAAAECplqr1BAAAAAAAAAAAjYkFZgAAAAAAAABAFBaYAQAAAAAA\nAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABAFBaYAQAA0Cwze8LMnJk9UaXHW9fM\n/mlms9LHfbUaj1suZjYynffkWs+lJcxs1/R5ODPbvsRrL0yvW1Cp+QEAAKA+sMAMAADqipktbWan\nmtlbZvalmc02s3fN7AEz26rW8yvEzHrlLMplPmo9vwZxmaTdJS0v6WVJr9d2OvnlvK7nVvExd2ou\nXzkfvcrwcLMkvZh+zC7x2qnpdS+UYR5FM7NlzOy04P1jopn91cwGlDhW+5yv55mVmjMAAECja1/r\nCQAAAAQulnRi+veJkr6S1FPS3pLul/TvGs0rxoeSPqj1JBpQv/TPu51zBzZ3opkt45z7tgpzqhez\nlSzcNukvaRlJcyS9lXP8m3wXm9nSkhY45wr+xw7n3EuSto6ZpHPueknXx1zbQn+QdHz69wmSvlby\n/rGPpHskja3BnAAAAFo17mAGAAD15mfpnxc453o75/5H0gqStlGwuGxmh5nZWDP7Kr1b8QUz+0nO\n53PvJj7ZzO4zs3lm9qaZbW9m/c3s3+m1z5hZn2D83c3sX+ldkF+b2YtmtlcJz+VG59zWuR/puAPM\n7Nt0XkemxzZI5+bM7JT02MHp/D4zs/lmNjNtHfHdndzBHa2Hm9nj6dfjRTPbyMx2S5/vHDN70MzW\nyLn2u9YXZvZLM3s/vfZBM1uruSeW3il6lpm9Y2bfmNnnZnZb7nVmtrqZ/dnMPkrP+dTMnjazny9h\nzF7pXd7rp4d+ls5vZPBa/ia9I3Wekv8gITNb28xuMbOP06/Vh2Y2wsxWyxn/u9YVZvbz9M85Znat\nmS1rZsPMbEY637Obee47mX83+jlN4+Y5d++cu2mfamnGnHOvBHmaln7qlSBr08zsL+m8JprZEWb2\nnpKF545mdqiZvZSTrRlm9g8z2yJnbpkWGZbT+sLMdjGz/6S5HWtmW+Zcm2mRYcm/MWdmj5rZCWY2\nJf36jzaz1XPO62Bm15jZF+m8/mhmF4XjLUHT+8c5zrkNct4/tlVyN3zu1/4gS/59zUtfn8fMbJum\n5y5pfs7pFzR9LQs8PgAAQJvDAjMAAKg3TT+f7GZme5nZGi7xgnPuuzs0Lflf1m+WtLmk6Ur+d/7v\nSbrLzI7OM+6FkjaTtEjJHbL3SnpYUldJS0vaLh2vafz9JD0kaed07KmStpL0QPq5aM65sZKaFjAv\nNbO1JY2UtJykf0m6NP3c9yRtIulzSePSz+8u6dHcheIc10rqruRruJWkv0v6q6R2kjpK2kPJHZ6h\nrSVdJGmupA7pefcWeBr3SjpfyWLwO5JMyeLes2a2Ys58fi6pi6Q3Jc1TstC30xLG/EbJ3blNdyR/\nltbvBuddoOR1eVfSgnQR+XlJBytZTBwvaVVJR0l6xsw6Bdd3k3SdpAWSOkk6RsmdrcelX4Nuks4z\ns92WMM/wLuIP0/o/eR7nLiVfm+Uk7aAqZSzQQ9INSr6+09NjW0vaWIuz1VHSIEmPmdmqRYzZTtIY\nScsq+fezuaQ7zKxdEdfuKOn36Xw6SdpL6X8oSP0/Sccq+bc5S0mGjitiXGnx+8fuZjbYzFZP3z+e\nd879t+kkM/utpL9I2lLJ6zdD0v9KesKS/4DT1B6kyQfK/xoDAAC0eSwwAwCAenNt+ufWkkZLmmbJ\nXbLnm9nykmRmHSWdnp43WlIvJf8b/NPpsQvMLPw550lJ62lx+43VJP3VOddHyeKqJG1rZsulf79Y\nycLgbZLWds71lnRjemxYkc+l6c7Wpo+/5nzuYklPKVlE+7eSO7RnSDokp33BlZJWds71cc5tpmRB\nUJI6S/pBnsf7c/p8LknrdSUNc85tJOnW9Nguea5rJ2lz51xfSf+XHtvKzHbO96TMbEdJg9Nyj/Qu\n0XWVLAivrWRxUJI2SP88xjk3wDnXS9Lqkq7KN65zblpwV+6Y9G7cC4JTJ0nq6ZzbRNJvlSw+dpfk\nJO3gnOunpKWKJPWWdFhw/TKSvp/Ob2rOeZtK6qukrYKU/2v13V3EOYea7lTfN8/j/CT9+v8xPVbu\njBVjGUlHOec2VLLo/aWkyyWtlJOtTdNzu0jas8hxf50+t9+m9bqS1iniuqUkbeWc20DS39Jju0iS\nmXXW4vzcl465rqRPipxT0/vHdunYH5vZf83s3Jz3j06SzknPuyD9uvdU8h93lpF0XtoeJHdjw+vT\n1/gnAgAAgIcFZgAAUFecc+dK+pGkB7R4Y7ENJJ0l6Za07qfkjlBJutM5t8g5N19Jj1UpuXu1ZzD0\nmHThdnLOsabFrUk5x1ZL7+BsWig7UNKitCXCkemx3ma2chFPp+nO1qaPd3Ke5yIld9zOVrLoKkm/\ncM59mHP9CkruZp1hZouU9JRt0j3P4zU9n8l5jjU9x9WV9XrO3Z135hzfOM+5UnJndZN/pl+bmZJW\nSY81Lb42PfZISzZqfFDS0ZI+WsK4xRrlnPtCkpxzC5XchSpJE9OFQTnnHkrnJElbBNfPdM49m74G\n76fH3nTOTXbOzZX0aXos39eqFLOcc01fg9z+yOXMWDHmKr1zOv234iStKOlvlrRdWSTp7Zzz82Ur\nnz+nf+Y+t2K+Zq86594Mrm26rreSu+il5N+2c87NVnK3dEHOuTMl7acke03vH32ULCg33T2+iRa/\nf5yVft0XKrmDWYrsOw0AANBWsckfAACoO865+yXdb2am5H+9H5H+uVeeO5OL1bTYtCDPsdx+uhZc\n954WLzjmWrqIx7wxXTBfktW0eKFLWtx7uOkuy38qWWT+Wsn/mj9fixd387UiKPY5ltO/84zdtGh7\nhqRnldwtvLGSO0L3kPQTJe1KYhV7N+uSzM75+4I8x5qeT5iFUn2R53HyjduSjBXj09xN/cysi5LW\nHF2VbKJZTLZCC9PFeKn555ZPvq9LvuuiMuucu1fSvel7xeZK2oNsJmnv9D0l19vyX/vcOQEAAKAI\n3MEMAADqSro52GaSlN69OFZS0921c9K7TscpWRiTpP3NbCkzW1rJnYtS0md2SuwcnHPTtfgu4DeV\ntF1o2lTtp5L+n3Pu49jxJSn93/VvVbKI2NTX9Twz2zz9ex8li8uSdLhzboAWt/cot//J2XwutwXA\nm/lOlvRSzt8vy/nabCPpN5KuTz+3naQnnXMnOOf+V9LQ9PimLbw7N1x4bJrP+k0bzZnZICV36UrB\n5m5l1JTBjqVeWI2M5T5cUG+kZHFZkoak2Tq5TI/VUuOV9GaWpB9L3y2I52sJk2HJRo2bSt/drf2y\nFv+fA3PShfY3tLgNyiOStsn52h8m6dz0cwu1uB+49xqb2X5p643/5m5QCAAA0BaxwAwAAOrNkZL+\nY2bTzWysmU1R0kJAkm6XJOfcl1rco3ZvJQt1k5VsoiZJZ6UL0S1xavrnXkr6QP/HzD5KH+fXRY5x\npJm9EHx0Sz93uZLWH1MkDVTS3mNpSbemPXonKemVK0k3mdnrSjbsq4RvJL1iZuO0uFfwy865x/Od\n7Jx7QtI/0vIOMxtvZm8o2RjtSSV3jUrJRm6fm9lEMxurxS0KPlDSb7pcrlHSt9kkPW1mbyrpzS1J\nEyX9qYyPlavpP3ycYGYvmVmpfZPLkbEYE5VsuCgl7Ute1+L2MjWV3hXd1Ed5fzObpOTfQr5NLfMZ\nKulVM/s0ff94X9L+6eduy3mMpr7eJ0j6MP3af6qkZccB6XlOixenf21m/zazputWUPIfgfqofHea\nAwAANCQWmAEAQL05U0n/5TmSNlSysDRByYLyd3dZOuculHS4pFeU9FxeUUmf4/2dc9erhZxzdypp\n59C08ddGSu56vFvSpUUOs6aStgO5Hx3MbG8lC2FO0hHOuTmSjlHS+mFDSZc652YquZv4LSU/s32r\nZCGyEl5WsqDZKX2ch5T0wW7Ovkr62v5XSb/rtZQsBP5B0hPpOXcqaaHRWUnf2zlKXts9cls2tJRz\n7lMlfXP/rKT9Qh8ld7HfKGm7nFYO5XaCkrthpaTP8wbNnJtRpoyVzDn3uZK7pN9W0g7ja0n7VOrx\nIpymZJF5lpJ/17dLGpV+7qslXZQ6Xcl/XJir5N/S6kruir5QizcjlHNumKRDlLxndFXy2s2UNFKL\n/0OIJP1SyR3mSynp9d07+lkBAAC0UlbGn+0BAADQQMzsCSV3Tz/pnNuptrMBEma2hqR56eZ+Te1k\nxipZMH7GObdDc9cDAACgutjkDwAAAEA92V7Sn8zsZSV3vW+l5E7kBZLOruXEAAAAkEWLDAAAAAD1\n5F0lG1/+j6Q9lfzO8oCk7ZfUFxwAAAC1Q4sMAAAAAAAAAEAU7mAGAAAAAAAAAERhgRkAAAAAAAAA\nEIUFZgAAAAAAAABAFBaYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABA\nFBaYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABAFBaYAQAAAAAAAABR\nWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABAFBaYAQAAAAAAAABRWGAGAAAAAAAAAERh\ngRkAAAAAAAAAEIUFZgAAAAAAAABAFBaYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUF\nZgAAAAAAAABAFBaYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABAlBYt\nMJvZIDN7x8wmmtmp5ZoUWjdygxjkBjHIDWKQG8QgNygVmUEMcoMY5AYxyA1KYc65uAvN2kkaL2k3\nSR9IeknSz5xzb5VvemhtyA1ikBvEIDeIQW4Qg9ygVGQGMcgNYpAbxCA3KFX7Fly7laSJzrlJkmRm\nd0jaR9ISw2ZmcavZqHvOOSvyVHKD75AbxCA3iFGp3JCZVu0z59yqRZ5LbiCJ71GIQ24Qg9wgBrlB\njGJy05IWGWtKmppTf5AeA5pDbhCD3CAGuUEMcoMmU0o4l9ygVGQGMcgNYpAbxCA3KElL7mAuipkN\nlTS00o+D1oXcIAa5QQxyg1KRGcQgN4hBbhCD3CAGuUEMcoMmLVlg/lBSj5x6rfSYxzk3QtIIidvl\nIYncIA65QQxygxgFc0NmkAe5Qan4HoUY5AYxyA1ikBuUpCUtMl6S1NvM1jGzZSQdIGl0eaaFVozc\nIAa5QQxygxjkBjHIDUpFZhCD3CAGuUEMcoOSRN/B7JxbYGbHS/qnpHaSbnbOjSvbzNAqkRvEIDeI\nQW4Qg9wgBrlBqcgMYpAbxCA3iEFuUCpzrnp3sHO7fOtVwk6kJSM3rRe5QQxygxiVyg2ZadXGOue2\nqMTA5Kb14nsUYpAbxCA3iEFuEKOY3FR8kz8AQMsMHDjQq5944gmvvvnmmzPXnHvuuV49derUzDkA\n2o727bM/8p155plefdZZZ3n1scce69UPPvhgZozPP//cq+fNmxc7RQAAAAANqiU9mAEAAAAAAAAA\nbRgLzAAAAAAAAACAKCwwAwAAAAAAAACisMAMAAAAAAAAAIhizlVvk0d2lGy92IkUMchNccJN/v71\nr38VvGbChAlevdtuu3l1I2/6R24Qo1K5aZTMrLbaapljb775plevtNJKzY5hlv0SPvfcc1790Ucf\nefWNN97o1ePGjcuMEV5TR8Y657aoxMCNkhuUju9RiEFuEIPcIAa5QYxicsMdzAAAAAAAAACAKCww\nAwAAAAAAAACisMAMAAAAAAAAAIhCD2aUBX18EIPcFGettdby6pNPPtmrjznmmMw17du39+rx48d7\n9Y477ujV06dPb8kUq4rcIEZb78Gcz7777uvVd999d7Pn5+vBXOrPkaecckrm2OWXX17SGFVED+Yy\nGD16dObYXnvt5dU9e/b06vfff7+ic6okvkchBrlBDHKDGOSmdjp27OjVvXr1KnmM8Pf2Tz/9tCVT\nKho9mAEAAAAAAAAAFcMCMwAAAAAAAAAgCgvMAAAAAAAAAIAo7QufgnrWvXt3rz7hhBO8+pBDDslc\ns8Yaa3h12FMxX49FNI6ddtqpqGO5zj333IrMBeXxwQcfePWJJ57o1X379s1cs8suu3j1Bhts4NWH\nH364V1966aWZMRYuXFjSPAE0lgcffNCrd9ttN6/ec889vTrfzwfdunXz6v3337/Zx/z1r3+dOTZt\n2jSvvuOOO5odA40lX5/uRYsW1WAmaCT5focJe3XHCHvPb7jhhl593nnnefVFF13U4sdEvAEDBnj1\nyiuv7NUrrriiV1977bWZMcJzwn1InnnmmZZMEQC+E667hHW/fv28OvyeJGV/3g5/jpoyZYpXDxs2\nLDPGTTfdVGiqFcEdzAAAAAAAAACAKCwwAwAAAAAAAACisMAMAAAAAAAAAIhi+fqiVezBzKr3YGUW\n9kEZOnSoV7///vte/Y9//KMi81h66aW9+uyzz/bq008/veAYs2fP9urp06d7ddirtRjOuYo1bm7k\n3JRDoT4+55xzTlXm8cQTT3j1zjvv3OIxyU159OjRI3Psb3/7m1dvsskmzY4R9m+XpGuuuaZlE6sQ\ncpN975fK00u9UM+vsFf3rFmzMmMMHz7cq+fMmePV8+fPb8kUo1UqN42SmWoJf4Z44YUXvDrshSlJ\nzz33nFdvt9125Z9YnLHOuS0qMXBrzs3GG2/s1Q888EDmnF69enn1Ouus49Xhz9WNpC1+j/rFL37h\n1V27di35mnBfmfbts1sFVWOfmPB7VjHPpRzaQm46d+7s1fvtt59X/+hHP8pcs+uuu3p1hw4dvDpm\nLSPcC+DKK68seYx60RZyg/IjN/kttZR//+2BBx7o1fvss49Xh/3cJalLly5eHa7fFaPQ72OhfL9b\nHXXUUV79l7/8peR5hIrJDXcwAwAAAAAAAACisMAMAAAAAAAAAIjCAjMAAAAAAAAAIAoLzAAAAAAA\nAACAKGzyV6TbbrvNq/fff3+vnjx5slcPGDAgM8YXX3xR0mO2a9cucyzcxKmYTf1CgwcP9upybEhI\no/g44es5cODAzDnhpn71ohyb/pGbyll77bW9evTo0V4dbvo3ceLEzBh77LGHV0+aNKlMs2sZciM9\n/fTTmWPbbrttDWZSWJijhx9+uCbzYJO/2gg3+dtyyy0z57zxxhtevcMOO3h1uOlWFbHJX4S9997b\nq++///6C17DJX3HqJTcTJkzw6nXXXderq7EZX7mEG77ddNNNXj137tyqzKPRc7Pyyitnjp1//vle\nPWjQIK8ON/v88ssvM2O89tprXv3888979WOPPebVo0aNyoyx6qqrevVTTz3l1eFGggsXLsyMUa8a\nPTcxVlllFa9+9NFHM+dsuummXh2uec2ePdurjzzyyMwY99xzT+wU615bzE1os802yxz77W9/69U/\n+clPmh0j3/e6MGsfffSRV4drgo8//nizjyFlN08Of+fLt5Hgu+++69V9+vQp+DiFsMkfAAAAAAAA\nAKBiWGAGAAAAAAAAAERhgRkAAAAAAAAAEKV9rSdQj8LepZL0wx/+sNlrwh5SJ510Uuacs846q6R5\n9OvXL3Os1J7L+fpQhb1zUT1hj51y9FcOX88nn3yy5GuKyUTYL/qcc85p9vNhjeoK+1eedtppXn3f\nffd59frrr58ZI+zPHvaWr1ZfQmStvvrqtZ5C0bp161brKaCG7rzzTq/O14M57C0X/vwT9nEGUFsd\nO3b06mr1XB4zZoxXhz/rhO8348aNKzhm2Pf366+/jpxd2xL+THj11Vdnztlqq628OuxFeuWVV3r1\n8OHDM2OMHz++pHlNmzYtcyzswRz+zNupUyevnjVrVkmPicrq27evV997771ena+3bKF9xrp06eLV\nN954Y+acHj16ePXll1/e7JiobyuttJJX33HHHZlz8v0+XKo//OEPXh3mZt68eV4d9gMvxh//+Eev\nPv744zPnrLfeeiWPWw7cwQwAAAAAAAAAiMICMwAAAAAAAAAgCgvMAAAAAAAAAIAo9GBWtn9y2N9L\nkjp06NDsGJMmTfLqSy65pOR5bLjhhl4d9j8txieffOLVF154Yeacr776quRxESfsQxzTc/m8887z\n6pj+ydVQqCczait8Pwn7FubrORUeW2aZZco/MUTZfffdM8fWWWcdr1555ZW9eoUVVvDqd999t+TH\nXXrppb16xIgRmXPCnnUHH3ywV+fbGwCt1+abb17rKQBooSFDhnj1aqut1uz57733XubYmWee6dU9\ne/b06mK+N8yYMcOrv/nmm4LXoDKuuuoqr95ss80y54Q/IxxzzDFln0e4d1K+vZSee+45rw7zTM/l\n+rb33nt7ddhzeebMmZlrrrvuOq8O++AOHjzYq6+44orMGOHvsi+++KJXh7lCfQvXTHr37t3iMd96\n663MsbC3/Mcff9zix+natatXh79bVWsfhGJwBzMAAAAAAAAAIAoLzAAAAAAAAACAKCwwAwAAAAAA\nAACiFOzBbGY3Sxos6VPn3MbpsZUk3Smpl6TJkn7qnMs2v2kQ22yzjVeHvZDzmTdvnleHPaVmz55d\n8jxOP/10r15jjTUKXvPZZ5959e9//3uvDntDV0tbyE0oX8/hsC9xKOwFFPZbzndOJYS9ofPNu1D/\n6HLMsy3mplbuuOMOrw57IzaStpibyZMnF3Ws0v72t79ljh177LFe/fbbb1drOiVpi7mphYEDB3p1\nvj5xU6ZMabauJ+QGMRo9N+HvSkst1fw9ShdddFHm2DPPPOPV7dv7v4buuuuukbNb7KmnnvLqen4v\nKaTeMjNgwACv7t+/v1fffffdmWsq0XM5NHz4cK8O95uQpIsvvtira/X7cTXUW27KYb/99mv283/5\ny18yx84444xmrxk5cqRXf/7555lzHnjgAa/eZ599vLo19WBujbkJ9evXz6udcyWP8fDDD3v1UUcd\nlTnnww8/LHncUNhz+a677mr28zHPpVKKuYN5pKRBwbFTJT3mnOst6bG0BnKNFLlB6UaK3KB0I0Vu\nULqRIjco3UiRG5RupMgNSjNSZAalGylyg9KNFLlBGRRcYHbOPSVpRnB4H0lNW/2OkvTDMs8LDY7c\nIAa5QQxygxjkBjHIDWKQG5SKzCAGuUEMcoNyKdgiYwlWd85NS//+saTVl3SimQ2VNDTycdC6kBvE\nIDeIQW4Qo6jckBkEyA1ikBuUip9tEIPcIAa5QcliF5i/45xzZrbEph/OuRGSRkhSc+ehbSE3iEFu\nEIPcIEZzuSEzWBJygxjkBqXiZxvEIDeIQW5QrNgF5k/MrJtzbpqZdZP0aTknVWmHHHKIV19wwQUl\nj/HQQw959aOPPlryGNdcc41XH3jggSWPceedd3r1lVdeWfIYVdTQuSmk0IZ++ey8884VmElh4YZ9\njz/+eMljhJv6VfC5tOrc1Mq7775b6ylUGrmpE/k2Aqxj5KaF9t13X69eccUVvTrfRiThhijTpk3L\nnFPnyE2OlVdeudZTaBR1mZtDDz00c+zII48saYxLL700c+yss87y6jXXXLOkMYsRvne8//77mXN+\n9rOfeXUtNshtgZplZuzYsV4dfq3333//zDUvvviiV998881e/dVXX5U8jw4dOnh1uPlgvs3aqrFh\nep2ry/eaYs2YEXZu8L333nstfowHH3wwc6xnz55evWjRohY/ToNp6NyErrrqKq/O95710ksvefWw\nYcO8+uWXX/bqBQsWtHheyy23XOZYuMHk9ttv3+wY8+fPzxwLN9atlmI2+ctntKQh6d+HSHqgmXOB\nJuQGMcgNYpAbxCA3iEFuEIPcoFRkBjHIDWKQG5Ss4AKzmd0u6XlJfczsAzM7QtLvJe1mZhMk7ZrW\nwHfIDWKQG8QgN4hBbhCD3CAGuUGpyAxikBvEIDcol4ItMpxzP1vCp3Yp81zQipAbxCA3iEFuEIPc\nIAa5QQxyg1KRGcQgN4hBblAuLd7krxH07dvXq8Oey2uttVbBMWbPnu3Vv/vd70qex/e+9z2vDvu+\nmFnBMUaOHOnVJ5xwQsnzQNsT9lgOezAXI+yxTD+zxrbOOuvUegpoQOH7wJZbblmjmaBehP12w351\n+XrLhSZMmFDWOaG2wgygsZx55pmZY0stVVpXxU6dOhV1rNy6devWbC1JY8aM8eqTTz7Zq//xj3+U\nf2Kt0BlnnOHVYX9TKbs30NFHH+3V48aN8+p77703M8Y999zj1YcddphXr7rqql6db2+lOXPmZI6h\ncbzwwgteveuuu3r1z3/+88w1YW4++OCDZh9j4cKFmWNTp05t9pr111/fq/Oty9xxxx1e/dxzzzU7\nJirnxBNPbLauln79+nn1qFGjMuf079/fq/PtX5Lr7rvvzhwL952rltgezAAAAAAAAACANo4FZgAA\nAAAAAABAFBaYAQAAAAAAAABRWl0P5tVXXz1zLOy1VUzP5dCgQYO8+tVXX/XqDh06ePXAgQMzY9xy\nyy1eveKKK5Y8j7B/0LrrruvVkyZNKnlMNLZy9FcO+ymHfVbR+hx44IG1ngIa0ODBg706Xw/mBQsW\nePW3335b0Tmhtg444ACv7t27d7Pnz507N3PsiiuuKOucUF2nnXaaV6+yyioFr7n88su9+sMPPyzr\nnBAv3+9SlfDZZ5959XvvvefV+XpOzpgxo9kxw76V+Z7LRhtt5NWjR4/26rCH7/nnn9/sY7ZVt99+\nu1e//vrrmXPC/taHHnqoV2+88cZeHe5PFOPHP/5x5li470jYG3rs2LEtflxUzg033ODVBx98sFdv\nvvnmmWvWTaiTAAAW70lEQVTefPNNr77++uu9+v777/fqLl26ZMYIj4W9n8Off/KN0b69v9xGD+a2\nJ9yLJPz5Z7PNNit5zKeeesqrjz/++NInViHcwQwAAAAAAAAAiMICMwAAAAAAAAAgCgvMAAAAAAAA\nAIAora4Hc9jbSZLWXnvtFo+7wgorePUDDzzg1Z07d/bqfD2Yy+HMM8/06rD3z9Zbb+3VM2fOrMg8\nUDv5etKVKuyxHPZgRuVstdVWXr3mmmtmzgn/3dbq9Qn7IdJLt+056KCDvPpXv/qVV8+fPz9zzdln\nn+3VvL+0HiNGjMgcO+KII5q9Juy5HPbklKTXXnutZRNDTS2//PJevdRShe9f+fLLL7164cKFZZ0T\n4uXbz2WTTTZp9prnn3/eq7/++uvMOVdffbVXv/POO15dTB/uWbNmNfv5Pn36eHX4PUuSfvGLX3h1\nu3btvPqkk07yanowF2fcuHGZY4cddphXn3rqqV693377efWyyy6bGaNXr15efdxxx3l1+HtR3759\nM2OEx8Jez2Hm//73v2fGCH8mHj58eOYcVMbUqVO9Ouy5fO2112auCV/jU045pdkakKTVVlvNq1dd\nddUWj3nNNdd49Q477FDwmvDnqEceecSr99lnH6/+6quvImdXftzBDAAAAAAAAACIwgIzAAAAAAAA\nACAKC8wAAAAAAAAAgCgsMAMAAAAAAAAAolg5Ngwr+sHMyv5gxxxzjFeHmwtJ2Wbdrdkee+zh1Q8/\n/HBVHtc5Z5UauxK5qYTHH388c2ynnXZq9przzjvPq/NtDllojHADrXDMfOfUi9aYm44dO3r1IYcc\n4tXDhg3z6i5dumTGmDdvnleHr9/hhx/u1dOnTy84r3BjrQsuuMCrl1lmmcw1xx9/vFfXy4YmrTE3\n9WLIkCFefd1113l1mJNw4xUpuyFPvahUblpzZsJ/80OHDs2cU+jnyN12282r832vrGNjnXNbVGLg\n1pSb8PvJ6aef7tXhhn6StO+++3r1Y489Vv6J1Uijf4/q0aNH5tj666/f7DXFbPJXC927d88ce+WV\nV7w6/D1xzpw5Xt21a9fyTyyPRs9NpfTr18+r33jjDa++9dZbvfqSSy7JjLHLLrt4dbi5YLiJZfiz\nfD6jRo3y6nDTuM8//7zgGOVAbvJvDrnxxht7dfizSOfOnb160KBBmTHefvttrw5/5v3Nb35TcG57\n7723V+fbQLIW2kJuwu9b/fv39+rwtZGkbbfd1qt79uzZ7GOYZb+M5VhfDTeoDd/nvvjiixY/Roxi\ncsMdzAAAAAAAAACAKCwwAwAAAAAAAACisMAMAAAAAAAAAIjSvtYTaKk999zTqxu53/Jdd93l1QsW\nLMicc+CBB3r1Pffc49XPPfdc+SeGojz55JOZY4X6J59zzjklP06hnsv12m+5NVp++eUzx2655Rav\n/uEPf9jsGPler969e3t1+D43duxYr87XGzmcW9iDOeyle/fdd2fGuOGGG7ITRqtx6KGHZo5dddVV\nXh3mZOHChV79u9/9ruzzQvWsvPLKXn399dd7dfjek0/YZzIc49lnn42cHVqLuXPnZo61pp7LrU2+\n3vr5jjWCRYsWZY7Nnz+/BjNBrLBfcij8+faTTz7JnPP666979eWXX+7VG2ywgVeHexpJ0o9//GOv\nDn+G+tGPftTs+RLve5WSr+f7yy+/3GwdCvcOyCfsxxv2YM6338Cbb75ZcFyUbpVVVskcC38nOeCA\nA7y6U6dOXl3NvehyTZkyxavD9zBJuv/++6s1nbLjDmYAAAAAAAAAQBQWmAEAAAAAAAAAUVhgBgAA\nAAAAAABEafgezD/4wQ+8uhy9VF577bXMsbC/7gknnFDyuJMmTfLqYcOGeXXYuzXfcwl7tMyaNcur\n8/UgQnXk66Ub02M5FPZYPvfcc1s8Jspj4MCBmWNhz+XwvePiiy/26qeeeiozxpFHHunVp5xyilev\nueaaXn3hhRcWnmxg4sSJXn3qqadmzsnXBx6Na8iQIV4d9luW8vcVz3X11Vd7NX2669eAAQO8escd\nd8ycs++++3r1tttuW/LjhP11Z86c6dUdOnTw6m+//bbkx0B9WWop//6U8DUOhX25gUpZdtllvbp/\n//6Zc9Zaay2vDn/fytfDF9XRtWvXzLGhQ4d69SOPPOLV5Xi9xo8f32wtSVdccYVXh3snhb2izz//\n/MwY9GBubJ07d2728/l6ME+ePLlCs2lbwn9f+dZYNtpoo2pNp0XCTDRyv+V8uIMZAAAAAAAAABCF\nBWYAAAAAAAAAQBQWmAEAAAAAAAAAURq+B/P3v/99r/7lL3+ZOeff//63V99+++1e/f7773v1okWL\nMmPsvvvuXh3Tg/mwww7z6meeeabkMegLVj922mknr3788cdbPGbYb1mi53I9Gzx4cMFz/vSnP3n1\nQw89VPCasF/7nDlzSptYHmHP5fC9kx5hja9jx45eHfbVDnv4L7PMMgXHHDFihFfzflQd4XvLzjvv\nXPCa3r17e3W4R4WZZa4pdd+KfGP06tXLqy+99NJm66effjozxn333efVV155ZUnzQnX16NHDq086\n6aRmzx87dmwlpwN8J/y+d/bZZ2fOCd/3wr0ELrnkkvJPDEVZbrnlMsfWWGMNr66Xn0N+85vfeHX4\nc/U666xTzemgCvbYY49aT6HNOvbYY726Ufot59O9e3evXmmllTLnzJgxo1rTKTvuYAYAAAAAAAAA\nRGGBGQAAAAAAAAAQhQVmAAAAAAAAAEAUFpgBAAAAAAAAAFEafpO/Rx55pNk6RufOnTPH7rrrrpLG\nePXVVzPH3n333eg5ofbCTfzCTf7Q+i2//PJePWDAgILXXHDBBV591FFHeXXfvn0z13Tq1Mmrl156\n6WKn+J0xY8Z49YEHHujVc+fOLXlM1I98G/QNGzbMq48//vhmx1i4cGHm2NFHH+3VI0eO9OpSN4VD\nfuHPGeH7xBFHHOHV4XuPVPi1KOa1KsfrWeoYO+ywQ+ZYu3btvJpN/oD60qVLF6++6KKLvDrcTG/6\n9OkVmce6667r1eHm2DvuuGPJY/7nP//x6nBTZNSXr7/+utZTkJTdHPvLL7+szUSAVmj77bf36oED\nB7Z4zHwbVZdq5syZXp1vjfCOO+7w6ltuucWrN9hgA68O15gkadNNN42dYs1xBzMAAAAAAAAAIAoL\nzAAAAAAAAACAKAUXmM2sh5k9bmZvmdk4M/tVenwlM3vEzCakf65Y+emiUZAbxCA3iEFuUCoygxjk\nBjHIDWKQG8QgN4hBblAuxfRgXiDpJOfcK2bWWdJYM3tE0qGSHnPO/d7MTpV0qqTfVm6q1XPSSSdl\njuXrf5jrm2++8epf//rXmXOmTZvWsok1lobPDT2Xa6KuczNv3jyvvvrqqzPnjBo1yqt79OjRbB3j\ntttu8+oLL7wwc86UKVO8ul561lVIXeemEvK95oV6Loffp371q19lzvnTn/7Usok1jppmpl+/fl5d\n6LUrh9deey1zbPz48SWNEfYqlaT+/ft79ZprrunV22yzTcF5XHbZZSXNo4ba3HsNyqLhc3PQQQd5\n9ZZbbunV8+fPL/tjrrLKKpljxx13XLPzKsbUqVO9+oknnih5jCpp+NygJshNheyyyy7Nfr7Un6nq\nTF3lJvw9pxJ7wOQbM/xd6brrrvPq4cOHe3UxPfsHDRrk1Q899JBXh78TSNLJJ5/s1ZdeemnBx6kX\nBe9gds5Nc869kv59jqS3Ja0paR9JTSspoyT9sFKTROMhN4hBbhCD3KBUZAYxyA1ikBvEIDeIQW4Q\ng9ygXIq5g/k7ZtZLUn9JL0pa3TnXdEvux5JWX8I1QyUNjZ8iGh25QQxygxjkBqUiM4hBbhCD3CAG\nuUEMcoMY5AYtUfQmf2bWSdK9kk50zs3O/ZxL7i/Pe9+6c26Ec24L59wWLZopGhK5QQxygxjkBqUi\nM4hBbhCD3CAGuUEMcoMY5AYtVdQdzGa2tJKg3eqcuy89/ImZdXPOTTOzbpI+rdQkK23nnXf26lNO\nOaXkMcJ+vU899VSL5tQaNFJuwtdPKr3ncjE93OjjXFgj5ebWW2/NHPvyyy+9+pxzzvHqCRMmePVb\nb72VGePDDz/06rAv7sKFC7160aJFhSfbyjVSbmJcddVVXn300UcXvKbQ3gA33HBDyyfWwGqZmbCX\n8U033eTVRxxxRMExwn0dwn51Y8aM8epZs2ZlxpgzZ07BxylVuGfFSiutVJN5VEprf69BZbS23Gy+\n+eZe3b17d6/+4osvMtd06NDBqzt37uzVRx11lFcfe+yxmTHCHu+FhP2WJen73/++V//3v/8tacxq\nam25iWFmXr3tttt69Z///OdqTuc7Q4YM8eo11ljDq++9995qTsdDbsqja9euXr3hhhs2e/4LL7xQ\nyelUXD3lZuTIkV69/fbbl/0xxo4dmzkW/t4e9kuOEX6PCX/Gf/DBBzPX7Lvvvl595ZVXevW3337b\n4nlVSsE7mC15V79J0tvOudxdWEZLanpnHSLpgfJPD42K3CAGuUEMcoNSkRnEIDeIQW4Qg9wgBrlB\nDHKDcinmDubtJB0s6Q0zezU9drqk30u6y8yOkDRF0k8rM0U0KHKDGOQGMcgNSkVmEIPcIAa5QQxy\ngxjkBjHIDcqi4AKzc+4ZSbaET+9S3umgtSA3iEFuEIPcoFRkBjHIDWKQG8QgN4hBbhCD3KBcLOnV\nXaUHM6vegzXj0EMP9eozzjjDq9ddd92CYzz33HNeHfZJ+eyzz+Im16Ccc0t6Q2qxauSmmv8OcoV9\nm8N+4K1do+cGtdEWchO+F/z973/36mWXXTZzzcyZM716n3328epnn322TLNrTJXKTb1kBhUxtlIb\n1jRybnr27OnVkyZNavb88L1Iyr6ntSat8XvUkUce6dXXX3+9V48fP96r//nPf2bGCH+/Gjx4cJlm\nt1jYcznstyzVb8/l1pibUnXs2DFz7OWXX/bqsJf3pptu6tXl6Om/9tprZ45ddtllXr3HHnt4ddjP\n9Qc/+EFmjErsN0BuKme99dbz6nAfndCOO+6YOfbMM8+UdU7lUu+56dKli1eHv8NstNFGBcf46KOP\nvPr//u//vPrJJ5/MXDN9+vRip1g2u+++e+ZY2M87fN+rxTyl4nJTsAczAAAAAAAAAAD5sMAMAAAA\nAAAAAIjCAjMAAAAAAAAAIAoLzAAAAAAAAACAKO1rPYFaOPjgg726mE39Fi1a5NXnnXeeV7e1Tf1a\nm3CzPUnaaaedKv44bW1TPwD5bb311l49YsQIrw439cu3SdHll1/u1W19Uz8AQHnceOONXn3++ed7\ndZ8+fZqtY+TbgHvy5Mlefd1113n16NGjvfqdd95p8TxQPV9++WXm2PHHH+/VjzzyiFePGTPGq8MM\n5DNt2jSvDtcChg4dmrmme/fuXh1m66CDDvLqSmzoh/oS5vXjjz+u0Uxan9mzZ3v1Xnvt5dXLL798\nwTFmzJjh1fX6+jz88MO1nkJZcQczAAAAAAAAACAKC8wAAAAAAAAAgCgsMAMAAAAAAAAAorTJHswv\nvviiVxfTa/enP/2pVz/66KPlnBJqLF8v5HPPPbfF44Y9l/P1egaAU045xavDfoBvvfWWV++6666Z\nMT755JPyTwwAAlOmTPHqdu3a1WgmqJWLL77Yq7fddluv3m+//QqOMXz4cK8O++JOnTo1c82oUaOK\nnSJaiccee8yrwz0qjjrqKK/ebrvtCo5pZl4d9vsOf+aSpJtuusmrr7/+eq8O84vW74svvvDqiRMn\n1mgmrV/Yfx/1izuYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEMXCnkMVfTCz6j0Yqso5\nZ4XPikNuWi9ygxitMTdhj8Fwb4A+ffp4NX3eSlep3PBe06qNdc5tUYmByU3r1Rq/R6HyyA1ikJvK\nWW+99bx6woQJXv388897dTH9v+sFuUGMYnLDHcwAAAAAAAAAgCgsMAMAAAAAAAAAorDADAAAAAAA\nAACI0r7WEwAAoK3bZZddaj0FAAAAAJLmzp3r1TNmzPDqe+65p5rTARoCdzADAAAAAAAAAKKwwAwA\nAAAAAAAAiMICMwAAAAAAAAAgCgvMAAAAAAAAAIAobPIHAAAAAAAASPrkk0+8epVVVqnRTIDGwR3M\nAAAAAAAAAIAoLDADAAAAAAAAAKKwwAwAAAAAAAAAiFLtHsyfSZoiaZX07/WuUeYp1XauPSs8Prmp\njFrPk9z4mGdxyI2PeRankrlpyoxU++dZLOZZnGrkptbPsRSNMld+Jq4fjTJPidzUk0aZp0Ru6gnz\nLA658THP4hSVG3POVXoi2Qc1e9k5t0XVH7hEjTJPqbHmGqtRniPzrC+N8jyZZ31plOfJPOtLozxP\n5lk/Guk5NspcG2WeLdEoz7FR5ik11lxjNcpzbJR5So0111iN8hyZZ31plOfJPMuLFhkAAAAAAAAA\ngCgsMAMAAAAAAAAAotRqgXlEjR63VI0yT6mx5hqrUZ4j86wvjfI8mWd9aZTnyTzrS6M8T+ZZPxrp\nOTbKXBtlni3RKM+xUeYpNdZcYzXKc2yUeUqNNddYjfIcmWd9aZTnyTzLqCY9mAEAAAAAAAAAjY8W\nGQAAAAAAAACAKFVdYDazQWb2jplNNLNTq/nYhZjZzWb2qZm9mXNsJTN7xMwmpH+uWMs5pnPqYWaP\nm9lbZjbOzH5Vr3MtF3LTcuSG3MQgN+QmYo5tLjNS/eamETKTzonckJuSkRtyE4PckJsYbTE39ZoZ\nidzUM3LTco2cm6otMJtZO0nXSNpDUl9JPzOzvtV6/CKMlDQoOHaqpMecc70lPZbWtbZA0knOub6S\ntpZ0XPp1rMe5thi5KRtyQ25ikBtyU6o2lRmp7nMzUvWfGYnckJs45IbcxCA35CZGm8pNnWdGIjd1\nidyUTePmxjlXlQ9J20j6Z059mqTTqvX4Rc6xl6Q3c+p3JHVL/95N0ju1nmOeOT8gabdGmCu5qZ8P\nclP7D3JTfx/khsy0xtw0WmbITe3nRm7q94PckBtyUx8frT039Z4ZclOfH+SG3FSzRcaakqbm1B+k\nx+rZ6s65aenfP5a0ei0nEzKzXpL6S3pRdT7XFiA3ZUZu6lZdvxbkpm7V7WvRRjIjNV5u6vq1IDd1\nq65fC3JTt+r6tSA3dauuX4s2kptGy4xU568Fualbdf1aNFpu2OSvSC75zwSu1vNoYmadJN0r6UTn\n3Ozcz9XbXNuyenstyE1jqLfXgtw0hnp6LchMY6i314LcNIZ6ey3ITWOot9eC3DSGenstyE1jqLfX\ngtw0hnp7LRoxN9VcYP5QUo+ceq30WD37xMy6SVL656c1no8kycyWVhK0W51z96WH63KuZUBuyoTc\nkJsY5IbclKqNZUZqvNzU5WtBbshNDHJDbmKQG3ITo43lptEyI9Xpa0FuyE2MRs1NNReYX5LU28zW\nMbNlJB0gaXQVHz/GaElD0r8PUdL7pKbMzCTdJOlt59xlOZ+qu7mWCbkpA3JDbmKQG3JTqjaYGanx\nclN3rwW5ITcxyA25iUFuyE2MNpibRsuMVIevBbkhNzEaOjfVbPgsaU9J4yW9K+mMaj52EXO7XdI0\nSfOV9Io5QtLKSnZnnCDpUUkr1cE8t1dyK/zrkl5NP/asx7mSm/p5LcgNuSE35IbMtL3cNEJmyA25\nITfkhtyQG3JTfx/1mhlyU98f5KZt58bSJwAAAAAAAAAAQEnY5A8AAAAAAAAAEIUFZgAAAAAAAABA\nFBaYAQAAAAAAAABRWGAGAAAAAAAAAERhgRkAAAAAAAAAEIUFZgAAAAAAAABAFBaYAQAAAAAAAABR\nWGAGAAAAAAAAAET5/90VRowh7ElsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8277f9e890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.ioff()\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "# Helper plotting routine.\n",
    "def display_images(gens, title=\"\"):\n",
    "    fig, axs = plt.subplots(1, 10, figsize=(25, 3))\n",
    "    fig.suptitle(title, fontsize=14, fontweight='bold')\n",
    "    \n",
    "    for i in xrange(10):\n",
    "        reshaped_img = (gens[i].reshape(28, 28) * 255).astype(np.uint8)\n",
    "        axs.flat[i].imshow(reshaped_img)\n",
    "        #axs.flat[i].axis('off')        \n",
    "    return fig, axs\n",
    "\n",
    "\n",
    "batch_xs, batch_ys = mnist.train.next_batch(10)\n",
    "list_of_images = np.split(batch_xs, 10)\n",
    "_ = display_images(list_of_images, \"Some Examples from the Training Set.\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0 Current train cost and accuracy:  0.230138 0.45\n",
      "Step 100 Current train cost and accuracy:  0.226261 0.55\n",
      "Step 200 Current train cost and accuracy:  0.223619 0.75\n",
      "Step 300 Current train cost and accuracy:  0.21876 0.85\n",
      "Step 400 Current train cost and accuracy:  0.219502 0.55\n",
      "Step 500 Current train cost and accuracy:  0.214932 0.75\n",
      "Step 600 Current train cost and accuracy:  0.21139 0.8\n",
      "Step 700 Current train cost and accuracy:  0.206449 0.8\n",
      "Step 800 Current train cost and accuracy:  0.209066 0.6\n",
      "Step 900 Current train cost and accuracy:  0.204195 0.8\n",
      "Step 1000 Current train cost and accuracy:  0.201159 0.75\n",
      "Step 1100 Current train cost and accuracy:  0.203107 0.7\n",
      "Step 1200 Current train cost and accuracy:  0.196832 0.6\n",
      "Step 1300 Current train cost and accuracy:  0.199181 0.75\n",
      "Step 1400 Current train cost and accuracy:  0.197522 0.65\n",
      "Step 1500 Current train cost and accuracy:  0.187442 0.75\n",
      "Step 1600 Current train cost and accuracy:  0.197548 0.55\n",
      "Step 1700 Current train cost and accuracy:  0.185647 0.8\n",
      "Step 1800 Current train cost and accuracy:  0.186247 0.7\n",
      "Step 1900 Current train cost and accuracy:  0.16892 0.9\n",
      "Step 2000 Current train cost and accuracy:  0.178446 0.75\n",
      "Step 2100 Current train cost and accuracy:  0.174332 0.85\n",
      "Step 2200 Current train cost and accuracy:  0.184868 0.65\n",
      "Step 2300 Current train cost and accuracy:  0.17641 0.7\n",
      "Step 2400 Current train cost and accuracy:  0.169444 0.85\n",
      "Step 2500 Current train cost and accuracy:  0.166927 0.75\n",
      "Step 2600 Current train cost and accuracy:  0.173014 0.75\n",
      "Step 2700 Current train cost and accuracy:  0.161946 0.85\n",
      "Step 2800 Current train cost and accuracy:  0.168196 0.8\n",
      "Step 2900 Current train cost and accuracy:  0.177778 0.55\n",
      "Step 3000 Current train cost and accuracy:  0.174895 0.75\n",
      "Step 3100 Current train cost and accuracy:  0.150949 0.75\n",
      "Step 3200 Current train cost and accuracy:  0.155011 0.85\n",
      "Step 3300 Current train cost and accuracy:  0.170687 0.7\n",
      "Step 3400 Current train cost and accuracy:  0.162307 0.8\n",
      "Step 3500 Current train cost and accuracy:  0.155008 0.75\n",
      "Step 3600 Current train cost and accuracy:  0.136315 0.95\n",
      "Step 3700 Current train cost and accuracy:  0.131926 0.85\n",
      "Step 3800 Current train cost and accuracy:  0.155367 0.75\n",
      "Step 3900 Current train cost and accuracy:  0.143406 0.8\n",
      "Step 4000 Current train cost and accuracy:  0.14694 0.85\n",
      "Step 4100 Current train cost and accuracy:  0.168205 0.8\n",
      "Step 4200 Current train cost and accuracy:  0.135516 0.9\n",
      "Step 4300 Current train cost and accuracy:  0.15245 0.85\n",
      "Step 4400 Current train cost and accuracy:  0.137384 0.85\n",
      "Step 4500 Current train cost and accuracy:  0.140731 0.75\n",
      "Step 4600 Current train cost and accuracy:  0.131943 0.85\n",
      "Step 4700 Current train cost and accuracy:  0.138197 0.8\n",
      "Step 4800 Current train cost and accuracy:  0.134809 0.7\n",
      "Step 4900 Current train cost and accuracy:  0.136694 0.8\n",
      "Completed Training\n",
      "('Average train cost: ', 0.17355074667781592)\n",
      "('Average train accuracy: ', 0.74736000073105102)\n",
      "('Test cost: ', 0.13260417)\n",
      "('Test accuracy: ', 0.80610001)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, 784])  # A placeholder for the input images\n",
    "W = tf.Variable(tf.zeros([784, 10]))  # The weights of the linear layer \n",
    "b = tf.Variable(tf.zeros([10]))   # The bias of the linear layer\n",
    "\n",
    "init = tf.global_variables_initializer()   # Create an op that initializes these parameters\n",
    "\n",
    "# model\n",
    "Y=tf.nn.softmax(tf.matmul(X, W) + b)   # Compute the predictions of the model\n",
    "\n",
    "# placeholder for correct answers\n",
    "Y_ = tf.placeholder(tf.float32, [None, 10])  # Create a placeholder to hold the true labels\n",
    "\n",
    "# loss function\n",
    "cross_entropy = -tf.reduce_mean(Y_ * tf.log(Y))  # Define the loss\n",
    "\n",
    "# % of correct answers found in batch\n",
    "is_correct = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))   # Is the model's prediction correct?\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct,tf.float32))   # Compute the average accuracy\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(0.003)   # Create the optimizer\n",
    "train_step = optimizer.minimize(cross_entropy)   # Create an op that minimises the loss\n",
    "\n",
    "sess = tf.Session()    # Create a session\n",
    "sess.run(init)     # Initialize the variables\n",
    "\n",
    "avg_train_cost = 0. \n",
    "avg_train_accuracy = 0.\n",
    "avg_test_cost = 0.\n",
    "avg_test_accuracy = 0.\n",
    "\n",
    "num_steps = 5000   # How many training steps do we want?\n",
    "\n",
    "for i in range(num_steps):\n",
    "    # load batch of images and correct answers\n",
    "    batch_X, batch_Y = mnist.train.next_batch(20)    # Get a batch of 20 MNIST images and labels\n",
    "    train_data={X: batch_X, Y_: batch_Y}\n",
    "\n",
    "    # train\n",
    "    sess.run(train_step, feed_dict=train_data)   # Run the training step\n",
    "\n",
    "    a,c = sess.run([accuracy, cross_entropy], feed_dict=train_data)   # Get the accuracy and cost\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print \"Step\", i, \"Current train cost and accuracy: \", c, a\n",
    "    \n",
    "    avg_train_cost += c\n",
    "    avg_train_accuracy += a\n",
    "    \n",
    "print(\"Completed Training\")\n",
    "\n",
    "avg_train_cost /= num_steps\n",
    "avg_train_accuracy /= num_steps\n",
    "\n",
    "print(\"Average train cost: \", avg_train_cost)\n",
    "print(\"Average train accuracy: \", avg_train_accuracy)\n",
    "\n",
    "\n",
    "# How well did we do on the test data? \n",
    "test_data={X:mnist.test.images, Y_:mnist.test.labels}\n",
    "a,c = sess.run([accuracy, cross_entropy], feed_dict=test_data)\n",
    "\n",
    "print(\"Test cost: \", c)\n",
    "print(\"Test accuracy: \", a)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Y2:  (?, 7, 7, 4)\n",
      "Shape of Y3:  (?, 100)\n",
      "Step 0 Current train cost and accuracy:  0.227722 0.3\n",
      "Step 100 Current train cost and accuracy:  0.0679895 0.85\n",
      "Step 200 Current train cost and accuracy:  0.0356579 0.9\n",
      "Step 300 Current train cost and accuracy:  0.0394868 0.9\n",
      "Step 400 Current train cost and accuracy:  0.0317542 0.95\n",
      "Step 500 Current train cost and accuracy:  0.0239666 0.95\n",
      "Step 600 Current train cost and accuracy:  0.00910121 1.0\n",
      "Step 700 Current train cost and accuracy:  0.0575398 0.85\n",
      "Step 800 Current train cost and accuracy:  0.00515653 1.0\n",
      "Step 900 Current train cost and accuracy:  0.00470956 1.0\n",
      "Step 1000 Current train cost and accuracy:  0.0217068 0.9\n",
      "Step 1100 Current train cost and accuracy:  0.011616 0.95\n",
      "Step 1200 Current train cost and accuracy:  0.00175393 1.0\n",
      "Step 1300 Current train cost and accuracy:  0.00702303 0.95\n",
      "Step 1400 Current train cost and accuracy:  0.013462 0.95\n",
      "Step 1500 Current train cost and accuracy:  0.00506183 1.0\n",
      "Step 1600 Current train cost and accuracy:  0.00628346 1.0\n",
      "Step 1700 Current train cost and accuracy:  0.00219931 1.0\n",
      "Step 1800 Current train cost and accuracy:  0.00103071 1.0\n",
      "Step 1900 Current train cost and accuracy:  0.00472215 1.0\n",
      "Step 2000 Current train cost and accuracy:  0.00518102 1.0\n",
      "Step 2100 Current train cost and accuracy:  0.0290325 0.9\n",
      "Step 2200 Current train cost and accuracy:  0.0101519 0.95\n",
      "Step 2300 Current train cost and accuracy:  0.00731005 0.95\n",
      "Step 2400 Current train cost and accuracy:  0.000848766 1.0\n",
      "Step 2500 Current train cost and accuracy:  0.00379227 1.0\n",
      "Step 2600 Current train cost and accuracy:  0.00294856 1.0\n",
      "Step 2700 Current train cost and accuracy:  0.0070137 1.0\n",
      "Step 2800 Current train cost and accuracy:  0.00820268 1.0\n",
      "Step 2900 Current train cost and accuracy:  0.00547438 0.95\n",
      "Step 3000 Current train cost and accuracy:  0.0245766 0.9\n",
      "Step 3100 Current train cost and accuracy:  0.00235673 1.0\n",
      "Step 3200 Current train cost and accuracy:  0.000852428 1.0\n",
      "Step 3300 Current train cost and accuracy:  0.00206753 1.0\n",
      "Step 3400 Current train cost and accuracy:  0.000440723 1.0\n",
      "Step 3500 Current train cost and accuracy:  0.0007917 1.0\n",
      "Step 3600 Current train cost and accuracy:  0.00236768 1.0\n",
      "Step 3700 Current train cost and accuracy:  0.00128383 1.0\n",
      "Step 3800 Current train cost and accuracy:  0.00801757 0.95\n",
      "Step 3900 Current train cost and accuracy:  0.000370482 1.0\n",
      "Step 4000 Current train cost and accuracy:  0.00532245 0.95\n",
      "Step 4100 Current train cost and accuracy:  0.0328061 0.95\n",
      "Step 4200 Current train cost and accuracy:  0.00093702 1.0\n",
      "Step 4300 Current train cost and accuracy:  0.0100241 0.95\n",
      "Step 4400 Current train cost and accuracy:  0.000153046 1.0\n",
      "Step 4500 Current train cost and accuracy:  0.00704128 1.0\n",
      "Step 4600 Current train cost and accuracy:  0.00233283 1.0\n",
      "Step 4700 Current train cost and accuracy:  0.0187663 0.95\n",
      "Step 4800 Current train cost and accuracy:  0.00084757 1.0\n",
      "Step 4900 Current train cost and accuracy:  0.00203014 1.0\n",
      "Completed Training\n",
      "('Average train cost: ', 0.014395325041767501)\n",
      "('Average train accuracy: ', 0.95580999443680048)\n",
      "('Test cost: ', 0.010065879)\n",
      "('Test accuracy: ', 0.96759999)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, 784])   # A placeholder for the input image\n",
    "X_reshaped = tf.reshape(X, [-1, 28, 28, 1])   # TensorFlow's convolutional operation wants a \"volume\" \n",
    "\n",
    "layer_1_depth = 2   # How deep is layer 1?\n",
    "layer_2_depth = 4   # How deep is layer 2? \n",
    "filter_size = 5     # What size of filters do we want?\n",
    "\n",
    "# Create the parameters for layer 1\n",
    "W1 = tf.Variable(tf.truncated_normal([filter_size, filter_size, 1, layer_1_depth] ,stddev=0.1))\n",
    "B1 = tf.Variable(tf.ones([layer_1_depth])/10)\n",
    "\n",
    "# Create the parameters for layer 2\n",
    "W2 = tf.Variable(tf.truncated_normal([filter_size, filter_size, layer_1_depth, layer_2_depth] ,stddev=0.1))\n",
    "B2 = tf.Variable(tf.ones([layer_2_depth])/10)\n",
    "\n",
    "# CONVOLUTIONAL LAYER 1: \n",
    "Y1 = tf.nn.relu(tf.nn.conv2d(X_reshaped, W1, strides=[1, 1, 1, 1], padding='SAME') + B1)\n",
    "\n",
    "# POOLING \n",
    "pool = tf.nn.max_pool(Y1, ksize=[1, 2, 2, 1],\n",
    "                      strides=[1, 2, 2, 1], \n",
    "                      padding='SAME')\n",
    "\n",
    "# CONVOLUTIONAL LAYER 2\n",
    "Y2 = tf.nn.relu(tf.nn.conv2d(pool, W2, strides=[1, 2, 2, 1], padding='SAME') + B2)\n",
    "\n",
    "print \"Shape of Y2: \", Y2.get_shape()\n",
    "\n",
    "fully_connected_size = 100\n",
    "\n",
    "# Create the parameters for the hidden fully connected layer\n",
    "W3 = tf.Variable(tf.truncated_normal([7*7*layer_2_depth, fully_connected_size] ,stddev=0.1))\n",
    "B3 = tf.Variable(tf.ones([fully_connected_size])/10)\n",
    "\n",
    "# HIDDEN FULLY CONNECTED LAYER:\n",
    "YY = tf.contrib.layers.flatten(Y2)  # First we need to flatten the volume\n",
    "Y3 = tf.nn.relu(tf.matmul(YY, W3) + B3)  # Now compute the hidden output\n",
    "\n",
    "print \"Shape of Y3: \", Y3.get_shape()\n",
    "\n",
    "# Create the parameters of the final logits layer\n",
    "W4 = tf.Variable(tf.zeros([fully_connected_size, 10]))  \n",
    "B4 = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "# Compute the model predictions! \n",
    "logits = tf.matmul(Y3, W4) + B4   # Compute the logits\n",
    "Y = tf.nn.softmax(logits)     # Compute the model predictions\n",
    "\n",
    "# placeholder for correct answers\n",
    "Y_ = tf.placeholder(tf.float32, [None, 10])   \n",
    "\n",
    "# loss function\n",
    "cross_entropy = -tf.reduce_mean(Y_ * tf.log(Y))\n",
    "\n",
    "# % of correct answers found in batch\n",
    "is_correct = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct,tf.float32))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(0.003)   # Now we try an Adam optimizer\n",
    "train_step = optimizer.minimize(cross_entropy)   # An op to minimise the cross entropy loss\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "avg_train_cost = 0.\n",
    "avg_train_accuracy = 0.\n",
    "avg_test_cost = 0.\n",
    "avg_test_accuracy = 0.\n",
    "\n",
    "num_steps = 5000\n",
    "\n",
    "for i in range(num_steps):\n",
    "    # load batch of images and correct answers\n",
    "    batch_X, batch_Y = mnist.train.next_batch(20)\n",
    "    train_data={X: batch_X, Y_: batch_Y}\n",
    "\n",
    "    # train\n",
    "    sess.run(train_step, feed_dict=train_data)\n",
    "\n",
    "    # success ? add code to print it\n",
    "    a,c = sess.run([accuracy, cross_entropy], feed_dict=train_data)\n",
    "    \n",
    "    if i % 100 == 0:\n",
    "        print \"Step\", i, \"Current train cost and accuracy: \", c, a\n",
    "    \n",
    "    avg_train_cost += c\n",
    "    avg_train_accuracy += a\n",
    "    \n",
    "print(\"Completed Training\")\n",
    "\n",
    "avg_train_cost /= num_steps\n",
    "avg_train_accuracy /= num_steps\n",
    "\n",
    "print(\"Average train cost: \", avg_train_cost)\n",
    "print(\"Average train accuracy: \", avg_train_accuracy)\n",
    "\n",
    "\n",
    "# success on test data ?\n",
    "test_data={X:mnist.test.images, Y_:mnist.test.labels}\n",
    "a,c = sess.run([accuracy, cross_entropy], feed_dict=test_data)\n",
    "\n",
    "print(\"Test cost: \", c)\n",
    "print(\"Test accuracy: \", a)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}