VGG_FINETUNE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package loaded.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc\n",
    "import scipy.io\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "%matplotlib inline\n",
    "print(\"Package loaded.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>LOAD DATA</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['trainlabel', 'trainimg', 'testimg', 'testlabel']\n"
     ]
    }
   ],
   "source": [
    "cwd = os.getcwd()\n",
    "loadpath = cwd+\"/data/data4vgg.npz\"\n",
    "l = np.load(loadpath)\n",
    "\n",
    "#show files\n",
    "print(l.files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Parse Data</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 train images loaded\n",
      "18 test images loaded\n",
      "37632 dimensional input\n",
      "2 classes\n",
      "shape of 'trainimg' is (69, 37632)\n",
      "shape of 'testimg' is (18, 37632)\n"
     ]
    }
   ],
   "source": [
    "trainimg = l['trainimg']\n",
    "trainlabel = l['trainlabel']\n",
    "testimg = l['testimg']\n",
    "testlabel = l['testlabel']\n",
    "\n",
    "ntrain = trainimg.shape[0]\n",
    "nclass= trainlabel.shape[1]\n",
    "dim = trainimg.shape[1]\n",
    "\n",
    "ntest = testimg.shape[0]\n",
    "\n",
    "print(\"%d train images loaded\" %(ntrain))\n",
    "print(\"%d test images loaded\"%(ntest))\n",
    "print(\"%d dimensional input\" %(dim))\n",
    "print(\"%d classes\" %(nclass))\n",
    "print(\"shape of 'trainimg' is %s\" %(trainimg.shape,))\n",
    "print(\"shape of 'testimg' is %s\"%(testimg.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> GENERATE TENSORS FOR TRAINING AND TESTING </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of trainimg_tensor is (69, 112, 112, 3)\n",
      "shape of testimg_tensor is (18, 112, 112, 3)\n"
     ]
    }
   ],
   "source": [
    "trainimg_tensor = np.ndarray( (ntrain,112,112,3) )\n",
    "\n",
    "for i in range(ntrain):\n",
    "    currimg = trainimg[i,:]\n",
    "    currimg = np.reshape(currimg, [112,112,3] )\n",
    "    trainimg_tensor[i,:,:,:] = currimg\n",
    "print(\"shape of trainimg_tensor is %s\"%(trainimg_tensor.shape,))\n",
    "\n",
    "\n",
    "testimg_tensor = np.ndarray((ntest, 112,112,3))\n",
    "for i in range(ntest):\n",
    "    currimg = testimg[i,:]\n",
    "    currimg = np.reshape(currimg,[112,112,3]) \n",
    "    testimg_tensor[i,:,:,:] = currimg\n",
    "print(\"shape of testimg_tensor is %s\"%(testimg_tensor.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>DEFINE A FUNCTION FOR USING PRETRAINED VGG NETWORK</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def net(data_path, input_image):\n",
    "    layers=(\n",
    "        'conv1_1','relu1_1','conv1_2','relu1_2','pool1',\n",
    "        'conv2_1','relu2_1','conv2_2','relu2_2','pool2',\n",
    "        'conv3_1','relu3_1','conv3_2','relu3_2','conv3_3',\n",
    "        'relu3_3','conv3_4','relu3_4','pool3',\n",
    "        'conv4_1','relu4_1','conv4_2','relu4_2','conv4_3',\n",
    "        'relu4_3','conv4_4','relu4_4','pool4',\n",
    "        'conv5_1','relu5_1','conv5_2','relu5_2','conv5_3',\n",
    "        'relu5_3','conv5_4','relu5_4'\n",
    "    \n",
    "    )\n",
    "    data = scipy.io.loadmat(data_path)\n",
    "    mean = data['normalization'][0][0][0]\n",
    "    mean_pixel = np.mean(mean,axis=(0,1))\n",
    "    weights = data['layers'][0]\n",
    "    net = {}\n",
    "    current = input_image\n",
    "    \n",
    "    for i, name in enumerate(layers):\n",
    "        kind = name[:4]\n",
    "        if kind == 'conv':\n",
    "            kernels, bias = weights[i][0][0][0][0]\n",
    "            \n",
    "            #matconvnet : weights are [width, height, in_channels, out_channels]\n",
    "            #tensorflow : weights are [height,width, in_channels, out_channels]\n",
    "            \n",
    "            kernels = np.transpose(kernels,(1,0,2,3))\n",
    "            bias = bias.reshape(-1)\n",
    "            current = _conv_layer(current,kernels, bias)\n",
    "        elif kind =='relu':\n",
    "            current = tf.nn.relu(current)\n",
    "        elif kind =='pool':\n",
    "            current = _pool_layer(current)\n",
    "        net[name] = current\n",
    "        \n",
    "    assert len(net) == len(layers)\n",
    "    return net, mean_pixel\n",
    "\n",
    "\n",
    "def _conv_layer(input,weights,bias):\n",
    "    conv = tf.nn.conv2d(input,tf.constant(weights), strides=(1,1,1,1), padding='SAME')\n",
    "    return tf.nn.bias_add(conv,bias)\n",
    "\n",
    "def _pool_layer(input):\n",
    "    return tf.nn.max_pool(input, ksize=(1,2,2,1), strides=(1,2,2,1), padding='SAME')\n",
    "\n",
    "def preprocess(image, mean_pixel):\n",
    "    return image-mean_pixel\n",
    "\n",
    "def unprocess(image,mean_pixel):\n",
    "    return image+mean_pixel\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> EXTRACT FEATURES FROM THE VGG NETWORK </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TYPE OF 'train_features' IS <class 'numpy.ndarray'> \n",
      "SHAPE OF 'train_features' IS (69, 7, 7, 512)\n",
      "TYPE OF 'test_features' IS <class 'numpy.ndarray'> \n",
      "SHAPE OF 'test_features' IS (18, 7, 7, 512)\n",
      "PREPROCESSING DONE\n"
     ]
    }
   ],
   "source": [
    "VGG_PATH = cwd+'/data/imagenet-vgg-verydeep-19.mat'\n",
    "\n",
    "with tf.Graph().as_default(),  tf.Session() as sess:\n",
    "    with tf.device('/gpu:0'):\n",
    "        img_placeholder = tf.placeholder(tf.float32, shape=(None,112,112,3))\n",
    "        net_val, mean_pixel = net(VGG_PATH, img_placeholder)\n",
    "        \n",
    "        train_features = net_val['relu5_4'].eval(feed_dict={img_placeholder: trainimg_tensor})\n",
    "        test_features = net_val['relu5_4'].eval(feed_dict={img_placeholder: testimg_tensor})\n",
    "        \n",
    "print(\"TYPE OF 'train_features' IS %s \" %(type(train_features)))\n",
    "print(\"SHAPE OF 'train_features' IS %s\" % (train_features.shape,))\n",
    "print(\"TYPE OF 'test_features' IS %s \"%(type(test_features)))\n",
    "print(\"SHAPE OF 'test_features' IS %s\"%(test_features.shape,))\n",
    "print(\"PREPROCESSING DONE\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>VECTORIZE CNN FEATURES</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SHAPE OF 'train_vectorized' IS (69, 25088)\n",
      "SHAPE OF 'test_vectorized' IS (18, 25088)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC8RJREFUeJzt3d+PHXd9xvHnye4aG9v5gRJSKzbEQiUSQgXDymoUFLWh\nQWmJaC96QSRQiyr5pkRBVEXQm4p/IKUXFdLKCU1FIEIJkSpUUhkSFCKBYZ0YcGy3QmloHJFuUit1\n7Pzwj316sZPKtazsrH2+M8d83i/J8u76+Hy+G+e9c86cMzNOIgC1XDb2AgAMj/CBgggfKIjwgYII\nHyiI8IGCpip827fZ/jfbv7T9xYFn32t7yfaBIeeeNX+b7cdsH7T9tO27Bp6/3vZPbP+sm//lIed3\na5ix/ZTt7ww9u5v/rO1f2N5ve3Hg2VfaftD2YduHbN/YdN60vI5ve0bSv0u6VdIRST+VdEeSgwPN\nv1nScUn/lOT9Q8w8Z/4WSVuSPGl7s6R9kv5kwO/fkjYmOW57TtITku5K8uMh5ndr+LykeUmXJ7l9\nqLlnzX9W0nySl0aYfZ+kHybZbXudpLcnebnVvGna4u+U9MskzyQ5KekBSX881PAkj0s6OtS888z/\ndZInu49fkXRI0nUDzk+S492nc92vwbYKtrdK+rik3UPNnBa2r5B0s6R7JCnJyZbRS9MV/nWSnjvr\n8yMa8H/8aWL7ekk7JO0deO6M7f2SliTtSTLk/K9I+oKk5QFnniuSvmd7n+1dA87dLulFSV/rnurs\ntr2x5cBpCh+SbG+S9JCkzyU5NuTsJGeSfFDSVkk7bQ/ylMf27ZKWkuwbYt5b+Ej3/f+hpL/snv4N\nYVbShyR9NckOSSckNd3HNU3hPy9p21mfb+2+Vkb33PohSfcn+fZY6+geZj4m6baBRt4k6RPdc+wH\nJN1i++sDzf4/SZ7vfl+S9LBWnn4O4YikI2c9wnpQKz8Impmm8H8q6bdtb+92bnxS0j+PvKbBdDvX\n7pF0KMndI8y/xvaV3ccbtLKT9fAQs5N8KcnWJNdr5d/90SSfGmL2m2xv7HaqqnuY/TFJg7zCk+QF\nSc/ZvqH70kclNd2pO9vyztciyWnbn5X0r5JmJN2b5Omh5tv+pqTfk3S17SOS/jbJPUPN18pW79OS\nftE9z5akv0nyLwPN3yLpvu7VlcskfSvJKC+rjeRaSQ+v/PzVrKRvJHlkwPl3Srq/2+g9I+kzLYdN\nzct5AIYzTQ/1AQyE8IGCCB8oiPCBgggfKGgqwx/47ZJTM5v5zB9q/lSGL2nM//ij/sMzn/lDDJnW\n8AE01OQNPHPrNmb9+qsu+O+fOnVCc3MXfnCSX3n1wmfrDc3pbRf89yXJcxf+hsiTy69p3WUbLmp+\nTp2+4L87ie//YjD/4ua/rhM6mTe82u2avGV3/fqrNL/zsy3uupfZR8c9yGv26mtHnX/6hf8adT7G\nszff73U7HuoDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlBQr/DHvJglgMlbNfzudMv/\noJWri7xP0h2239d6YQDa6bPFH/VilgAmr0/4XMwS+A0zsZ17tnfZXrS9eOrUiUndLYAG+oTf62KW\nSRaSzCeZv5iTaABor0/4pS9mCfwmWvUMPGNfzBLA5PU69VZ3xdahrtoKoDHeuQcURPhAQYQPFET4\nQEGEDxRE+EBBhA8URPhAQYQPFET4QEFNrpa7PGe9+ltzLe66l8tHm7zixI53jTr/bd/larl4a2zx\ngYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKKjPZbLvtb1k+8AQCwLQXp8t\n/j9Kuq3xOgAMaNXwkzwu6egAawEwEJ7jAwVNLHzbu2wv2l48/fqJSd0tgAYmFn6ShSTzSeZn12+c\n1N0CaICH+kBBfV7O+6akH0m6wfYR23/RflkAWlr1ZJtJ7hhiIQCGw0N9oCDCBwoifKAgwgcKInyg\nIMIHCiJ8oCDCBwoifKAgwgcKWvUtuxdieVZ69ZrxfqZs+IMPjzZbkpY+PDfq/Hc/Pu7RkcsnOCx7\n2rHFBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oKA+F9TYZvsx2wdtP237\nriEWBqCdPkfnnZb0V0metL1Z0j7be5IcbLw2AI2susVP8uskT3YfvyLpkKTrWi8MQDtreo5v+3pJ\nOyTtbbEYAMPoHb7tTZIekvS5JMfO8+e7bC/aXjzzGidiAKZZr/Btz2kl+vuTfPt8t0mykGQ+yfzM\nhnHPAAPgrfXZq29J90g6lOTu9ksC0FqfLf5Nkj4t6Rbb+7tff9R4XQAaWvXlvCRPSPIAawEwEN65\nBxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQX3OwLNmyxuiY79zssVd9zL7+rrRZkvS\nqc0Zdb62bxt3/oHD487HqtjiAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+\nUFCfK+mst/0T2z+z/bTtLw+xMADt9Dk67w1JtyQ53l1D7wnb303y48ZrA9BInyvpRNLx7tO57tfI\nx50CuBh9r5Y7Y3u/pCVJe5LsbbssAC31Cj/JmSQflLRV0k7b7z/3NrZ32V60vXjm+IlJrxPABK1p\nr36SlyU9Jum28/zZQpL5JPMzmzZOan0AGuizV/8a21d2H2+QdKskzq0EXML67NXfIuk+2zNa+UHx\nrSTfabssAC312av/c0k7BlgLgIHwzj2gIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwrq\n8179Nbt20zF9/nf3tLjrXu7OraPNlqT3bn9h1Pn/+fK7Rp3/xp/dOOr89/z1j0adfylgiw8URPhA\nQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBvcPvLpz5lG0upgFc4tayxb9L0qFW\nCwEwnL6Xyd4q6eOSdrddDoAh9N3if0XSFyQtN1wLgIH0uVru7ZKWkuxb5Xa7bC/aXjx+9NTEFghg\n8vps8W+S9Anbz0p6QNIttr9+7o2SLCSZTzK/6R1zE14mgElaNfwkX0qyNcn1kj4p6dEkn2q+MgDN\n8Do+UNCaTraZ5AeSftBkJQAGwxYfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGC1vSW\n3b7eOXNSd171qxZ33cvfzWa02ZL0nstfGnX+f2zeNur8M5vPjDofq2OLDxRE+EBBhA8URPhAQYQP\nFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEG9DtLpLp/1iqQzkk4nmW+5KABtreXovN9PMu5hZwAm\ngof6QEF9w4+k79neZ3vX+W5w9mWyX/xvjscGplnfh/ofSfK87XdK2mP7cJLHz75BkgVJC5I0/4H1\n454JA8Bb6rXFT/J89/uSpIcl7Wy5KABtrRq+7Y22N7/5saSPSTrQemEA2unzUP9aSQ/bfvP230jy\nSNNVAWhq1fCTPCPpAwOsBcBAeDkPKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGC1nIG\nnt6OLUvff22mxV33kjMebbYkHTi6ZdT5pzcvjzr/stfYnkw7/oWAgggfKIjwgYIIHyiI8IGCCB8o\niPCBgggfKIjwgYIIHyiI8IGCeoVv+0rbD9o+bPuQ7RtbLwxAO30P0vl7SY8k+VPb6yS9veGaADS2\navi2r5B0s6Q/l6QkJyWdbLssAC31eai/XdKLkr5m+ynbu7tr6AG4RPUJf1bShyR9NckOSSckffHc\nG9neZXvR9uL/HD0z4WUCmKQ+4R+RdCTJ3u7zB7Xyg+D/SbKQZD7J/BXvGO8kHABWt2r4SV6Q9Jzt\nG7ovfVTSwaarAtBU3736d0q6v9uj/4ykz7RbEoDWeoWfZL+k+cZrATAQ3rkHFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEGEDxRE+EBBhA8U5CSTv1P7RUm/uoi7uFrSSxNazqU0m/nMv9j5705yzWo3\nahL+xbK9mGSUg4LGnM185g81n4f6QEGEDxQ0reEvFJ3NfOYPMn8qn+MDaGtat/gAGiJ8oCDCBwoi\nfKAgwgcK+l9dXAVEDRzQ6AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f83181ef320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#VGG 끝 레이어를 통과한것\n",
    "train_vectorized = np.ndarray((ntrain, 7*7*512))\n",
    "test_vectorized = np.ndarray((ntest, 7*7*512))\n",
    "\n",
    "for i in range(ntrain):\n",
    "    curr_feat = train_features[i,:,:,:]\n",
    "    curr_feat_vec = np.reshape(curr_feat,(1,-1))\n",
    "    train_vectorized[i,:] = curr_feat_vec\n",
    "    \n",
    "for i in range(ntest):\n",
    "    curr_feat = test_features[i,:,:,:]\n",
    "    curr_feat_vec = np.reshape(curr_feat,(1,-1))\n",
    "    test_vectorized[i, :] = curr_feat_vec\n",
    "    \n",
    "print(\"SHAPE OF 'train_vectorized' IS %s\" %(train_vectorized.shape,))\n",
    "print(\"SHAPE OF 'test_vectorized' IS %s\" %(test_vectorized.shape,))\n",
    "\n",
    "\n",
    "#TrainingImage가 relu5_3까지 전부 통과했을때의 모습\n",
    "\n",
    "currimg = np.reshape(train_vectorized[0], [7,7,512] )\n",
    "\n",
    "plt.matshow(currimg[:,:,0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> DEFINE NETWORKS AND FUNCTIONS(ADD 2LAYER MLP)</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready to go!!\n"
     ]
    }
   ],
   "source": [
    "#Parameters\n",
    "learning_rate = 0.0001\n",
    "training_epochs = 100\n",
    "batch_size = 100\n",
    "display_step = 10\n",
    "\n",
    "#Network\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    n_input = dim\n",
    "    n_output = nclass\n",
    "    \n",
    "    weights ={\n",
    "        'wd1': tf.Variable(tf.random_normal([7*7*512,1024], stddev=0.1)),\n",
    "        'wd2': tf.Variable(tf.random_normal([1024, n_output], stddev=0.1))\n",
    "    }\n",
    "    \n",
    "    biases = {\n",
    "        'bd1': tf.Variable(tf.random_normal([1024], stddev=0.1)),\n",
    "        'bd2': tf.Variable(tf.random_normal([n_output], stddev=0.1))\n",
    "    }\n",
    "    \n",
    "    def conv_basic(_input, _w, _b, _keepratio):\n",
    "        #input\n",
    "        _input_r = _input\n",
    "        \n",
    "        #vectorize\n",
    "        _dense1 = tf.reshape(_input_r, [-1,_w['wd1'].get_shape().as_list()[0]])\n",
    "        \n",
    "        #Fc1\n",
    "        _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1']))\n",
    "        _fc_dr1 = tf.nn.dropout(_fc1,_keepratio)\n",
    "        #Fc2\n",
    "        _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2'])\n",
    "        \n",
    "        #Return everything\n",
    "        out = {'input_r': _input_r, 'dense1':_dense1, 'fc1':_fc1, 'fc_dr1': _fc_dr1, 'out':_out}\n",
    "        \n",
    "        return out\n",
    "    \n",
    "#tf graph input\n",
    "x = tf.placeholder(tf.float32, [None, 7*7*512])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "keepratio = tf.placeholder(tf.float32)\n",
    "\n",
    "\n",
    "#Functions!!\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    _pred = conv_basic(x, weights, biases, keepratio)['out']\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=_pred,labels=y))\n",
    "    optm = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(cost)\n",
    "    _corr = tf.equal(tf.argmax(_pred,1), tf.argmax(y,1))\n",
    "    accr = tf.reduce_mean(tf.cast(_corr, tf.float32))\n",
    "    init = tf.global_variables_initializer()\n",
    "    \n",
    "print(\"Network Ready to go!!\")\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> CNN Finetuning</h1>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/100 cost: 5.969704628\n",
      "Training accuracy: 0.760\n",
      "Test accuracy: 0.444\n",
      "Epoch: 010/100 cost: 0.134050012\n",
      "Training accuracy: 0.960\n",
      "Test accuracy: 0.889\n",
      "Epoch: 020/100 cost: 0.000003248\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 030/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 040/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 050/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 060/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.833\n",
      "Epoch: 070/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 080/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Epoch: 090/100 cost: 0.000000000\n",
      "Training accuracy: 1.000\n",
      "Test accuracy: 0.944\n",
      "Optimization Finished!\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "#Training cycle\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    num_batch = int(ntrain/batch_size)+1\n",
    "    \n",
    "    #Loop over all batches\n",
    "    for i in range(num_batch):\n",
    "        randidx = np.random.randint(ntrain,size=batch_size)\n",
    "        batch_xs = train_vectorized[randidx, :]\n",
    "        batch_ys = trainlabel[randidx, :]\n",
    "        \n",
    "        #Fit training using batch data\n",
    "        sess.run(optm, feed_dict={x:batch_xs, y:batch_ys,keepratio:0.7})\n",
    "        \n",
    "        #Compute average loss\n",
    "        avg_cost += sess.run(cost, feed_dict={x:batch_xs, y:batch_ys, keepratio:1.})/num_batch\n",
    "        \n",
    "    #Display logs per epoch step\n",
    "    if epoch % display_step == 0:\n",
    "        print(\"Epoch: %03d/%03d cost: %.9f\" %(epoch, training_epochs, avg_cost))\n",
    "        train_acc= sess.run(accr, feed_dict={x:batch_xs, y:batch_ys, keepratio:1.})\n",
    "        print(\"Training accuracy: %.3f\" %(train_acc))\n",
    "        \n",
    "        test_acc = sess.run(accr, feed_dict={x:test_vectorized, y:testlabel, keepratio:1.})\n",
    "        print(\"Test accuracy: %.3f\" %(test_acc))\n",
    "        \n",
    "print(\"Optimization Finished!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}