tensorflow-autoencoder-3-layer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# all values between 0 and 1\n",
    "file = r'carbon_nanotubes.csv'\n",
    "df = pd.read_csv(file, delimiter=';', header=None, decimal=',', skiprows=1)\n",
    "x_train = df.drop([0,1], axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data shape:  (10721, 6)\n",
      "No of training examples:  10721\n",
      "No of features:  6\n"
     ]
    }
   ],
   "source": [
    "print('data shape: ', x_train.shape)\n",
    "m = x_train.shape[0]\n",
    "n = x_train.shape[1]\n",
    "print('No of training examples: ', m)\n",
    "print('No of features: ', n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_mini_batch(X, Y, mini_batch_size = None):\n",
    "    '''Created minibatches for feeding into NN. First the dataset if randomized and then mini batches are\n",
    "    made from the randomized set.\n",
    "    \n",
    "    Arguments:\n",
    "    X -- array, shape(m, n)\n",
    "    Y -- array, shape depends on how many classes. If it is binary then it is a vector.\n",
    "    \n",
    "    Return:\n",
    "    mini_batches -- list of tuples which contain arrays.'''\n",
    "    \n",
    "    if mini_batch_size==None:\n",
    "        return print('Need to specify a mini_batch_size.')\n",
    "    else: \n",
    "        m = X.shape[0]\n",
    "        \n",
    "        mini_batches = []\n",
    "         \n",
    "        # shuffle (X, Y)\n",
    "        permutation = list(np.random.permutation(m))\n",
    "        shuffled_X = X[permutation, :]\n",
    "        shuffled_Y = Y[permutation, :]#.reshape(m,1) if this was a label vector of shape(m, 1)\n",
    "        \n",
    "        # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.\n",
    "        num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning\n",
    "        for k in range(0, num_complete_minibatches):\n",
    "        \n",
    "            mini_batch_X = shuffled_X[k * mini_batch_size: (k + 1) * mini_batch_size, :]\n",
    "            mini_batch_Y = shuffled_Y[k * mini_batch_size: (k + 1) * mini_batch_size, :]\n",
    "        \n",
    "            mini_batch = (mini_batch_X, mini_batch_Y)\n",
    "            mini_batches.append(mini_batch)\n",
    "        \n",
    "        # Handling the end case (last mini-batch < mini_batch_size)\n",
    "        if m % mini_batch_size != 0:\n",
    "            mini_batch_X =shuffled_X[num_complete_minibatches *mini_batch_size ::, :]\n",
    "            mini_batch_Y =shuffled_Y[num_complete_minibatches *mini_batch_size ::, :]\n",
    "            mini_batch = (mini_batch_X, mini_batch_Y)\n",
    "            mini_batches.append(mini_batch)\n",
    "            \n",
    "        return mini_batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tf.reset_default_graph()\n",
    "layer1 = 5\n",
    "layer2 = 3\n",
    "layer3 = 5\n",
    "layer4 = n\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer() # This is a He initializtion\n",
    "\n",
    "weights = {'W1': tf.get_variable(name='W1', shape=[n, layer1], initializer=initializer),\n",
    "           'W2': tf.get_variable(name='W2', shape=[layer1, layer2], initializer=initializer),\n",
    "           'W3': tf.get_variable(name='W3', shape=[layer2, layer3], initializer=initializer),\n",
    "           'W4': tf.get_variable(name='W4', shape=[layer3, layer4], initializer=initializer)}\n",
    "\n",
    "biases =  {'b1': tf.zeros(shape=[1, layer1], name='b1'),\n",
    "           'b2': tf.zeros(shape=[1, layer2], name='b2'),\n",
    "           'b3': tf.zeros(shape=[1, layer3], name='b3'),\n",
    "           'b4': tf.zeros(shape=[1, layer4], name='b4')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def encoder(input_placeholder):\n",
    "    '''Create encoder which outputs the latent vector.\n",
    "    \n",
    "    Arguments:\n",
    "    input_placeholder -- tensor, shape[None, m]\n",
    "    \n",
    "    Return:\n",
    "    latent_vector -- tensor, activation of shape[None, layer2]'''\n",
    "    \n",
    "    # linear -> non-linear\n",
    "    Z1 = tf.add(tf.matmul(input_placeholder, weights['W1']), biases['b1']) # linear\n",
    "    A1 = tf.nn.relu(Z1) # non-linear\n",
    "    \n",
    "    Z2 = tf.add(tf.matmul(A1, weights['W2']), biases['b2'])\n",
    "    latent_vector = tf.nn.relu(Z2)\n",
    "    \n",
    "    return latent_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def decoder(latent_tensor):\n",
    "    '''Decode the encoder.\n",
    "    \n",
    "    Arguments:\n",
    "    latent_vec -- tensor, shape[None, layer2]\n",
    "    \n",
    "    Return:\n",
    "    output_layer -- tensor, shape[None, n]'''\n",
    "    Z3 = tf.add(tf.matmul(latent_tensor, weights['W3']), biases['b3'])\n",
    "    A3 = tf.nn.relu(Z3)\n",
    "    \n",
    "    # Use a sigmoid activation our original data is between 0,1\n",
    "    Z4 = tf.add(tf.matmul(A3, weights['W4']), biases['b4'])\n",
    "    output_layer = tf.nn.sigmoid(Z4)\n",
    "    return output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0 has loss of 0.08401456358562036\n",
      "epoch 100 has loss of 0.04163502062480133\n",
      "epoch 200 has loss of 0.03924129687878022\n",
      "epoch 300 has loss of 0.038914895515485\n",
      "epoch 400 has loss of 0.03876366524631717\n",
      "epoch 500 has loss of 0.03865462604416421\n",
      "epoch 600 has loss of 0.038514928555632204\n",
      "epoch 700 has loss of 0.03835406392274133\n",
      "epoch 800 has loss of 0.03814600080431226\n",
      "epoch 900 has loss of 0.03784526514270938\n",
      "epoch 1000 has loss of 0.03766180823994688\n",
      "epoch 1100 has loss of 0.03756432345204324\n",
      "epoch 1200 has loss of 0.0375547479972782\n",
      "epoch 1300 has loss of 0.037507826395063525\n",
      "epoch 1400 has loss of 0.0374849979956466\n",
      "epoch 1500 has loss of 0.03742947638124587\n",
      "epoch 1600 has loss of 0.0374205797147679\n",
      "epoch 1700 has loss of 0.03738746257970134\n",
      "epoch 1800 has loss of 0.037315670445740946\n",
      "epoch 1900 has loss of 0.03734371779733392\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.001\n",
    "epochs = 2000\n",
    "mini_batch_size = 128\n",
    "\n",
    "X = tf.placeholder(dtype=tf.float32, shape=[None, n], name='input')\n",
    "\n",
    "encoded_out = encoder(X)\n",
    "reconstruction = decoder(encoded_out)\n",
    "\n",
    "cost = tf.reduce_mean(tf.square(tf.subtract(X, reconstruction)))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "losses = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    \n",
    "    for epoch in range(epochs):  \n",
    "        epoch_loss = 0\n",
    "        \n",
    "        num_mini_batches = int(m/mini_batch_size)  # No. of mini_batches that have mini_batch size.\n",
    "        \n",
    "        minibatches = random_mini_batch(x_train, x_train, mini_batch_size=mini_batch_size)  # inibatches is a list of tuples.\n",
    "        \n",
    "        for minibatch in minibatches:\n",
    "            \n",
    "            (minibatch_X, _) = minibatch\n",
    "            \n",
    "            _, minibatch_loss = sess.run([optimizer, cost], feed_dict={X: minibatch_X})\n",
    "            \n",
    "            epoch_loss += minibatch_loss/num_mini_batches\n",
    "        \n",
    "        if epoch % 5 == 0:\n",
    "            losses.append(epoch_loss)\n",
    "        if epoch % 100 == 0:\n",
    "            print('epoch {} has loss of {}'.format(epoch, epoch_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+cXHV97/HX+5yZ2V/ZbBISMSQoqBEvWEWaIrZqtf4i\naInK41qsFqV9XIqFVm+vWmwfbW0ftre2tiqWQtGi4i/UKm1uS8VqlfoLJSAiyK8kooCBBMjP3ezO\nzszn/nHOJodlZ3Y22ZlZkvfz8ZhHZs7Pz57dzHu+5/s9ZxQRmJmZzSbpdQFmZvb44MAwM7O2ODDM\nzKwtDgwzM2uLA8PMzNriwDAzs7Y4MOywI+kNkr7c6zrMDjcODOsaSfdIemmn9xMRn4qIl3di2/nP\nsE/SXkkPSPqYpEVtrnucpJBU6kRts+z7JZLukDQm6WuSntxi2WWSrpY0Kuknkn693W1JenE+bZek\nezr4I1kPODDssNKlN+NfjYhFwMnAc4B3dWGfB03ScuCLwB8Dy4CNwGdbrHIJUAWOBt4AXCrppDa3\nNQpcAbxjfn8KWwgcGLYgSHqVpJsl7ZT0bUnPKsy7SNJmSXsk/UjSawrz3izpW5LeL+lh4N35tG8W\nlglJ50u6O9/+JZKUz0sl/a2khyT9WNKF7bYCIuIB4Fqy4Jja1yslfV/Sbkn3Snp3YZX/zv/dmbdQ\nnpev85uSbpe0Q9K1rT79H6TXArdFxOcjYhx4N/BsSc+YvqCkIeAs4I8jYm9EfBP4V+A32tlWRHwv\nIj4BbJnnn8EWAAeG9Zyk55B9Kv1t4CjgH4ENkvryRTYDLwBGgD8DPilpZWETzyV7gzoa+Ismu3kV\n8AvAs4DXAa/Ip/8vYB3Zm/4pwKvnUPfqfN1NhcmjwDnAEuCVwFskTW3zhfm/SyJiUUR8R9J64A/J\n3ohXAN8APtNinztbPC5qstpJwA+mXkTEaF7zSTMs+3SgFhF3Fab9oLDsXLZlhxkHhi0E5wH/GBHf\njYh6RHwcmABOA8g/zf4sIhoR8VngbuDUwvo/i4gPRUQtIvY12cdfRcTOiPgp8DUOtApeB3wwIu6L\niB3AX7VR779I2gPcC2wD/nRqRkR8PSJ+mNd6C9mb/y+32Nb5wP+NiNsjogb8JXBys1ZGRCxp8WhW\n+yJg17Rpu4HhJsvubrHsXLZlhxkHhi0ETwb+T/HTMnAscAyApHMKp6t2As8ElhfWv7eNfTxQeD5G\n9sZHvo/i+u1s69URMQy8CHhGsRZJz807fbdL2kUWCMtn3gyQ/ewfLPxsjwACVrVRR7v2AounTRsB\n9hzEsnPZlh1mHBi2ENwL/MW0T8uDEfGZ/JP2h4ELgaMiYglwK9mb6pRDueXyVmB14fWx7a4YEdcB\nHwPeV5j8aWADcGxEjACXcaDWmeq8F/jtaT/7QER8e6Z95n0fzR5/2KTU24BnF7YxBDw1nz7dXUBJ\n0prCtGcXlp3Ltuww48CwbitL6i88SmSBcH7+6VyShvLO42FgiOyNdjuApHPJWhjz5XPAWyWtkrQE\n+IM5rv8B4GWSpt5Eh4FHImJc0qlAcUjqdqABPKUw7TLgXYVRSCOS/mezneV9H80ef9lktauBZ0o6\nS1I/2Sm0H0TEHTNsf5RsFNSf57+H5wNnAp9oZ1uSknx6OXupfkmVZj+PPb44MKzbrgH2FR7vjoiN\nZJ3Pfw/sIOtEfTNARPwI+FvgO8CDwM8B35rHej4MfBm4Bfh+Xl8NqLezckRsB64E/iSf9Dtkb7Z7\n8mmfKyw7RtYp/638FNRpEXE18F7gKkm7yVpP6+bjB5tW41n5vneQ9f+cPTVf0h9K+o/CKr8DDJD1\nz3waeEtE3NbOtsg69veRHccn5c99EeVhQv4CJbMDJK0DLouI+R7aava45xaGHdEkDUg6Q1JJ0iqy\nUyxX97ous4XILQw7okkaBK4jG+20D/h34K0RMX1oqdkRz4FhZmZt8SkpMzNrS9fvmtlJy5cvj+OO\nO67XZZiZPW7ceOOND0XEinaWPawC47jjjmPjxo29LsPM7HFD0k/aXdanpMzMrC0ODDMza4sDw8zM\n2uLAMDOztjgwzMysLQ4MMzNriwPDzMza4sAALv7q3Vx31/Zel2FmtqA5MIBLv76Zb97twDAza8WB\nAZQSUW/0ugozs4XNgQEkiag3nBhmZq04MIA0EXXf5t3MrCUHBnlguIFhZtaSAwNI5VNSZmazcWDg\nFoaZWTscGECSQMN9GGZmLTkwgFKSUG84MMzMWnFgAIlwYJiZzcKBwVQfhgPDzKwVBwaQJomvwzAz\nm4UDA0gTn5IyM5uNA4Op6zAcGGZmrXQ0MCSdLulOSZskXTTDfEm6OJ9/i6RTCvP+t6TbJN0q6TOS\n+jtVZ5LIw2rNzGbRscCQlAKXAOuAE4HXSzpx2mLrgDX54zzg0nzdVcDvAWsj4plACpzdqVpL7vQ2\nM5tVJ1sYpwKbImJLRFSBq4D105ZZD1wZmeuBJZJW5vNKwICkEjAI/KxThSYSNQeGmVlLnQyMVcC9\nhdf35dNmXSYi7gfeB/wU2Arsiogvz7QTSedJ2ihp4/btB/clSGkiGg4MM7OWFmSnt6SlZK2P44Fj\ngCFJb5xp2Yi4PCLWRsTaFStWHNT+fHtzM7PZdTIw7geOLbxenU9rZ5mXAj+OiO0RMQl8EfjFThXq\nC/fMzGbXycC4AVgj6XhJFbJO6w3TltkAnJOPljqN7NTTVrJTUadJGpQk4CXA7Z0q1MNqzcxmV+rU\nhiOiJulC4FqyUU5XRMRtks7P518GXAOcAWwCxoBz83nflfTPwE1ADfg+cHmnak3cwjAzm1XHAgMg\nIq4hC4XitMsKzwO4oMm6fwr8aSfrm1LydRhmZrNakJ3e3ZYkHlZrZjYbBwZZH4aH1ZqZtebAwMNq\nzcza4cAgD4y6A8PMrBUHBvmwWrcwzMxacmAwNay211WYmS1sDgw8rNbMrB0ODLI+jJqbGGZmLTkw\nyG5v7lG1ZmatOTDwd3qbmbXDgQGkSeLAMDObhQODvIXhTm8zs5YcGPj25mZm7XBgkJ2SAnw/KTOz\nFhwYZKekAN+x1sysBQcG2ZXegC/eMzNrwYFB1ocBHlprZtaKA4PsSm/wSCkzs1YcGBQCw7c4NzNr\nyoGBWxhmZu1wYHAgMDys1sysOQcGBzq9PazWzKw5BwYHhtV6lJSZWXMODA60MHwdhplZcw4MoJS6\nhWFmNhsHBtkXKIEDw8ysFQcGHlZrZtYOBwaFwHALw8ysKQcGvpeUmVk7HBi4hWFm1g4HBr69uZlZ\nOxwYQGl/C6PHhZiZLWAODA4Mq601nBhmZs10NDAknS7pTkmbJF00w3xJujiff4ukU/LpJ0i6ufDY\nLeltnarzwM0HO7UHM7PHv1KnNiwpBS4BXgbcB9wgaUNE/Kiw2DpgTf54LnAp8NyIuBM4ubCd+4Gr\nO1Xr1Hd6+zoMM7PmOtnCOBXYFBFbIqIKXAWsn7bMeuDKyFwPLJG0ctoyLwE2R8RPOlVommSHoe4m\nhplZU50MjFXAvYXX9+XT5rrM2cBnmu1E0nmSNkrauH379oMqdKrTu+Zv3DMza2pBd3pLqgBnAp9v\ntkxEXB4RayNi7YoVKw5qP+X8nNSkA8PMrKlOBsb9wLGF16vzaXNZZh1wU0Q82JEKc5VSdhiq9Xon\nd2Nm9rjWycC4AVgj6fi8pXA2sGHaMhuAc/LRUqcBuyJia2H+62lxOmq+7A+MmvswzMya6dgoqYio\nSboQuBZIgSsi4jZJ5+fzLwOuAc4ANgFjwLlT60saIhth9dudqnFKJXVgmJnNpmOBARAR15CFQnHa\nZYXnAVzQZN1R4KhO1jdlqoUx4cAwM2tqQXd6d0vf/j4MB4aZWTMODA6MkvIpKTOz5hwYZLcGSRM5\nMMzMWnBg5Cpp4sAwM2vBgZGrlBL3YZiZteDAyFVKbmGYmbXiwMhVUrcwzMxacWDk+tzCMDNryYGR\n8ykpM7PWHBg5d3qbmbXmwMh5WK2ZWWsOjJxPSZmZtebAyPmUlJlZaw6MnE9JmZm15sDI+ZSUmVlr\nDoxcpZT4+zDMzFpwYOT63IdhZtaSAyPnPgwzs9YcGDn3YZiZtebAyHlYrZlZaw6MXCVNqTeCeiN6\nXYqZ2YLkwMhVSv5ebzOzVhwYOQeGmVlrDoxcKREAtYYDw8xsJg6MXJoHRj3ch2FmNhMHRm6qheFO\nbzOzmbUVGJKeKqkvf/4iSb8naUlnS+uuZOqUVN2BYWY2k3ZbGF8A6pKeBlwOHAt8umNV9cBUC6Ph\nU1JmZjNqNzAaEVEDXgN8KCLeAazsXFndl+7v9HZgmJnNpN3AmJT0euBNwL/l08qdKak3UvdhmJm1\n1G5gnAs8D/iLiPixpOOBT3SurO5zp7eZWWuldhaKiB8BvwcgaSkwHBHv7WRh3ZYmWXY6MMzMZtbu\nKKmvS1osaRlwE/BhSX/X2dK6K82PhPswzMxm1u4pqZGI2A28FrgyIp4LvLRzZXWfWxhmZq21Gxgl\nSSuB13Gg03tWkk6XdKekTZIummG+JF2cz79F0imFeUsk/bOkOyTdLul57e73YLgPw8ystXYD48+B\na4HNEXGDpKcAd7daQVIKXAKsA04EXi/pxGmLrQPW5I/zgEsL8z4IfCkingE8G7i9zVoPSiLfS8rM\nrJV2O70/D3y+8HoLcNYsq50KbMqXRdJVwHrgR4Vl1pOd4grg+rxVsRIYA14IvDnfXxWotlPrwSql\n+YV7zgszsxm12+m9WtLVkrbljy9IWj3LaquAewuv78untbPM8cB24KOSvi/pI5KGmtR2nqSNkjZu\n3769nR9nRqnvVmtm1lK7p6Q+CmwAjskf/y+f1ikl4BTg0oh4DjAKPKYPBCAiLo+ItRGxdsWKFQe9\nw1TuwzAza6XdwFgRER+NiFr++Bgw27vz/WT3nJqyOp/WzjL3AfdFxHfz6f9MFiAd4yu9zcxaazcw\nHpb0Rklp/ngj8PAs69wArJF0vKQKcDZZK6VoA3BOPlrqNGBXRGyNiAeAeyWdkC/3Eh7d9zHvpvow\nHBhmZjNrq9Mb+E3gQ8D7gQC+Td4h3UxE1CRdSDa6KgWuiIjbJJ2fz78MuAY4A9hE1tF9bmETvwt8\nKg+bLdPmzbtUvvmgmVkr7Y6S+glwZnGapLcBH5hlvWvIQqE47bLC8wAuaLLuzcDaduqbD6lvb25m\n1tKhfOPe789bFQtAKb/S21+gZGY2s0MJDM1bFQtAnhfuwzAza+JQAuOwemedamHUfUrKzGxGLfsw\nJO1h5mAQMNCRinrE37hnZtZay8CIiOFuFdJr+6/DqPtKbzOzmRzKKanDilsYZmatOTByJQ+rNTNr\nyYGRcwvDzKw1B0buQB+GA8PMbCYOjNz+u9X6lJSZ2YwcGLkkEZIv3DMza8aBUVBK5D4MM7MmHBgF\naSIaDgwzsxk5MApSuYVhZtaMA6MgTeQ+DDOzJhwYBaU0cWCYmTXhwChIfErKzKwpB0ZBKRH1hm8+\naGY2EwdGQdaH0esqzMwWJgdGQeoWhplZUw6MAl+4Z2bWnAOjIE3k25ubmTXhwChIE1Hz3WrNzGbk\nwCjwhXtmZs05MApKiXx7czOzJhwYBYlbGGZmTTkwCkruwzAza8qBUZD6lJSZWVMOjAJ3epuZNefA\nKEiTxBfumZk14cAoKPkb98zMmnJgFKS+NYiZWVMOjIJUvvmgmVkzHQ0MSadLulPSJkkXzTBfki7O\n598i6ZTCvHsk/VDSzZI2drLOKZVSQrXmwDAzm0mpUxuWlAKXAC8D7gNukLQhIn5UWGwdsCZ/PBe4\nNP93yosj4qFO1ThdfzlhfNKBYWY2k062ME4FNkXEloioAlcB66ctsx64MjLXA0skrexgTS31l1PG\na/Ve7d7MbEHrZGCsAu4tvL4vn9buMgF8RdKNks5rthNJ50naKGnj9u3bD6ng/nLK+KQDw8xsJgu5\n0/v5EXEy2WmrCyS9cKaFIuLyiFgbEWtXrFhxSDvsL2WnpMJXe5uZPUYnA+N+4NjC69X5tLaWiYip\nf7cBV5Od4uqovnIKwIQ7vs3MHqOTgXEDsEbS8ZIqwNnAhmnLbADOyUdLnQbsioitkoYkDQNIGgJe\nDtzawVqB7JQUwIQ7vs3MHqNjo6QioibpQuBaIAWuiIjbJJ2fz78MuAY4A9gEjAHn5qsfDVwtaarG\nT0fElzpV65T+cpaf47U6I5Q7vTszs8eVjgUGQERcQxYKxWmXFZ4HcMEM620Bnt3J2mbSV8paGO74\nNjN7rIXc6d11+1sYPiVlZvYYDoyCfrcwzMyacmAUTHV6OzDMzB7LgVFwoNPbp6TMzKZzYBS4hWFm\n1pwDo+BAp7cDw8xsOgdGwdSwWl+4Z2b2WA6Mgv2npHzHWjOzx3BgFPiUlJlZcw6MAt9LysysOQdG\nQTlNSBP5lJSZ2QwcGNNMfSeGmZk9mgNjmoFKyuhErddlmJktOA6MaY47aohN2/b2ugwzswXHgTHN\niccs5vatu2k0/DWtZmZFDoxpTjpmMaPVOvfuGOt1KWZmC4oDY5oTV44A8MP7d/W4EjOzhcWBMc0J\nTxxmsJLyvR8/0utSzMwWFAfGNJVSwtrjlvGdzQ/3uhQzswXFgTGD5z3lKO7etpct2z1aysxsigNj\nBmedsopFfSUu+sIP2bVvstflmJktCA6MGTxhcT/vefUzuemnO/iV932dL926tdclmZn1nAOjiVc/\nZxX/csEvsXrpAOd/8ib+7st3EuFrM8zsyOXAaOGZq0b43PnP43VrV3Pxf23iU9/9aa9LMjPrGQfG\nLPpKKX/12mfx/Kct531fvpO9vs+UmR2hHBhtSBLx9lecwM6xSS6/bnOvyzEz6wkHRptOPnYJr3nO\nKv7h65v591vcCW5mRx4Hxhy8+8yTeNbqES749E38xj99l+u3+OI+MztyODDmYGSgzGfOO42L1j2D\nux7cw9mXX895V27kjgd297o0M7OOc2DMUV8p5fxffirXvePFvP3lT+c7mx/mVRd/k498Y0uvSzMz\n6ygHxkHqL6dc+Ctr+MYfvJiXnXg07/n32/nLa273tRpmdthyYByiJYMV/v7XT+Gc5z2Zy/97C+/6\n4g8dGmZ2WCr1uoDDQZqIPzvzJIb7S1zytc0cu2yQC178tF6XZWY2rzrawpB0uqQ7JW2SdNEM8yXp\n4nz+LZJOmTY/lfR9Sf/WyTrngyTe/vITWH/yMfzNtXf6/lNmdtjpWGBISoFLgHXAicDrJZ04bbF1\nwJr8cR5w6bT5bwVu71SN800S7z3rWZx87BLe9tmb+dod23pdkpnZvOlkC+NUYFNEbImIKnAVsH7a\nMuuBKyNzPbBE0koASauBVwIf6WCN866/nPKRN63laU9YxG99/Ab+4eubqNYavS7LzOyQdTIwVgH3\nFl7fl09rd5kPAO8EWr7bSjpP0kZJG7dv335oFc+T5Yv6+Ox5z+MVJz2Rv/7Snbz8/dfxpVu3Um+4\nM9zMHr8W5CgpSa8CtkXEjbMtGxGXR8TaiFi7YsWKLlTXnqG+Epe+8ef52Lm/QJqI8z95Ey9473/x\nwa/czQO7xntdnpnZnHVylNT9wLGF16vzae0scxZwpqQzgH5gsaRPRsQbO1hvR7zohCfwS09bzldv\nf5BPffenvP8rd/HBr97FLz1tefZ46nJOPGYxaaJel2pm1pI6dc2ApBJwF/ASshC4Afj1iLitsMwr\ngQuBM4DnAhdHxKnTtvMi4O0R8arZ9rl27drYuHHjvP0MnfDTh8e46oaf8pXbH+SuB7PvDB8ZKPOs\n1SMsG6pwwhOHWTZYYdlQhdVLBxnuL7FiuI9ympAo61g3M5svkm6MiLXtLNuxFkZE1CRdCFwLpMAV\nEXGbpPPz+ZcB15CFxSZgDDi3U/UsFE86apB3nv4M3nn6M9i2Z5zvbH6Yb216iDse2MPdD+7lX2/+\nWcv1Vy8d2B8eQ30lBispQ5USg30lhiopQ30l+ssJ45MNFveXGavWWDHcx+L+Mn3lhIFySn/+yJ4n\n+19PPS8lYvd4jcX9JQeUme3XsRZGLzweWhizGavW2Dk2ybY9E/xs5z72jE/y0N4q9UYwWW9wz8Nj\nRASNCMaqdcYm6oxWa4xV64xOZP+OVWv0l1PGqnXKqZisz+13nAgaAcN9JZS3aoYqKQN5IA1WUibr\nwUA55QnDfQTQiGDJQJllQ30MVPIQKqXs2jfJov4SxywZYGKyTgBrnrCISikhkSinCZVSQl8poZwm\nPjVn1mULooVhB2ewUmKwkr3BnnzskkPa1mS9QSqxe3ySvRM1xicbjE/W80f2fN/U61qD8Wr2fKLW\nYLi/xNZd4zQiSKT9YTQVTpU0Ye9EjR8/NEqSgBA7x6rsHj+0byQcGShTSsTRi/sZrKSsGO5Dgkqa\nhdAxSwZYOlimv5wyWCkxUEkYKJdYvqhCAIv7yywdKtNXSg+pDjN7LAfGYaycZoPglgxWWDJY6co+\n6414VBAN95XZMzHJ1l3jVNKERgRbto9Sj6CRt5omag0m69l6O8aqTNYb/GznOPuqdTZt20uQhd9Y\ntc72PRNt1bGor8TSoTLLBissHaoc+Dd/LB2cel5m6WCFUpow3FcicQvHrCkHhs2rNBFDfSWG+g78\naY0Mllm9dHD/6+c8aelBb398ss6e8Rrjk3XGqlkwjU3U2LZngiQRe8Yn2TFa5ZHRSXaMVXlkNHvc\n/eBedoxVGavWm267nIqVIwOsWjLA0Yv7qJQSnn70MCMDZRYPlFncX86fl1g8UGZRxQFjRxYHhj2u\nTHXQH6ypVswjo1V2jE7y8OgEj4xmfUQPj1a5b8c+7t8xxsaf7GB0osbnNt7XdFuJYDgPkZGB8v7w\nGMgHDwxWUo4ZGaC/nJIkYulgmTQR+6r1rM+mlHDUUB/1RlApJdQbDfrLKYv6ssEGjQjqjWD5ouy0\nXK0e9JUS+soJfaWUveM1FvWXSCVqjQb1CCh0V/XlAxv2TdYpJSJNRCJRSjRj0EUEtUbsb5maTefA\nsCNKfzll5cgAK0cGZl02Iti9r8bu8Ul27Ztk975Jdo9PsntfLXudT5+aVw8Q2cCFHaNVNm+v8rU7\ntjNeq9OrsSWtBj1MBUeah0gjgvFag6OH+9g9XqOvlDDYl7JjdJJlQxVq9QYDlWwgQ5qI4f4yg5WU\nrbvGKSfKBy+kLBuqMNxfYrLeoK+Ukqbikb1V6hH0l1OG+0s0GsFgpcRErU6Sj8SbqGWtv/5yyuL+\nMv3lhCQR/aWURgTVeoNqrUEiMVlvUEoSyiWxYlFW72S9gQAJUoknjgywc1+Vej0Y7CuxqC/l/p3j\nrFhUoa+UMl6rMzJQZve+SY5e3M9YtU5/OWWy3mBRX4kdY1VWjgxQSsXOsUmWDpap5qdQJyYbHLWo\nwshAmdGJGkN9JYbz8N68fZTBSvbBJk3E4oESoxN16o0GSwYr1OrZqdisJZ5Sq2cDWBJBIiFBkojh\nvhKN/G8qScT4ZB2JnvbPOTDMmpDEyGCZkcHyo64unatGfkuYnfsmaeRvmmPVGtVag4f2VukrJdTq\nkbU+JrPRbo2I/S2CbXvG85ZBQrVeZyIfsDBYKbF3IhtkUEqFJIoNh0f2Vhmt1lm+qJK3VqDeaBz4\nd9q0IKvtwd3jLO4v7+83Gu4vsWNskkqaMDpRY8lgmUYepqPVGv/jiYupR2R9UbUGD+4ZZ8cjVSql\nbPkIWDZUIVE2MOLeR8ZIEzE2UaMvby1GBH2llCAYn2ywc6zKZD2oR1CtNfYPfKikCUHWuqvWG9Tq\nWatIygJwqpHViOhZSM+XqbAvp2LJYIWH9k7QX0qzEZONRhb4ylqOTxju4+vveHHHa3JgmHXY1Omf\nZUMHBh4syvt4in07NrNGHgjTrwmKPBR2j08y3F9+1JDsWr3B1l3jHLWoQjlNGJuos7daY2SgzM6x\n7BTkQCVrPQ1WUh4ZrTLUV8pOF5YT9oxny27dtY96I1gyWGHnWJW+UpqfEky4b8c+qrUGQ33ZEPa9\neSvnyUcNMVatUW9krabRap1FfWkemJOU0qxVN1bN+uP6ywmDlTQLukbQCPafIp26puqR0QlWjgyw\ncywL4v5yFhxTj8FKd1odvg7DzOwINpfrMNy7ZWZmbXFgmJlZWxwYZmbWFgeGmZm1xYFhZmZtcWCY\nmVlbHBhmZtYWB4aZmbXlsLpwT9J24CcHufpy4KF5LGe+uK65cV1zs1DrgoVb2+FW15MjYkU7Cx5W\ngXEoJG1s92rHbnJdc+O65mah1gULt7YjuS6fkjIzs7Y4MMzMrC0OjAMu73UBTbiuuXFdc7NQ64KF\nW9sRW5f7MMzMrC1uYZiZWVscGGZm1pYjPjAknS7pTkmbJF3U5X0fK+lrkn4k6TZJb82nv1vS/ZJu\nzh9nFNZ5V17rnZJe0cHa7pH0w3z/G/NpyyT9p6S783+XdrMuSScUjsnNknZLeluvjpekKyRtk3Rr\nYdqcj5Gkn8+P9SZJF2v6V8vNT11/I+kOSbdIulrSknz6cZL2FY7dZV2ua86/uy7V9dlCTfdIujmf\n3s3j1ez9oXd/Y9nXHB6ZDyAFNgNPASrAD4ATu7j/lcAp+fNh4C7gRODdwNtnWP7EvMY+4Pi89rRD\ntd0DLJ827a+Bi/LnFwHv7XZd0353DwBP7tXxAl4InALceijHCPgecBog4D+AdR2o6+VAKX/+3kJd\nxxWXm7adbtQ1599dN+qaNv9vgT/pwfFq9v7Qs7+xI72FcSqwKSK2REQVuApY362dR8TWiLgpf74H\nuB1Y1WKV9cBVETERET8GNpH9DN2yHvh4/vzjwKt7WNdLgM0R0erK/o7WFRH/DTwywz7bPkaSVgKL\nI+L6yP5nX1lYZ97qiogvR0Qtf3k9sLrVNrpVVws9PV5T8k/irwM+02obHaqr2ftDz/7GjvTAWAXc\nW3h9H63fsDtG0nHAc4Dv5pN+Nz99cEWhydnNegP4iqQbJZ2XTzs6Irbmzx8Aju5BXVPO5tH/iXt9\nvKbM9Rityp93s8bfJPuUOeX4/PTKdZJekE/rZl1z+d11+3i9AHgwIu4uTOv68Zr2/tCzv7EjPTAW\nBEmLgC/0+cejAAADyElEQVQAb4uI3cClZKfJTga2kjWJu+35EXEysA64QNILizPzTyo9GZMtqQKc\nCXw+n7QQjtdj9PIYNSPpj4Aa8Kl80lbgSfnv+veBT0ta3MWSFuTvruD1PPqDSdeP1wzvD/t1+2/s\nSA+M+4FjC69X59O6RlKZ7I/hUxHxRYCIeDAi6hHRAD7MgdMoXas3Iu7P/90GXJ3X8GDevJ1qgm/r\ndl25dcBNEfFgXmPPj1fBXI/R/Tz69FDHapT0ZuBVwBvyNxry0xcP589vJDvv/fRu1XUQv7tuHq8S\n8Frgs4V6u3q8Znp/oId/Y0d6YNwArJF0fP6p9WxgQ7d2np8f/Sfg9oj4u8L0lYXFXgNMjd7YAJwt\nqU/S8cAass6s+a5rSNLw1HOyDtNb8/2/KV/sTcC/drOugkd96uv18ZpmTscoP7WwW9Jp+d/DOYV1\n5o2k04F3AmdGxFhh+gpJaf78KXldW7pY15x+d92qK/dS4I6I2H86p5vHq9n7A738GzuUXvzD4QGc\nQTb6YDPwR13e9/PJmpO3ADfnjzOATwA/zKdvAFYW1vmjvNY7OcRRGC3qegrZaIsfALdNHRfgKOCr\nwN3AV4Bl3awr388Q8DAwUpjWk+NFFlpbgUmy88K/dTDHCFhL9ka5Gfh78jswzHNdm8jOb0/9nV2W\nL3tW/ju+GbgJ+NUu1zXn31036sqnfww4f9qy3Txezd4fevY35luDmJlZW470U1JmZtYmB4aZmbXF\ngWFmZm1xYJiZWVscGGZm1hYHhtkcSKrr0XfMnbc7HCu7E+qtsy9p1hulXhdg9jizL7LbQpgdcdzC\nMJsHyr4z4a/z7xz4nqSn5dOPk/Rf+c31virpSfn0o5V9L8UP8scv5ptKJX1Y2fcffFnSQM9+KLNp\nHBhmczMw7ZTUrxXm7YqInyO7kvYD+bQPAR+PiGeR3fDv4nz6xcB1EfFssu9iuC2fvga4JCJOAnaS\nXVlstiD4Sm+zOZC0NyIWzTD9HuBXImJLfsO4ByLiKEkPkd3uYjKfvjUilkvaDqyOiInCNo4D/jMi\n1uSv/wAoR8R7Ov+Tmc3OLQyz+RNNns/FROF5Hfcz2gLiwDCbP79W+Pc7+fNvk90FGeANwDfy518F\n3gIgKZU00q0izQ6WP72Yzc2ApJsLr78UEVNDa5dKuoWslfD6fNrvAh+V9A5gO3BuPv2twOWSfous\nJfEWsjummi1Y7sMwmwd5H8baiHio17WYdYpPSZmZWVvcwjAzs7a4hWFmZm1xYJiZWVscGGZm1hYH\nhpmZtcWBYWZmbfn/ToWiGvHbj7EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21802343668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "% matplotlib inline\n",
    "plt.plot(np.arange(0, epochs, 5), losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "ax=plt.title('Learing Rate = ' + str(learning_rate))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}