2. TensorFLow Way.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Agenda\n",
    "* Ops in a Computational Graph\n",
    "* Layering Nested Layers\n",
    "* Working with Multiple Layers\n",
    "* Implementing Loss Functions\n",
    "* Implementing BackPropogation\n",
    "* Working with Batch & Stochastic Training\n",
    "* Evaluating Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_vals = np.array([1., 3., 5., 7., 9.])\n",
    "x_data = tf.placeholder(tf.float32)\n",
    "m_const = tf.constant(3.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_product = tf.mul(x_data,m_const)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.0\n",
      "9.0\n",
      "15.0\n",
      "21.0\n",
      "27.0\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "for x_val in x_vals:\n",
    "    print(sess.run(my_product, feed_dict={x_data:x_val}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Layering Nested Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_array = np.array([[1., 3., 5., 7., 9.],\n",
    "                   [-2., 0., 2., 4., 6.],\n",
    "                   [-6., -3., 0., 3., 6.]])\n",
    "\n",
    "x_vals = np.array([my_array, my_array + 1])\n",
    "\n",
    "x_data = tf.placeholder(tf.float32, shape=(3, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[  1.,   3.,   5.,   7.,   9.],\n",
       "        [ -2.,   0.,   2.,   4.,   6.],\n",
       "        [ -6.,  -3.,   0.,   3.,   6.]],\n",
       "\n",
       "       [[  2.,   4.,   6.,   8.,  10.],\n",
       "        [ -1.,   1.,   3.,   5.,   7.],\n",
       "        [ -5.,  -2.,   1.,   4.,   7.]]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m1 = tf.constant([[1.],[0.],[-1.],[2.],[4.]])\n",
    "m2 = tf.constant([[2.]])\n",
    "a1 = tf.constant([[10.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prod1 = tf.matmul(x_data, m1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prod2 = tf.matmul(prod1, m2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Const_1:0' shape=(5, 1) dtype=float32>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "add1 = tf.add(prod2, m2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 94.]\n",
      " [ 58.]\n",
      " [ 50.]]\n",
      "[[ 106.]\n",
      " [  70.]\n",
      " [  62.]]\n"
     ]
    }
   ],
   "source": [
    "for x_val in x_vals:\n",
    "    print (sess.run(add1, feed_dict={x_data:x_val}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  2.,   4.,   6.,   8.,  10.],\n",
       "       [ -1.,   1.,   3.,   5.,   7.],\n",
       "       [ -5.,  -2.,   1.,   4.,   7.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Working with Multiple Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#2D image with numpy. Image with 4X4 pixel\n",
    "#image number, height,width,channel\n",
    "x_shape = [1, 4, 4, 1]\n",
    "x_val = np.random.uniform(size=x_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 4, 4, 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_data = tf.placeholder(tf.float32, shape=x_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_filter = tf.constant(0.25, shape=[2,2,1,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_strides = [1,2,2,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "mov_avg_layer = tf.nn.conv2d(x_data, my_filter, my_strides, padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def custom_layer(input_matrix):\n",
    "    input_matrix_sqeezed = tf.squeeze(input_matrix)\n",
    "    A = tf.constant([[1., 2.], [-1., 3.]])\n",
    "    b = tf.constant(1., shape=[2, 2]) \n",
    "    temp1 = tf.matmul(A,input_matrix_sqeezed)\n",
    "    temp = tf.add(temp1, b)\n",
    "    return tf.sigmoid(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('Custome_Layer') as scope:\n",
    "    custom_layer1 = custom_layer(mov_avg_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.90649736,  0.93201917],\n",
       "       [ 0.83657342,  0.90521866]], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(custom_layer1, feed_dict={x_data:x_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://www.safaribooksonline.com/library/view/tensorflow-machine-learning/9781786462169/graphics/B05480_02_03.jpg\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_vals = tf.linspace(-1., 1., 500)\n",
    "target = tf.constant(0.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l2_y_vals = tf.square(target - x_vals)\n",
    "l2_y_out = sess.run(l2_y_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1_y_vals = tf.abs(target - x_vals)\n",
    "l1_y_out = sess.run(l1_y_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x12e2abc62e8>]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Wd0VFUXBuB3h9B7lV5UioBgiYiKiihNFEFBKQICgpEi\nqIgUkQ4qNpQmAlakfKKCCqKgiIiUgPQamoL0SG8hOd+Pd6JjSJkkt81kP2tlkczcuXeTTHbOPWUf\nMcZAKaVUaAlzOwCllFLW0+SulFIhSJO7UkqFIE3uSikVgjS5K6VUCNLkrpRSIUiTu1JKhSBN7kop\nFYI0uSulVAgKd+vCRYoUMeXLl3fr8kopFZTWrFlzzBhTNLXjXEvu5cuXR1RUlFuXV0qpoCQi+wI5\nTrtllFIqBGlyV0qpEKTJXSmlQpAmd6WUCkGa3JVSKgSlmtxFZJqIHBGRTck8LyLyjohEi8gGEbnJ\n+jCVUkqlRSAt9w8BNErh+cYAKvo+ugKYmPGwlFJKZUSqyd0YsxRATAqHPATgY0MrABQQkRJWBZjY\ntm1A7x6XcenMJbsuoZRSthk28CKWLrX/Olb0uZcC8Kff1/t9j11BRLqKSJSIRB09ejRdF9u99SLG\njg/H1499mq7XK6WUW3aN+QKDR2XH0q9Sai9bw9EBVWPMZGNMhDEmomjRVFfPJqlh0+wonedvvD+/\nJDB7tsURKqWUTXbtwrQB0QhDHJ54Jp/tl7MiuR8AUMbv69K+x2yRJQvQqVc+fI8G2NtxKPtplFLK\n4y6XqYAPcnVD4/tiUbq8/ZVfrEju8wC0982aqQ3gpDHmoAXnTVanLlkAEXxgngAeeQQ4c8bOyyml\nVPoZAxw+jPnfheHgqTzo0iOHI5cNZCrkDAC/AagsIvtFpLOIRIpIpO+Q+QB2A4gG8D6AbrZF61Ou\nHNCggWBanp6IuxAL/PWX3ZdUSqn0eestoGpVTHnnLIoXB+6/35nLpnpvYIxpncrzBkB3yyIK0JNP\nAi1b5sDCKVtwfyXXilsqpVTyli0D+vbFgfpP4Nvvc6FvXyBrVmcuHbQrVJs2BYoWBd7/IBy4eBHo\n3RtYtcrtsJRSig4fBh59FKhQAR9GjEN8vKBzZ+cuH7TJPVs2oEMH4OuvgUO7zwFffQW0bAkcP+52\naEqpzO7yZaBVK+DECcT/bw6mTs+Be+4Brr3WuRCCNrkD7JqJiwM+nFsQ+Pxz4NAh4PHHgfh4t0NT\nSmVm588DOXMCEyfip+M1sGcP85WTgjq5V64M3HUXMGUKYG6OAMaOBb77Dhgxwu3QlFKZWd68wLff\nAh06YMoUoGBB4OGHnQ0hqJM7wL+Gu3YBS5YAeOopoF07YPRonUGjlHLe7t2cDrN/PyCCY8eAL75g\nWsrhzAzIfwR9cm/RAsifn613iACTJgG//gqULOl2aEqpzOTCBSak334DYmMBAJ9+Cly65HyXDBAC\nyT1nTnazz5kDxMQAyJULuMlXdXjhQn5nlVLKbj17Ar//DnzyCVChAoxho/PWW4Hrr3c+nKBP7gDQ\npQtnQ37qX0tswwagUSOgTx/X4lJKZRIffshMPmAA8MADAIAVK4DNm91ptQMhktxr1gQiIoD33+dK\nXwBAjRrAs88C774LzJzpanxKqRAWG8txvnr1gGHD/nl48mQgTx7gscfcCSskkjvA1vumTfxr+Y9X\nXwXuuIN/OrdudS02pVQIy5qV43wzZrCyIdhFPHMmu4zz5nUnrJBJ7q1b86/kpEl+D2bNCsyaBeTO\nzQJj58+7Fp9SKsQYA3zwAVvuRYoAxYr989THH3N8NTIyhdfbLGSSe968/Cs5a5ZvYDVBqVL8ixoZ\n6fxcJKVU6HrjDaBTJ+B///vPw8awkXnbbewydkvIJHcAePppDqx++GGiJ+rVA555hlMlz551IzSl\nVChZuhTo149TH1v/t7bikiXA9u3uttqBEEvuNWoAt9/Ov5pJViBYtgyoUAFYudLx2JRSIeLgQY6S\nXnMNMHUqG41+Jk3iitSWLV2KzyekkjvA1vvOncCPPybxZLVq7H9v2RI4dszx2JRSQc4YoH174ORJ\n1rPK99/t8g4d4orUjh25BsdNIZfcW7QAChcGJk5M4smCvgJjhw8Dbduy6phSSgVKBBg8GPjooyRX\nJk2bxoKQXbu6EFsiIZfcc+TgX825c5MpL3PzzZz7/v33wPDhjsenlApSR4/y3zp1kuxziYsD3nuP\nQ3yVKzscWxJCLrkDrB8WF+erN5OULl14a7V5s5YHVkqlbtcuoFKlZLoE6LvvgD/+YNewF4j5Z0mn\nsyIiIkxUVJRt52/YkLl7714gPKld+C5d4jz4RIMhSin1H+fPc6bGvn3A2rVA+fJJHvbAA8CaNUzw\ndm6lJyJrjDERqR0Xki13gH89DxwAvvkmmQOyZWNi370b6NyZcyiVUiqxHj2AdetYECyZxL5vHzB/\nPhfDO7VHampCNrk/8ABQunSKd1G0fj1HQZ57zpG4lFJBZNo0frz0EtCkSbKHTZ7MtmKXLg7GloqQ\nTe7h4fxGf/89EB2dwoHNmzOxT5gAfPaZY/EppYLAqVOsLjtkSLKHXLrE6e733w+ULetcaKkJ2eQO\n8BYpSxb+VU3RK69wBLxLF3bUK6UUAPTuze3yfAXBkvLVV5xd7ZWB1AQhndxLlgSaNeNd1YULKRyY\nUGAsTx6gb1/H4lNKeZAxnE/97bf8OizlNDl+PLviGza0P7S0COnkDvCv6fHjV9T2uVLJkpzLNH26\nI3EppTxqzBgWqNq5M9VDN2xgmZlu3VJs3Lsi5JN7vXpAlSpct5SqG28EChTgzJmffrI9NqWUx/z8\nM9C/Pxcp9eqV6uHjxnHhZOfODsSWRiGf3EW4teHq1WmoF/byy0CDBtzoVimVOSQUBKtYkSsgU1kD\nExPDrT3btgUKFXIoxjQI+eQOcDFqvnzAO+8E+IL+/Tns/eij/y45VkqFtokTgdOngTlzrigIlpRp\n07i+qWdPB2JLh0yR3PPk+bem/sGDAbygQAEWGDt6FGjTRguMKZUZDBnC2/tq1VI9NC6Os6fvvNPd\nDTlSkimSOwB0785qbe+9F+ALbryRw+CLFmmBMaVC2eLFrBkQFgZUrx7QS+bPB/bs8W6rHQgwuYtI\nIxHZLiLRItIviefzi8jXIrJeRDaLSEfrQ82Ya6/lIoNJk7joICCdOwMDBnC5q1Iq9OzcCTz8MKsN\npsG773IHz2bNbIrLAqkmdxHJAmA8gMYAqgJoLSJVEx3WHcAWY0xNAHUBvCEi2SyONcN69uRig1Sn\nRfobORKI8NXo0Q22lQod585xA4jwcLb6ArRtG/DDD9xGzyt1ZJISSMu9FoBoY8xuY8wlADMBPJTo\nGAMgr4gIgDwAYgBctjRSC9SvzzrLAU2LTGzAAODuu7XAmFKhwBhOTt+4kVNeypUL+KXjx7PuoBc2\n5EhJIMm9FIA//b7e73vM3zgA1wH4C8BGAL2MMZ4rlB4WxgJvK1cCq1al8cW1anE+5bPP2hKbUspB\nn3zC3ZReeglo3Djgl506xfVNjz0GFCtmX3hWsGpAtSGAdQBKArgBwDgRuWIukYh0FZEoEYk66tIU\nww4dgLx509F6b9YMeOEFTpfSVaxKBbcmTbieZfDgNL3so4+AM2e8PZCaIJDkfgBAGb+vS/se89cR\nwBeGogHsAVAl8YmMMZONMRHGmIiiRYumN+YMyZuX0yJnzeJmtmkyahRw1128H9u0yZb4lFI2On2a\nMyoKFwaGDk1TzYD4eHbJ3HorcMstNsZokUCS+2oAFUWkgm+QtBWAeYmO+QPAvQAgIlcBqAxgt5WB\nWql7dyA2NoBqkYmFhwMzZwJFirCohFIqeMTHczlpgwbp2l5z4UJg+/bgaLUDASR3Y8xlAD0ALASw\nFcBsY8xmEYkUkUjfYcMB3C4iGwEsBvCiMeaYXUFnVMWKnBY5cWIapkUmKFGCP+E2bWyJTSllk9de\nA77+GnjkkVQrPSblzTdZXzCJvbE9KandRa9gjJkPYH6ixyb5ff4XgAbWhmavnj05jvL55+nI0zly\n8N+EVaxeK+SslPqvn34CBg4EWrXirIo02riR6xlHjeJMmWCQaVaoJtagAadFvvUWZ0WlmTEcWH3m\nGeDXXy2PTyllkQMHmNQrVwbefz/VgmBJefttIFeuNK91clWmTe5hYZzVGBUF/PJLOk4gAnzwAefH\nPvoocOSI5TEqpSxw9CjLNs6Zw0JTaXT4MKfCd+jgzeqPycm0yR1gtcjChdmXli4JBcZiYrTAmFJe\ndcMNnN123XXpennC2FwA5d09JVMn95w5uUht3ryANl1J2g03sDzc4sXA7NmWxqeUyoA5c4A+fTg1\nLp3bJF24wF/vBx5gr04wydTJHWByz5qVfWrp1rEj8P337NdTSrlvxw7+Xv7yS7qmPSaYPp29OsG4\nMD3TJ/fixYHHH2f3eUxMBk5Uvz774aOjgb17rQpPKZVWCQXBsmVjlcDs2dN1GmM44aJGDeCeeyyO\n0QGZPrkD/Kt8/nwaar0nJzYWuO8+vrEuXLAkNqVUGhjDqcmbNrHZXbZsuk+1aBGweTPzQzom2LhO\nkztYn79BA9abSfOiJn9Zs3IvvzVrgN69LYtPKRWgzZuZ1AcPBho2zNCp3nwTuOoqoHVri2JzmCZ3\nn+ee4xZ8M2dm8ERNmwIvvsjbgE8+sSQ2pVSAqldn42rQoAydZutW4LvvWKoknb06rtPk7tOgAbdO\nfPPNdC5q8jdiBFC3Llc8aIExpewXEwMsWMDPa9ZMV3kBf2+9xaQeGZn6sV6lyd1HhK339euBH3/M\n4MnCw4EZMzhSW6ZM6scrpdIvPp6LVpo142rUDDp0iKV9O3YEXCpeawlN7n7atGEB/jfesOBkxYuz\n7GT+/Ny9KcO3A0qpJL3yCvDtt7ztLpV4H6G0GzuWcyOef96C2Fykyd1PjhysKbRggYW9KX//Ddx+\nO+/zlFLWWryY/eutW3PRSgadOsUVqY88Alx7rQXxuUiTeyLdugG5c7M6qCUKFGD9mb59gWXLLDqp\nUgpHjjCpV67Mu2QL5iu+/z5w8iR/XYOdJvdEChcGunRhl/m+fRacMKHAWIUKLDB2+LAFJ1VKoWhR\nZuF0FgRL7NIl3mDfc09w7LSUGk3uSXjuOf6b7oJiieXPzwJjf//NlsblyxadWKlM6sQJNpz69El3\nQbDEZszgeGwotNoBTe5JKlOGu3FNmQIcP27RSWvWZGfe3r3AX39ZdFKlMqHPPweuuYY7aFgkPp5d\nsTVqZHjtk2dock9G374sUTFunIUnfeIJjtRmYEm0Upna9u3c4b5SJUvLNM6fD2zZwt/7YCw1kBRN\n7smoWpWLTd95Bzh71sIT58rFqZHPPw/s9uwe4kp5z9mznMaSPTvLa1u4391rr7HN9eijlp3SdZrc\nU/Dii1z4NnWqxSc+dAiYNk0LjCkVKGO4XHTLFuCzzyxdHPjbb6wM/NxzLA8VKjS5p+D224E6dbio\nKTbWwhOXK8e6M7//zp26lVIpu3iRLa2hQ1le20JjxgAFCwKdO1t6Wtdpck9Fv37AH39YUFAssQce\nAPr356jthx9afHKlQkyOHMDXXwMDB1p62q1bga++YoEwC2ZTeoom91Tcfz8Lzb32mg0VBIYNA+rV\nY8HokyctPrlSIeD4caBlS7awwsIyXBAssdGjud1msO2PGghN7qkQYd/7pk0sX2GphAJjP/zAufBK\nqX/FxwPt2gFz53KcymK7drH7PjISKFLE8tO7TpN7AB57DChfHhg50obWe7FiQEQEP1+6VAuMKZVg\n5EgWeho7FqhVy/LTv/oq21fBXiAsOZrcA5A1K/veV6xgnSJb/PQTcPfdFpWkVCrI/fADd1Nq29aW\nour793Ooq3NnoGRJy0/vCWJcailGRESYqKgoV66dHhcvclHcNdcAP/9swwWM4STbL79kQfm77rLh\nIkoFAWNY3OXCBWDlSlbys1ivXsCECdzPvlw5y09vKxFZY4yJSO04bbkHKHt2rl5bupQflhPhhPpr\nrmE/kA19jEoFBRHucTdvni2J/fBhVn9s1y74EntaaHJPgy5duGHu8OE2XSBfPtbNOHkSaNUKiIuz\n6UJKedSXX3JRSZEiwNVX23KJt97inXi/frac3jM0uadBzpwsQrdoEfvfbXH99WxWtGhh+bQvpTxt\n1izg4YfZX2KTmBhg/Hj2gFaqZNtlPEH73NPozBnOnKldG/jmGwcueOmSpTU0lPKkrVs5I6ZGDWDJ\nEtvqAAwdCgwZAmzYwHZUMLK0z11EGonIdhGJFpEkb2ZEpK6IrBORzSJix5CjJ+TJwxoU334LrF1r\n88V++IF7fe3aZfOFlHLRmTMsCJYzJwuC2ZTYT5/mrMqHHgrexJ4WqSZ3EckCYDyAxgCqAmgtIlUT\nHVMAwAQATY0x1QC0tCFWz+jenbvnjRhh84UqVuQbv0UL4Px5my+mlEu6d2cp3xkzLNngOjkTJnC/\nHIsrGHhWIC33WgCijTG7jTGXAMwE8FCiY9oA+MIY8wcAGGOOWBumt+TPDzzzDMd+LNwv4Erly7PA\n2Lp1WmBMha7ISNbWvvde2y5x+jQLhDVqFBpb6AUikOReCsCffl/v9z3mrxKAgiKyRETWiEj7pE4k\nIl1FJEpEoo4ePZq+iD2iVy920YwcafOFmjRhU2PqVJYJVipUnDrFf2+7ja13G40bxzI1Q4faehlP\nsWo6RjiAmwE0AdAQwCARuWIs2hgz2RgTYYyJKFq0qEWXdkehQmxMz54NbN5s88WGDmWr5qefbL6Q\nUg45fpyDp2PG2H6pU6d4mSZNbKli4FmBJPcDAPwr45f2PeZvP4CFxpizxphjAJYCqGlNiN71/PNs\nvQ8ZYvOFsmRh8aSPP7b5Qko5IC6OZQUOHgTuucf2y73zDvvaM1OrHQgsua8GUFFEKohINgCtAMxL\ndMxcAHVEJFxEcgG4FcBWa0P1nsKFgd69ue5o3TqbL5Y7N1fu7drFMpVaYEwFqxEjgIULmXUjUp3R\nlyEnT7JcU9OmwM0323opz0k1uRtjLgPoAWAhmLBnG2M2i0ikiET6jtkK4DsAGwCsAjDFGLPJvrC9\n47nnOHPG9tZ7gvnzWVzegdtZpSy3cCGb0O3bA1272n65t98GTpxw8PfTQ3QRkwVGjAAGDQJWr7a9\nIcIW+2OPAXPmsMDY3XfbfEGlLDRpEldg//ILN4u30d9/AxUqcD+cL76w9VKOCnQRkyZ3C5w6xTfR\nrbeyYe3IBW+5hfecv/8OlCjhwEWVskhsrCM7Ub/8MutArV/PsdtQoVUhHZQvHytGLljAndQdueCc\nOZy8G+rVj1RoGDiQe6ACjiT2mBh2ybRoEVqJPS00uVukRw9uqjRokEMXrF6dNRDGjnXogkql08yZ\nwKhRjk7lfeMNLu4ePNixS3qOJneL5M7NRvTixTZt5pGUunU5mnvhArBmjUMXVSoNtmwBnnwSuOMO\n7mvngKNHORGnZUu2gTIrTe4Wioxk9/fLLzs8U7FHD44a7dzp4EWVSkVCXaTcuVnO14HuGIA3CefO\nZb557YlpcrdQzpzAgAHcqWnRIgcvPGgQd/pt0YLvaqW84P33WRBs5kxbC4L527ePBcI6dgSqVHHk\nkp6lyd1iXboAZcsC/fsD8fEOXbRcOeDTT1nFrHt3XeCkvKFXL+DXXx1ZhZpg6FCu9cvMfe0JNLlb\nLHt2YNgwdoF//rmDF27cGHjpJW7pPnWqgxdWKpHff2cTOiyMu9o4ZMsW4KOP2EtZpkzqx4c6nedu\ng7g44IYbOM65ZYtjXY288FNPAd26ATfd5NBFlfJz9Cjfe8WLA6tWsRntkIcf5oSG3btZGiRU6Tx3\nF2XJAoweDURHO9yIzpIFmDLl38QeG+vgxVWml1AQ7OhR4L33HE3sK1dyf4U+fUI7saeFJnebNGkC\n1KnDPsCzZ10I4NlnuXWZYx3/KtMbNoxbQ777rqN3jsZwjKtoUb7tFWlyt4kIp/UeOsSVco67+mqu\nCHztNRcurjKdH37gWv8OHTiv3UGLFnF91KBBLMGtSPvcbdasGd94jvcDGgO0acPdRBYtcnTGgsqE\njh9ny330aNsLgvmLj+cGHMePA9u2cUJDqNM+d48YNYprOUaNcvjCIpxnXKkS0KoV8NdfDgegMoWL\nF4FLl9hyGTvW0cQOcEbamjX8u5IZEntaaHK3WdWqwBNPcA/HffscvniePCwwdvkyZy4oZbXnnmPZ\n6YsXHb/0xYvsa7/+et6kqv/S5O6AIUNcXFhRtSqwZw/7h5Sy0mefcTnoHXe40mweP57dna+/zoli\n6r80uTugTBngmWe4Bervv7sQQL58/Hf2bOCrr1wIQIWczZu5HPvOO9nP7rCYGI7fNmwINGjg+OWD\ngiZ3hwwYwG7J555zqTpAXBzw5pvc3mzHDhcCUCHj9GlOs82b19GCYP6GD+eeNa+/7vilg4Ymd4cU\nKMA570uWAPMSby/uhCxZ2HLPlk0LjKmM2b+fg6gzZ7qyC1h0NLtkOnXK3CV9U6NTIR10+TJ3hYmN\n5V1ttmwuBLFwIevQtGvHOjQOriJUIeTSJZfewGybfPcdk3zx4q6E4CqdCulB4eHcISah5eGKhg05\nsvvxxw7XJVZBb/lyru+PjXUtsS9bxglgL76YORN7WmjL3QWNGrEWRnS0S3Uw4uM5sNq8ubbcVWCO\nHGFJgezZObG8QAHHQzCGRSb37+ewUe7cjofgCdpy97DXX+dgkGs7xYSFsYSeCHdviolxKRAVFOLi\nOJH82DGuGnIhsQPs4l+1Chg5MvMm9rTQ5O6C6tWBrl05RXjbNhcDOXMGuP12zqDRAmMqOUOGsJbu\nhAnAjTe6EsLZs8ALL/DmoV07V0IIOprcXTJ0KFsfL7zgYhB58vAX99tvgVdecTEQ5Vl//snic506\n8cMlo0cDBw5w42tdsBQYTe4uKVYMGDgQ+OYbFtRzTbduQOvWLKm3eLGLgShPKlOGo5jjxrkWQsIq\n1LZtuRhWBUYHVF108SJQrRrXgKxf79oEBHbP1KrFPtXff3dsM2PlYRcvAr/8Atx3n9uRoHlzNoC2\nb9e3JqADqkEhe3YW0tu2jbebrkkoMNaoEVcdKtW7N9f1b9niahiLFnFi14ABmtjTSlvuHvDgg1y5\nun07ULKk29GAq63Cw92OQrnlk084yN63L3eccUls7L97EW/eDOTI4VoonqIt9yDy9tt8I/ft63Yk\n4NZRtWpxypvKfDZu5Cbrd9/NOYcumjCBNw5vvqmJPT0CSu4i0khEtotItIj0S+G4W0Tksoi0sC7E\n0HfNNUzs06cDS5e6HEyhQuwv6tRJC4xlNmfOsCBY/vycVO7i3dvRo1xI3aAB0LSpa2EEtVSTu4hk\nATAeQGMAVQG0FpGqyRz3KoDvrQ4yM+jXDyhbFujZk70irsmWjQXGsmfnL7oru3srV+TOzQUYs2a5\nvrb/hRf41nv7bV1EnV6BtNxrAYg2xuw2xlwCMBPAQ0kc1xPAHABHLIwv08iVC3jrLWDDBmDiRJeD\nKVOGGzFs3gxERrpUo1g56tw5ZtE+fYC77nI1lKVLgY8+YijXXedqKEEtkOReCsCffl/v9z32DxEp\nBaA5ALfTUlBr3py3oYMGsZSHq+rX5wKn337j7sMqdP36K1C+PLBihduR4NIl4OmngXLl+Hug0s+q\nAdW3AbxojElxDbuIdBWRKBGJOnr0qEWXDh0inBJ57hzw/PNuRwPgpZeAtWuBIkXcjkTZ5cgR4NFH\nuVtXlSpuR4O33uIg6rvvOr7XdsgJJLkfAFDG7+vSvsf8RQCYKSJ7AbQAMEFErti00xgz2RgTYYyJ\nKFq0aDpDDm2VK3PT308/9UBF3rAw/tJfuMCJxtqCDy1xcVydHBPDdQ4uFQRLsG8fMGwYt/t98EFX\nQwkJgST31QAqikgFEckGoBWA/+wlZIypYIwpb4wpD+BzAN2MMbpZZzr17w9UrMjb0/Pn3Y4GXGX1\n+uus2KQFxkLHyy8DP/7IQZ6aNd2OBs88w3/HjnU3jlCRanI3xlwG0APAQgBbAcw2xmwWkUgRibQ7\nwMwoRw5g0iTWe3d5qjHdcAOnLSxY4JGAVIbFxQGbNgFPPgk88YTb0WDuXG4/OWQIZ42pjNMVqh7W\noQMwYwbLvVSr5nIwxrDl/tln3Kqvfn2XA1IZFh/PebeuFTWis2eBqlXZA7h2rSv7bQcVXaEaAl5/\nnaVeIiM90BsiArz3Hn8LO3ViYSkVfC5cYGt93z6Oqbic2AGO2//xB3uHNLFbR5O7hxUtygS/bBkw\nbZrb0YCLXObMYSWn7NndjkalR69ewNSpXFDhAStWsI+9WzegTh23owkt2i3jccYAdevyd3HrVtcX\nDv7X2rXcGkcFh48/Zl9fv37c/cJlFy/y7XPqFNfL5cvndkTBQbtlQkRCb8j582zdeGax6BdfADff\nzKXqyvs2bmT/3j33AMOHux0NAP592bKFkwc0sVtPk3sQqFKF2/J9+SXLvnjCgw9y/9Unn3R5I1gV\nkOef5zz2GTM8Uc550yZg1Cjuu92kidvRhCbtlgkSly8zl+7Zw9aOJ9aAHTjADZOLFgVWruSmH8qb\njh/nfqg33OB2JIiL43Z5u3Z56L0cRLRbJsSEh3NQ9eRJoEcPt6PxKVWKLcFt21gD3DN9RuofP/3E\ngi2FC3sisQMsLbByJQdSNbHbR5N7EKlenYsKZ89ml7cn3HsvO09152Lv+eUXrkfwSB87wIV5Awey\nK6Z1a7ejCW3aLRNkYmOBW29lj8iWLWyQeUpcHJAli9tRqEOHOBUld24gKoobcLgsLo7VhLds4fhu\n6dJuRxSctFsmRGXNCnzwAWs99erldjSJfP01a5RogTF3Xb4MtGoFnDjBdQkeSOwA12wsXw6MG6eJ\n3Qma3INQzZq8tZ0+nb+7nlGiBLBzJ/D44x5YUpuJvfwy8PPPXPJZo4bb0QBgS/3ll7m5V5s2bkeT\nOWhyD1IDBwIREdwV7eBBt6PxiYhgQfrvvgNGjHA7msyraVOu6e/Qwe1IAHA8t107zsScOFG3zXOK\nJvcglTU17+2WAAAV7ElEQVQra76fP89SL56ZqNK1K3+ThwxhgTHlnIT60LVre2oQdfhwYP164P33\ndXaMkzS5B7HKlYExY9hQnjDB7Wh8RLjksFo11qBRzrhwgcVZhgxxO5L/WLWKk6k6dOANhXKOzpYJ\ncsYA99/PLta1az2xUxrFxAAFC+o9uFO6dAGmTAG++cYzSz7PnuWEnfPn2efukXHdoKezZTIJES5u\nypWLvSGxsW5H5FOoEIPbuRN44w23owltH3zAxD5ggGcSO8CdlXbuBD76SBO7GzS5h4ASJYDJkzmd\nedgwt6NJZNo0oE8fYOZMtyMJTevWsaLcvfd66oc/axZ/9AMGsFaZcp4m9xDx8MPcLW3UKGDJErej\n8TNsGFevPvkkV68oa61fzzrQn33mmcVje/dyXL12bWDwYLejyby0zz2EnDnDKrynT/N33jMzEw4c\nYOdr4cIcYdMCY9Y6fx7ImdPtKABw/dRdd7E++7p1QIUKbkcUerTPPRPKk4d1Z2JigPbtPbSOKKHA\n2Pbt2pSzyrhx3FUa8ExiB1ia+rff2E2oid1dmtxDTM2awNtvc3rk66+7HY2fevWAzz/nMkWVMUuX\nAr17c6GDhyxZAowcCXTsCDz2mNvRKO2WCUHGAI8+ys09li5lHXhPuXCBOyJXquR2JMHn4EF2ceXL\nB6xe7ZktjA4fZlh58gBr1mjPm520WyYTE+FqwLJlWVY1JsbtiBJ5/HHgvvuAY8fcjiS4JBQEO3WK\nRYU8ktgvX+b77O+/gf/9TxO7V2hyD1EFCnA62sGDvE32TP87wPlxR44AbduyDqwKzPTpvBV77z0W\n9/eIQYO4J8ikSZ6pU6agyT2k3XILyxPMm+eJze7/ddNN3I7n++89VQPF89q3Z72exx93O5J/zJsH\nvPIKpz62b+92NMqf9rmHOGOYC2bMAObPBxo1cjsiH2N4S/Hxxx4LzIN272Zfm8emn+zaxam3114L\nLFsG5MjhdkSZQ6B97u5vg65sldD/vnkz62hHRQFXX+12VGBgEyZwU9irrnI7Gu86f54r1M6eBbZu\n5Wa6HnD+PGuzh4VxEpQmdu/RbplMIFcu7rlqDPPEuXNuR+STKxen9Nx4I7/21MCAR3TvzhVpY8d6\nJrEbAzz9NMP69FOgfHm3I1JJ0eSeSVx9NbtmNmxgAUHP1H8H/s0WPXu6HYm3TJ3KomAvvcTSnx7x\n9tssBjZ4sKfCUolocs9EGjXi+OVnn7Eh6Bki3Mh5wgQGp7h2v3t3Thn1UI32hQtZB+7hh3U9mtdp\ncs9k+vcHmjcHnn+eq1g9Y/RobjbRpQsHCDK7cuU4Eu6hgmA7dnDlafXqbLmHafbwtIB+PCLSSES2\ni0i0iPRL4vm2IrJBRDaKyHIRqWl9qMoKYWGcoFKjBlexbtrkdkQ+WbNyYn7evBypO33a7YjcYQw3\nHS1YkDXaPVL97cQJ4MEH+WOaO1cXKgWDVJO7iGQBMB5AYwBVAbQWkaqJDtsD4G5jzPUAhgOYbHWg\nyjp58gBff81/H3yQ64k8oWRJ1n3/809gxQq3o3HHmDHAnXdyFpFHJCyM3b2bC2N1ADU4BNJyrwUg\n2hiz2xhzCcBMAA/5H2CMWW6M+dv35QoApa0NU1mtdGm2wA4dYjfNhQtuR+RTty4Lgtev73Ykzluy\nhP1m5ct7prSAMez6X7iQQyJ33eV2RCpQgST3UgD+9Pt6v++x5HQGsCCpJ0Skq4hEiUjU0aNHA49S\n2eKWW9hFs3w599LwzAyahK6ImTMZXGZw8CCbxxUrsjvGI3vPvvoqy/f278/hEBU8LB0SEZF7wOT+\nYlLPG2MmG2MijDERRT3Sl5jZtWzJMq3TpwMDB7odjZ8LFxjQo496qN/IJrGxHKk8fZr9Hnnzuh0R\nAI7l9u/PomAjRrgdjUqrQJL7AQBl/L4u7XvsP0SkBoApAB4yxhy3JjzlhP79gchITlh59123o/HJ\nkYNLH48d49LaUC4wtn8/sG8flxJXq+Z2NABYn6xjR+DuuznVXmfGBJ9AfmSrAVQUkQoikg1AKwDz\n/A8QkbIAvgDQzhizw/owlZ1EuLFPs2ZAr14s2+oJN94IjB8PLF7sqbnelqtQgfvLtmnjdiQAOBO1\nWTPgmmu4gDh7drcjUumRanI3xlwG0APAQgBbAcw2xmwWkUgRifQd9jKAwgAmiMg6EdGKYEEmSxbe\nht9+O6dXe2aT7c6dgU6d2C+wdq3b0Vhr507gxRc59TF3brejAcAZMfXr88Zp/nzOyFTBSatCqv+I\nieFMvP37meATyr646vx5Fsdp08YzA40Zdu4cULs2Nw9ftw4oUyb119js4EGuIztxgt0yHukhUono\nTkwqXQoV4srVAgXYgvPEYtGcObmxhwgQHQ1cvOh2RBljDNCtG1eQTZ/uicQeEwM0aMCx6wULNLGH\nAk3u6gplyrCbO1s24N57uezcEw4fZgHx3r3djiRjpkzh+v1BgzxRx/7MGRYA27mTax9q1XI7ImUF\nTe4qSddeywQfH88Ev3ev2xGBdd8jI7mf26efuh1N+vz9Nwv7NGjgicpbZ88CTZqwzv+sWUC9em5H\npKyiyV0l67rrgB9+YMuuXj32w7tu5EjOz+va1UOFcdKgYEFuL/jpp64XBDt7li32ZcsYzkMPpf4a\nFTw0uasU1azJpefHjjGn7tvnckDh4Vy5mj8/C4ydOuVyQAGKj2cWBTiQ6vIiPv/EPn06F8eq0KLJ\nXaWqVi224I8fZ22RXbtcDqh4cfYh3Hxz8MyeefVVTkP69Ve3I9HEnkloclcBufVW4Mcf2UVz993A\n9u0uB3TXXZyYnzevh4riJOPHH7mbUqtWXEjgohMnOIariT30aXJXAbvpJs59j41lgvfENMk//uCt\nhQdaxEk6cIAZtHJllhdw8U7j8GEW3Vy5kj1bmthDmyZ3lSbXXw/8/DNrjdx5pweKNubLxxkoXiww\nllAQ7Nw5FgRzcYeLvXu5QGnnTtbyb9nStVCUQzS5qzSrUoUN5SJFOE1y7lwXgylQgIkzJoblC71U\nYCw8nIO+U6Zw6pFLNm8G7riDg+KLFgENG7oWinKQJneVLhUqMMHXqMHNkt97z8VgatbkThI//uiJ\nueMAWC9GBHj2WVf7P37+mXdY8fEsKXDbba6FohymyV2lW9GizKeNGnFt0aBBLo5tduzIImP/+x+n\ng7hpxw5uuvHTT66G8fHHLCFx1VX8Q3z99a6GoxymyV1lSO7cwFdf/Vu4MaGL2RXvvgusXu1uhcWz\nZ9kVc/Ysa+a6wBjewHTo8O+4yNVXuxKKcpEmd5VhWbOyW/nVV7m/Rp063OPacTlzcnHThQtcyer0\nxrDGAE8/zU7uzz4DypZ19vpgAc22bYHhw3kj8913WrY3s9LkriwhAvTty5kY0dHcn/W331wKZtky\nzivv1cvZ606eDHzyCTcWadDA2WsD2LOHA6czZgCvvMKZl1mzOh6G8ghN7spSTZoAK1Zw1l/dusDE\niS70w993HzfBmDyZHc9OMObfAYiXXnLmmn4WLOCC3T17gG++4X8/WBbvKntocleWq1oVWLWKxca6\ndeNkEcdLwIwYwb8ukZHAxo32X0+EK4M+/9zRDUfj43mj0KQJe4Giovi5UprclS0KFQK+/ZbdA3Pm\nsFX5++8OBpBQYKxAAXZCx8fbc534ePZH7d3LBO/gYO5ff3HO+tChQLt2HDh1aQxXeZAmd2WbsDB2\nDyxZwoG+2rWBsWPty7NXuOoq/mX58EP7WtOjRwNjxnDk0kFffMGpjcuXs/fpww+BXLkcDUF5nCZ3\nZbs6dbhNaP363ETJ0c0/bruNRXEA66udLV7MOYdt2gBPPWXtuZNx5gzw5JOcbXn11bwb6tJF+9fV\nlTS5K0cUKcKZNFOmAGvWsNU5ebKDg63TpnFj0F9+seZ8Bw6w3EGVKvyPOJBdFy3iYtxp04ABA9hq\nr1TJ9suqIKXJXTlGhHOvN25kCeGnnmKf8c6dDlz8kUdYM+Gxx4BDhzJ+vj592Nc0Z47t/ezHj3MB\nbv36HEpYupTT+HWao0qJJnfluHLluNPc+PEsP1u9OjB4MHOlbfLnZyI+cYIt7suXM3a+8eM557BK\nFWviS0J8PGdyVq3KbfAGDgTWr2c3l1Kp0eSuXBEWxmmS27YBLVoAw4YxyX/1lY1dNTVqcOL9kiUs\nhJMea9awKFihQixqb5MVKzhc0KEDUL48pziOGAHkyGHbJVWI0eSuXFWiBHcEWrwYyJYNaN6cmyyt\nWGHTBTt0AHr0YMZMq+3bOXe+d2+ro/rH3r2c1njbbSzh8NFHXOlbs6Ztl1QhSpO78oR69dgXP2kS\n++Bvu40telvWH7377r+zWwK9TUgoCJYjB0czLXbgAO9kKlViYcv+/fm3pH17R9dEqRCibxvlGeHh\nzLnR0Vx1uXAhe1KaN2e3hOXmzGHXSmoFxowBunYFtmxhQbDSpS0LYc8e4JlnuPhoyhROa9y1Cxg1\nitvDKpVemtyV5+TJwwHWffv475IlLETWoAGnU1q22VKOHJwa2bNnysdNmsSkPnQop6xYYNUqTty5\n9loOA7RtyzLw48cDpUpZcgmVyWlyV55VqBBb8Pv2cSHoli1A06ZMiGPGWLBlapMm7GKZMgX44IPk\nj7v5ZjapBw7M0OVOneKU+Ftv5cfChZxRuWcPMHVq+oYBlEqOGJe2zomIiDBRttxrq1AVG8v9WseN\n4/ZxWbKwNf/448BDD6VzunlcHE+yfDlHLm+44b8XzOBk8kuXWCxy1ixg9mxuZFKtGnt5OnbUrheV\ndiKyxhgTkdpxAbXcRaSRiGwXkWgR6ZfE8yIi7/ie3yAiN6UnaKVSkjUrB1mXLAE2bQJeeIH/tm0L\nFCsGNGvGRvjBg2k4aZYsLIBeqBC7XhLEx/MvxosvpjnOEyeAL78EnniC5W0aN2b3fps2nAW0cSP7\n2TWxKzuFp3aAiGQBMB5AfQD7AawWkXnGmC1+hzUGUNH3cSuAib5/lbJFtWrsqhk5kt3ms2axCuXc\nuXy+Rg1uMXfHHVz0U7p0ChUCihXjiG3x4v8+NnIki6Q/+GCqsRw+zJcvX84pnatX829D/vz8+9Ci\nBbvqdY66clKq3TIichuAIcaYhr6v+wOAMWa03zHvAVhijJnh+3o7gLrGmGTbUNoto6xmDFvy33zD\nval/+42FtgDWtqlenR/XXcfa52XKMOkXKuSX+HfsYN96QlP7k08QbwQnTgDHjnHueXQ0P3bsANau\nBfbv50uzZGFf+r33cr+Q2rU5d18pKwXaLZNqyx1AKQD+O2Lux5Wt8qSOKQUgLTfISmWICAuSXX89\n54lfvgxs2AD8+iv/3bSJpXETEn6CsDD21+fJA+Q+kw84PQqXs76Fy0tK4kIxQUzMlWWKs2fn9MU7\n7wQiIjib58YbeQ6lvCCQ5G4ZEekKoCsAlHVh82CVuYSHs9rvTX4jQMZwk4v9+/nx558szHXmDD/O\nni4GOS4IL1oQ4TnDkC0bW/0JHyVLcrZOqVK6uEh5WyDJ/QCAMn5fl/Y9ltZjYIyZDGAywG6ZNEWq\nlAVEmJhLlWIXypXCAFzlcFRKWS+QtsdqABVFpIKIZAPQCsC8RMfMA9DeN2umNoCTKfW3K6WUsleq\nLXdjzGUR6QFgIYAsAKYZYzaLSKTv+UkA5gO4H0A0gHMAOtoXslJKqdQE1OdujJkPJnD/xyb5fW4A\ndLc2NKWUUumlQ0JKKRWCNLkrpVQI0uSulFIhSJO7UkqFIE3uSikVglwr+SsiRwHsS+fLiwA4ZmE4\nVvFqXIB3Y9O40kbjSptQjKucMaZoage5ltwzQkSiAimc4zSvxgV4NzaNK200rrTJzHFpt4xSSoUg\nTe5KKRWCgjW5T3Y7gGR4NS7Au7FpXGmjcaVNpo0rKPvclVJKpSxYW+5KKaVS4NnkLiItRWSziMSL\nSLKjyslt3i0ihUTkBxHZ6fu3oEVxpXpeEaksIuv8Pk6JSG/fc0NE5IDfc/c7FZfvuL0istF37ai0\nvt6OuESkjIj8JCJbfD/zXn7PWfr9yshm76m91ua42vri2Sgiy0Wkpt9zSf5MHYqrroic9Pv5vBzo\na22O6wW/mDaJSJyIFPI9Z+f3a5qIHBGRTck879z7yxjjyQ8A1wGoDGAJgIhkjskCYBeAqwFkA7Ae\nQFXfc68B6Of7vB+AVy2KK03n9cV4CJybCgBDAPSx4fsVUFwA9gIoktH/l5VxASgB4Cbf53kB7PD7\nOVr2/Urp/eJ3zP0AFgAQALUBrAz0tTbHdTuAgr7PGyfEldLP1KG46gL4Jj2vtTOuRMc/COBHu79f\nvnPfBeAmAJuSed6x95dnW+7GmK3GmO2pHFYLQLQxZrcx5hKAmQAe8j33EICPfJ9/BKCZRaGl9bz3\nAthljEnvgq1AZfT/69r3yxhz0Biz1vf5aQBbwT14rZbS+8U/3o8NrQBQQERKBPha2+Iyxiw3xvzt\n+3IFuNuZ3TLyf3b1+5VIawAzLLp2iowxSwHEpHCIY+8vzyb3ACW3MTcAXGX+3Q3qEKzbOy2t522F\nK99YPX23ZNOs6v5IQ1wGwCIRWSPc0zatr7crLgCAiJQHcCOAlX4PW/X9Sun9ktoxgbzWzrj8dQZb\nfwmS+5k6Fdftvp/PAhGplsbX2hkXRCQXgEYA5vg9bNf3KxCOvb8c3SA7MRFZBKB4Ek8NNMbMteo6\nxhgjIgFPC0oprrScV7gtYVMA/f0enghgOPgGGw7gDQCdHIyrjjHmgIgUA/CDiGzztTYCfb1dcUFE\n8oC/hL2NMad8D6f7+xWKROQeMLnX8Xs41Z+pjdYCKGuMOeMbD/kKQEWHrh2IBwH8aozxb027+f1y\njKvJ3RhzXwZPkdLG3IdFpIQx5qDvtueIFXGJSFrO2xjAWmPMYb9z//O5iLwP4Bsn4zLGHPD9e0RE\nvgRvB5fC5e+XiGQFE/t0Y8wXfudO9/crCRnZ7D1rAK+1My6ISA0AUwA0NsYcT3g8hZ+p7XH5/RGG\nMWa+iEwQkSKBvNbOuPxcceds4/crEI69v4K9WyalzbvnAejg+7wDAKvuBNJy3iv6+nwJLkFzAEmO\nqtsRl4jkFpG8CZ8DaOB3fde+XyIiAKYC2GqMeTPRc1Z+vzKy2Xsgr7UtLhEpC+ALAO2MMTv8Hk/p\nZ+pEXMV9Pz+ISC0wpxwP5LV2xuWLJz+Au+H3nrP5+xUI595fdowYW/EB/iLvB3ARwGEAC32PlwQw\n3++4+8HZFbvA7pyExwsDWAxgJ4BFAApZFFeS500irtzgmzx/otd/AmAjgA2+H14Jp+ICR+LX+z42\ne+X7BXYxGN/3ZJ3v4347vl9JvV8ARAKI9H0uAMb7nt8Iv5layb3XLPo+pRbXFAB/+31/olL7mToU\nVw/fddeDA723e+H75fv6CQAzE73O7u/XDAAHAcSC+auzW+8vXaGqlFIhKNi7ZZRSSiVBk7tSSoUg\nTe5KKRWCNLkrpVQI0uSulFIhSJO7UkqFIE3uSikVgjS5K6VUCPo/y06ZR9CChOQAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12e2abc67f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_array = sess.run(x_vals)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.plot(x_array, l1_y_out, 'r--', label='L1 Loss')\n",
    "plt.plot(x_array, l2_y_out, 'b-', label='L2 Loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Back Propogation\n",
    "* It can keep track of operations & automatically update model variables based on back propogation\n",
    "* Minimize loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_vals = np.random.normal(1, 0.1, 100)\n",
    "y_vals = np.repeat(10., 100)\n",
    "x_data = tf.placeholder(shape=[1], dtype=tf.float32)\n",
    "y_target = tf.placeholder(shape=[1], dtype=tf.float32)\n",
    "A = tf.Variable(tf.random_normal(shape=[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,  10.,\n",
       "        10.])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.96727524,  1.04696738,  1.04569504,  0.90581905,  1.02735944,\n",
       "        0.92236171,  1.07339619,  0.80527031,  0.89690801,  0.9187634 ,\n",
       "        0.99167368,  1.21924735,  1.06508388,  0.91346623,  0.87038911,\n",
       "        1.00209266,  1.0988418 ,  0.9564379 ,  1.08451251,  1.14720616,\n",
       "        0.97421059,  0.75005957,  1.00514868,  0.88947382,  0.92551077,\n",
       "        1.02922274,  0.80994198,  0.93239088,  0.95713069,  1.06055085,\n",
       "        1.00317452,  0.98869983,  0.83858811,  1.08068019,  1.0897093 ,\n",
       "        1.04847203,  1.14777602,  0.99948277,  0.97286985,  1.02884656,\n",
       "        0.78426923,  1.0099864 ,  0.92375207,  0.86365626,  0.98346889,\n",
       "        1.06127619,  1.0180148 ,  1.06508045,  0.91185556,  1.10779932,\n",
       "        0.94185985,  0.91680898,  0.94315404,  1.00096129,  1.05616261,\n",
       "        1.01207684,  0.80418888,  0.96706901,  1.24391973,  0.81147358,\n",
       "        1.08656906,  0.93477843,  0.96271066,  0.98447399,  0.9583596 ,\n",
       "        0.94378991,  0.96300808,  1.02769399,  1.08669075,  0.99506158,\n",
       "        0.959161  ,  0.95122205,  0.86644056,  0.85841082,  1.07295983,\n",
       "        1.00563485,  1.03276571,  1.02331752,  0.84562402,  1.15385203,\n",
       "        0.89268452,  1.14016902,  0.94465678,  0.93136944,  1.24489321,\n",
       "        0.78892832,  1.04107605,  0.78536649,  0.93555014,  0.92780373,\n",
       "        1.05578411,  0.86902438,  1.15236419,  1.02327726,  1.08611432,\n",
       "        1.06768882,  0.85840625,  0.98598138,  1.05564931,  0.93265112])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_output = tf.mul(x_data, A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.square(my_output - y_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_opt = tf.train.GradientDescentOptimizer(learning_rate=0.02)\n",
    "train_step = my_opt.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss [ 72.73432922]\n",
      "Loss [ 64.07601166]\n",
      "Loss [ 62.92261124]\n",
      "Loss [ 54.55478668]\n",
      "Loss [ 51.98026276]\n",
      "Loss [ 47.87427139]\n",
      "Loss [ 38.76089478]\n",
      "Loss [ 51.03902435]\n",
      "Loss [ 48.17140579]\n",
      "Loss [ 37.31684113]\n",
      "Loss [ 40.37000275]\n",
      "Loss [ 32.57803726]\n",
      "Loss [ 25.75200272]\n",
      "Loss [ 36.30051804]\n",
      "Loss [ 21.6240921]\n",
      "Loss [ 29.28386688]\n",
      "Loss [ 32.52169037]\n",
      "Loss [ 17.96669006]\n",
      "Loss [ 13.47986507]\n",
      "Loss [ 12.76614761]\n",
      "Loss [ 19.17869949]\n",
      "Loss [ 20.2830162]\n",
      "Loss [ 14.81246662]\n",
      "Loss [ 13.37816238]\n",
      "Loss [ 14.1294775]\n",
      "Loss [ 7.3286171]\n",
      "Loss [ 11.84623051]\n",
      "Loss [ 4.77955675]\n",
      "Loss [ 6.64821625]\n",
      "Loss [ 6.661201]\n",
      "Loss [ 3.77145076]\n",
      "Loss [ 7.11289835]\n",
      "Loss [ 11.79492664]\n",
      "Loss [ 10.70361805]\n",
      "Loss [ 6.14356899]\n",
      "Loss [ 6.20595598]\n",
      "Loss [ 6.22654009]\n",
      "Loss [ 3.49200058]\n",
      "Loss [ 1.3263427]\n",
      "Loss [ 5.53844547]\n",
      "Loss [ 0.94972771]\n",
      "Loss [ 2.33993626]\n",
      "Loss [ 4.78512096]\n",
      "Loss [ 0.0718959]\n",
      "Loss [ 10.40126133]\n",
      "Loss [ 6.16333675]\n",
      "Loss [ 2.61272669]\n",
      "Loss [ 0.00093092]\n",
      "Loss [ 3.50964808]\n",
      "Loss [ 7.70241213]\n",
      "Loss [ 0.28372109]\n",
      "Loss [ 0.25876635]\n",
      "Loss [ 0.52568227]\n",
      "Loss [ 1.83016694]\n",
      "Loss [ 0.45572698]\n",
      "Loss [ 0.48276213]\n",
      "Loss [ 0.44316658]\n",
      "Loss [ 0.40726978]\n",
      "Loss [ 1.73382187]\n",
      "Loss [ 9.1772871]\n",
      "Loss [ 0.60822999]\n",
      "Loss [ 0.42535207]\n",
      "Loss [ 0.33060503]\n",
      "Loss [ 2.36780334]\n",
      "Loss [ 2.42730927]\n",
      "Loss [ 1.18498182]\n",
      "Loss [ 2.84865737]\n",
      "Loss [ 0.00092958]\n",
      "Loss [ 2.93753028]\n",
      "Loss [ 0.54700136]\n",
      "Loss [ 0.04498966]\n",
      "Loss [ 0.86139899]\n",
      "Loss [ 0.42659196]\n",
      "Loss [ 0.15547812]\n",
      "Loss [ 0.87274265]\n",
      "Loss [ 0.49202991]\n",
      "Loss [ 1.5993309]\n",
      "Loss [ 0.44460636]\n",
      "Loss [ 0.05165171]\n",
      "Loss [ 3.90794539]\n",
      "Loss [ 3.76253271]\n",
      "Loss [ 0.00876387]\n",
      "Loss [ 0.67963058]\n",
      "Loss [ 0.23244938]\n",
      "Loss [ 3.62066817]\n",
      "Loss [ 0.72360188]\n",
      "Loss [ 2.06664491]\n",
      "Loss [ 0.01594828]\n",
      "Loss [ 0.15647924]\n",
      "Loss [ 0.0492121]\n",
      "Loss [ 0.09551284]\n",
      "Loss [ 0.00489422]\n",
      "Loss [ 0.11933664]\n",
      "Loss [ 0.71467352]\n",
      "Loss [ 0.08831209]\n",
      "Loss [ 0.45770946]\n",
      "Loss [ 0.00451208]\n",
      "Loss [ 0.65844494]\n",
      "Loss [ 0.44617462]\n",
      "Loss [ 0.32031819]\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    rand_index = np.random.choice(100)\n",
    "    rand_x = [x_vals[rand_index]]\n",
    "    rand_y = [y_vals[rand_index]]\n",
    "    sess.run(train_step, feed_dict={x_data:rand_x, y_target:rand_y})\n",
    "    print('Loss ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y})))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}