Hackers Guide to Neural Networks notes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://karpathy.github.io/neuralnets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup imports\n",
    "\n",
    "import random\n",
    "import math\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backprop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Local Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forwardMultiplyGate(x,y):\n",
    "    return x*y\n",
    "\n",
    "x = -2; y = 3\n",
    "\n",
    "tweak = 0.01\n",
    "best_out = -float('inf')\n",
    "best_x = x\n",
    "best_y = y\n",
    "\n",
    "count = 0\n",
    "while (count < 100):\n",
    "    x_try = x + tweak * (random.random() * 2 - 1)\n",
    "    y_try = y + tweak * (random.random() * 2 - 1)\n",
    "    score = forwardMultiplyGate(x_try, y_try)\n",
    "    if score > best_out:\n",
    "        best_out = score\n",
    "        best_x = x_try\n",
    "        best_y = y_try\n",
    "    count+=1\n",
    "\n",
    "print(best_out, best_x, best_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numerical Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd0VHX+//HnO40WeglFOkivQTpB\nlK4CKijoYkUsoBTdVVfXddXd1d39hmLBhnXV2BUpUhQTuhTpHUQBqaJABAKBz++PDHvyi4FMwkzu\nZPJ6nHOPd+793JsXd67vubmZeY855xARkfAS4XUAEREJPBV3EZEwpOIuIhKGVNxFRMKQiruISBhS\ncRcRCUMq7iIiYUjFXUQkDKm4i4iEoSivfnCFChVcrVq18rTtb7/9RokSJQIbKABCNReEbjblyh3l\nyp1wzLV8+fKDzrmKOQ50znkyxcfHu7yaO3dunrcNplDN5VzoZlOu3FGu3AnHXMAy50eN1W0ZEZEw\npOIuIhKGVNxFRMKQiruISBhScRcRCUM5FnczK2pm35rZKjNbZ2Z/y2ZMETN738y2mtkSM6sVjLAi\nIuIff67c04DLnHMtgJZAbzNrn2XM7cAvzrl6wDjgmcDGFBGR3MixuPveWpnqexjtm7J+N19/4E3f\n/EfA5WZmAUuZyc+paby7IY0jJ04FY/ciImHBnB/foWpmkcByoB7wvHPuwSzr1wK9nXO7fI+3Ae2c\ncwezjBsODAeIi4uLT0pKynXgxXvSeWnVCUoXieDmJjG0quTZh2x/JzU1ldjYWK9jZCtUsylX7ihX\n7oRjrm7dui13zrXJcaA/n3Q6OwFlgLlA0yzL1wEXZXq8DSh/vn1dyCdUJ382x/Ual+xqPjjV3fvu\nCnfw6Ik87yuQQvXTcM6Fbjblyh3lyp1wzEUwPqHqnPsV+AbonWXVLqA6gJlFAaWBQ7nZd27UKR3J\nlJGdGdP9Ymas3UP3xGQ+X7n77AuLiEih58+7ZSqaWRnffDGgO7Axy7ApwM2++YHA1y7IlTYmKoJR\n3esz7b4u1CxfglFJK7n9zWX89OvxYP5YEZECwZ8r9yrAXDNbDSwFZjvnpprZE2bWzzdmMlDezLYC\nY4GHghP39y6OK8nHd3fk0SsasXDbQXqOS+GdJT9w5oyu4kWk8Mrxr5HOudVAq2yWP5Zp/gQwKLDR\n/BcZYQzrUoeejSvz0CereeTTtUxZ+RNPX9uc2hVCr92niEiwhdUnVGuUL847w9rxzLXNWL/nCL3H\np/ByyjbST5/xOpqISL4Kq+IOYGZcf0kN5oztSsLFFfnH9I1cM2khG/Yc8TqaiEi+CbviflZcqaK8\nPDSe525oxe5fjnPVs/NJnLWJtPTTXkcTEQm6sC3ukHEVf2XzqswZ25WrWlRl4tdbuXLifFb8+IvX\n0UREgiqsi/tZZUvEMO76lrx+yyWkpqVz7aSFPPHFeo6dTPc6mohIUBSK4n5Wt4aVmDUmgRvb1eC1\nBd/Ta3wKC7YezHlDEZECplAVd4CSRaN5akAz3h/enqiICG58dQkPfrSaw8fViExEwkehK+5ntatT\nnhmjunBX17p8tGIXPRKTmblur9exREQCotAWd4Ci0ZE81Kchn93TifKxRbjz7eWMeGcFB46meR1N\nROSCFOriflazi0ozZWQnHuh5MbPX76PHuGQ+WbFLjchEpMBScfeJjoxg5GX1mT6qM3UqlGDsB6u4\n9Y2l7FYjMhEpgFTcs6hXqSQf3tWRv17VmCXbD9EzMZm3F+1QIzIRKVBU3LMRGWHc2qk2s8Yk0Lpm\nWf7y+ToGv7yY7QdSc95YRCQEqLifR/VyxXnrtrb8e2BzNu49Qu8J85j0jRqRiUjoU3HPgZkxqE11\n5oztSrcGFXnmy40MeGEB63467HU0EZFzUnH3U6VSRXlpaBsm3diavYfT6PfcAv49cyMnTqkRmYiE\nHhX3XOrTrApzxiYwoGU1np+7jSsmzmP5D0H7ulgRkTxRcc+DMsVj+L/rWvDmbW05ceoMA19cxONT\n1nEiXe+oEZHQoOJ+AbpeXJGZYxK4qX1N3ly0g0fmHydl8wGvY4mIqLhfqNgiUfytf1M+uLMD0ZFw\n02vf8sCHq/j12Emvo4lIIabiHiCX1CrHEx2Lcc+ldfn0u910T0xhxpo9XscSkUJKxT2AYiKNP/Vu\nyOcjOlGpZBHufmcFd/93OfuPnvA6mogUMiruQdC0Wmk+H9mJP/ZqwFcb99MjMYUPl+1UIzIRyTcq\n7kESHRnBiG71mH5fF+pXiuWPH63mpte+ZeehY15HE5FCIMfibmbVzWyumW0ws3VmNiqbMZea2WEz\nW+mbHgtO3IKnXqVYPrizA0/0b8KKH36h1/gU3ljwvRqRiUhQ+XPlng7c75xrBLQHRphZ42zGzXPO\ntfRNTwQ0ZQEXEWHc1KEWM8ck0KZWOR7/Yj3XvbSIrfvViExEgiPH4u6c2+OcW+GbPwpsAKoFO1g4\nuqhscd689RL+b1ALtuxPpe+EeTw/dyun1IhMRALMcvNHPjOrBaQATZ1zRzItvxT4GNgF/AQ84Jxb\nl832w4HhAHFxcfFJSUl5Cp2amkpsbGyetg2m3OQ6nOZ4e30ay/adpkbJCG5rGkOt0pEhkS0/KVfu\nKFfuhGOubt26LXfOtclxoHPOrwmIBZYD12SzrhQQ65vvC2zJaX/x8fEur+bOnZvnbYMpL7lmrNnj\n2jw129V5eJp7esYGd/xkeuCDufA6ZvlBuXJHuXLnQnIBy5wfNduvd8uYWTQZV+bvOOc+yeYF4ohz\nLtU3Px2INrMK/uy7sOvdtDJzxnTl2tbVmPTNNvpOmMfSHWpEJiIXxp93yxgwGdjgnEs8x5jKvnGY\nWVvffn8OZNBwVrp4NP8a2IL/3t6Ok6fPMOjFRTz2+VpS09K9jiYiBVSUH2M6AUOBNWa20rfsz0AN\nAOfci8BA4G4zSweOA4N9vz5ILnSuX4GZoxP4z6xNvLFwB3PW7+Pv1zSjW4NKXkcTkQImx+LunJsP\nWA5jngOeC1SowqxEkSj+elUTrmxelQc/Xs2try/lmlbV+MuVjSlbIsbreCJSQOgTqiEqvmZZpt3X\nmXsvq8eUVT/RY1wy01bvUQsDEfGLinsIKxIVyf09GzBlZGeqlC7GiHdXcOfby9l/RI3IROT8VNwL\ngMZVS/HpPR15uE9Dkjcf4PLEZD5YqkZkInJuKu4FRFRkBHd2rcuMUV1oVKUUf/p4NUMnf8uPP6sR\nmYj8nop7AVOnYixJd7TnqQFNWbnzV3qNT2Hy/O85rUZkIpKJinsBFBFh/KF9TWaNSaBdnXI8OXU9\nA19cyJZ9R72OJiIhQsW9AKtaphiv33IJ469vyY6Dv3HFxPlM/GoLJ9PViEyksFNxL+DMjAGtqjF7\nbFd6Na1M4uzN9HtuPqt3/ep1NBHxkIp7mKgQW4Rnh7TilZva8Muxkwx4fgH/nL6BE6dOex1NRDzg\nT/sBKUB6NI6jbe1yPD1jAy+lbGfmur08fW1zr2OJSD7TlXsYKl0smn9e05x3h7XjjIPBLy/mzXVp\nHD1xyutoIpJPVNzDWMd6FfhydBeGda7NNzvT6Tkuha837vM6lojkAxX3MFc8JopHr2zMo+2LUrJo\nFLe9sYzRSd9x6LeTXkcTkSBScS8k6paJZOq9XRh1eX2mrdlD98Rkpqz6SS0MRMKUinshEhMVwZge\nF/PFvZ2pXrYY9733HXe8tZy9h9WITCTcqLgXQg0rl+KTezrxSN9GzN96gB6Jybz37Y+6ihcJIyru\nhVRkhHFHQh2+HJVAk2qlePiTNdzwyhJ++Pk3r6OJSACouBdytSqU4N1h7fnH1c1Yu/swvcan8Oq8\n7WpEJlLAqbgLERHGDe1qMGtsAp3qVuCpaRu4ZtJCNu1VIzKRgkrFXf6nSulivHpzGyYOacXOQ8e4\n8tl5jJ+zWY3IRAogFXf5/5gZ/VpUZc7YrvRtVoXxc7Zw1bPzWblTjchEChIVd8lWuRIxTBjcisk3\nt+Hw8VNc88ICnpq6nuMn1YhMpCBQcZfzurxRHLPGJjC4bQ1enf89vcansHDbQa9jiUgOcizuZlbd\nzOaa2QYzW2dmo7IZY2Y20cy2mtlqM2sdnLjihVJFo/nH1c147472RBjc8MoSHv5kNUfUiEwkZPlz\n5Z4O3O+cawS0B0aYWeMsY/oA9X3TcGBSQFNKSOhQtzwzRiVwZ0Id3l+6kx6JycxZr0ZkIqEox+Lu\nnNvjnFvhmz8KbACqZRnWH3jLZVgMlDGzKgFPK54rFhPJw30b8dmITpQtHsOwt5Zx73vf8XNqmtfR\nRCSTXN1zN7NaQCtgSZZV1YCdmR7v4vcvABJGml9UhikjOzO2x8V8uTajEdln3+1WCwOREGH+/s9o\nZrFAMvB359wnWdZNA/7pnJvve/wV8Cfn3PIs44aTcduGuLi4+KSkpDyFTk1NJTY2Nk/bBlOo5oLg\nZtt99AyT16ax/fAZWlSM5KbGMZQv5t91Q6geM+XKHeXKnQvJ1a1bt+XOuTY5DnTO5TgB0cBMYOw5\n1r8EDMn0eBNQ5Xz7jI+Pd3k1d+7cPG8bTKGay7ngZ0s/fca9Om+7a/joDNfksS/d24t2uNOnz3ie\nK6+UK3eUK3cuJBewzPlRt/15t4wBk4ENzrnEcwybAtzke9dMe+Cwc25Pjq8sEjYiI4zbO9dm5ugE\nWlQvzaOfrWXIK4v5/qAakYl4wZ/fnTsBQ4HLzGylb+prZneZ2V2+MdOB7cBW4BXgnuDElVBXo3xx\n/nt7O/51bXPW7zlC7/EpvJS8jfTTamEgkp+ichrgMu6jWw5jHDAiUKGkYDMzrrukOl0bVOTRz9by\nzxkbmbZmD89c25xGVUp5HU+kUNAnVCVo4koV5eWh8Tx/Q2t++vU4Vz07n8RZm0hLVwsDkWBTcZeg\nMjOuaF6F2WO60q9FVSZ+vZUrJs5n+Q+/eB1NJKypuEu+KFsihsTrW/L6rZdwLC2dgS8u5G9frCMt\nXe+LFwkGFXfJV90aVGLW2K4MbV+T1xfs4JEFx5m/RY3IRAJNxV3yXWyRKJ7o35QP7uxApMEfJi/h\nTx+t4vBxNSITCRQVd/FM29rleLJTMe6+tC4fr9hNj8RkZq7b63UskbCg4i6eiok0HuzdkM/u6UT5\n2CLc+fZyRryzggNH1YhM5EKouEtIaHZRaaaM7MQfezVg9vp9dE9M5uPlu9SITCSPVNwlZERHRjCi\nWz2mj+pMvUqx3P/hKm55fSm7fz3udTSRAkfFXUJOvUol+fDODjx+VWOW7jhEz8Rk3lq0gzNndBUv\n4i8VdwlJERHGLZ0yGpG1rlmWxz5fx/UvL2LbgVSvo4kUCCruEtKqlyvOW7e15d8Dm7Np71H6TJjH\nC99s5ZQakYmcl4q7hDwzY1Cb6sy5vyuXNajEv77cxIDnF7B292Gvo4mELBV3KTAqlSzKi0PjmXRj\na/YdSaP/8wv498yNnDilRmQiWam4S4HTp1kV5oxN4OpW1Xh+7jb6TpzHsh2HvI4lElJU3KVAKlM8\nhv8MasFbt7Ul7dQZBr20iMenrOO3tHSvo4mEBBV3KdASLq7IrDEJ3NyhFm8u2kHPcSmkbD7gdSwR\nz6m4S4FXokgUj/drwod3dqBIdAQ3vfYtD3y4il+PnfQ6mohnVNwlbLSpVY7p93VhRLe6fPrdbron\npjBjjb6nXQonFXcJK0WjI/ljr4ZMGdmJuFJFuPudFdz19nL2HznhdTSRfKXiLmGpSdXSfD6iEw/2\nbsjXm/bTPTGZD5ftVCMyKTRU3CVsRUVGcPeldZkxqgsNKpfkjx+t5qbXvmXnoWNeRxMJOhV3CXt1\nK8by/vAOPNm/CSt++IVe41N4Y8H3akQmYU3FXQqFiAhjaIdazByTwCW1yvH4F+sZ9NIitu4/6nU0\nkaDIsbib2Wtmtt/M1p5j/aVmdtjMVvqmxwIfUyQwLipbnDduvYTE61qw7UAqfSfM57mvt6gRmYQd\nf67c3wB65zBmnnOupW964sJjiQSPmXFN64uYPaYrPZrE8Z9Zm+n3nBqRSXjJsbg751IANe6QsFOx\nZBGev6E1Lw2N52BqRiOyp2eoEZmEh0Ddc+9gZqvMbIaZNQnQPkXyRa8mlZkzpisDW1/Ei8nb6Dth\nHpsOqcBLwWb+vO/XzGoBU51zTbNZVwo445xLNbO+wATnXP1z7Gc4MBwgLi4uPikpKU+hU1NTiY2N\nzdO2wRSquSB0s4VarnUHT/P6ujQOHndcViOKQRfHUCzKvI71P6F2vM5Srty5kFzdunVb7pxrk+NA\n51yOE1ALWOvn2B1AhZzGxcfHu7yaO3dunrcNplDN5VzoZgvFXL+lnXJ3vPClq/XQVNfhH3Pc1xv3\neR3pf0LxeDmnXLl1IbmAZc6PWnzBt2XMrLKZmW++LRm3en6+0P2KeKV4TBQ3NCrCR3d1pESRKG59\nfSlj31/JL7+pEZkUHP68FfI9YBHQwMx2mdntZnaXmd3lGzIQWGtmq4CJwGDfq4tIgRZfsyxT7+vM\nfZfVY8qqn+iemMzU1T+phYEUCFE5DXDODclh/XPAcwFLJBJCikRFMrZnA/o0q8KfPlrNyHe/Y0rj\nn3hyQFPiShX1Op7IOekTqiJ+aFSlFJ/e05GH+zQkefMBuicm8/7SH3UVLyFLxV3ET1GREdzZtS5f\njk6gUZVSPPjxGv4weQk//qxGZBJ6VNxFcql2hRIk3dGepwY0ZdXOw/Qan8Lk+d9zWo3IJISouIvk\nQUSE8Yf2NZk1JoEOdcvz5NT1XDtpIZv3qRGZhAYVd5ELULVMMSbf3IYJg1vyw8+/ccXEeUz8agsn\n09WITLyl4i5ygcyM/i2rMWdsV3o3rULi7M30e24+q3b+6nU0KcRU3EUCpHxsEZ4d0opXbmrDL8dO\ncvULC/jn9A0cP6k+NZL/VNxFAqxH4zhmj+3K9ZdU56WU7fSZkMKibfrQtuQvFXeRIChVNJp/XtOc\nd4e144yDIa8s5s+fruHIiVNeR5NCQsVdJIg61qvAzNEJ3NGlNknf/kjPxBS+3rjP61hSCKi4iwRZ\nsZhIHrmiMZ/c04nSxaK57Y1ljEr6jp9T07yOJmFMxV0kn7SsXoYv7u3M6O71mb5mDz3GpTBllRqR\nSXCouIvko5ioCEZ3v5ip93aherni3Pfed9zx1jL2Hj7hdTQJMyruIh5oULkkn9zdkUevaMT8rQfp\nkZjMu0t+5IxaGEiAqLiLeCQywhjWpQ4zRyfQtFpp/vzpGm54dTE7Dv7mdTQJAyruIh6rWb4E797R\njqevaca63UfoPSGFV1K2qxGZXBAVd5EQYGYMbluD2WO70rleBf4+fQPXvLCATXvViEzyRsVdJIRU\nLl2UV25qw7NDWrHrl+Nc+ew8xs3erEZkkmsq7iIhxsy4qkVVZo/tyhXNqjDhqy1c+ew8vvvxF6+j\nSQGi4i4SosqViGH84Fa8dksbjp5I55pJC3lvQxrHTqZ7HU0KABV3kRB3WcM4Zo1J4MZ2NZj5Qzq9\nx89j4daDXseSEKfiLlIAlCwazVMDmvFQ26JEGNzw6hIe+ng1h4+rEZlkT8VdpABpWC6SL0cncGfX\nOnywbCc9xyUze70akcnvqbiLFDBFoyN5uE8jPhvRibLFY7jjrWWMfHcFB9WITDLJsbib2Wtmtt/M\n1p5jvZnZRDPbamarzax14GOKSFbNLyrDlJGdub/Hxcxat48eicl89t1uNSITwL8r9zeA3udZ3weo\n75uGA5MuPJaI+CMmKoJ7L6/PtPs6U6tCCUa/v5Lb3ljKT78e9zqaeCzH4u6cSwEOnWdIf+Atl2Ex\nUMbMqgQqoIjkrH5cST66qyOPXdmYxdsP0XNcCm8v/kGNyAqxQNxzrwbszPR4l2+ZiOSjyAjjts61\nmTUmgZbVy/CXz9Yy+JXFfK9GZIWS+XN/zsxqAVOdc02zWTcN+Kdzbr7v8VfAn5xzy7MZO5yMWzfE\nxcXFJyUl5Sl0amoqsbGxedo2mEI1F4RuNuXKHX9zOeeYtzud9zaeJP0MXF0vml61oomMME9z5bdw\nzNWtW7flzrk2OQ50zuU4AbWAtedY9xIwJNPjTUCVnPYZHx/v8mru3Ll53jaYQjWXc6GbTblyJ7e5\n9h4+7u54c6mr+eBUd+XEeW7d7sMhkSu/hGMuYJnzo24H4rbMFOAm37tm2gOHnXN7ArBfEblAcaWK\n8tLQeJ6/oTV7Dh+n33Pz+b9Zm0hLP+11NAmyqJwGmNl7wKVABTPbBfwViAZwzr0ITAf6AluBY8Ct\nwQorIrlnZlzRvAod65bnyWnrefbrrcxYu5dnrm1OfM2yXseTIMmxuDvnhuSw3gEjApZIRIKibIkY\nEq9rSb8WVXnk07UMfHEht3SsxQM9G1CiSI6lQAoYfUJVpJC5tEElZo5JYGj7mry+YAe9xqcwb8sB\nr2NJgKm4ixRCsUWieKJ/Uz64swMxkREMnfwtf/poFYePqRFZuFBxFynE2tYux/RRXbj70rp8vGI3\n3ccl8+XavV7HkgBQcRcp5IpGR/Jg74Z8PqITFWOLcNd/lzPinRUcOKpGZAWZiruIANC0Wmk+H9mJ\nP/ZqwOwN++iemMzHy3epEVkBpeIuIv8THRnBiG71mH5fF+pViuX+D1dx8+tL2fXLMa+jSS6puIvI\n79SrFMuHd3bgb/2asGzHIXqNS+GtRTvUiKwAUXEXkWxFRBg3d6zFzNEJtK5Zlsc+X8f1Ly9i24FU\nr6OJH1TcReS8qpcrzlu3teU/g1qweV8qfSbM44VvtnLq9Bmvo8l5qLiLSI7MjIHxFzF7bALdG1Xi\nX19uYsDzC1i7+7DX0eQcVNxFxG+VShblhRvjefEPrdl3JI3+zy/gX19u5ORp3YsPNWooISK51rtp\nFTrUqcBT09bzwjfbqFzCKFf3EG1qlfM6mvjoyl1E8qR08Wj+PagFb93WllOnYdBLi/jr52tJTUv3\nOpqg4i4iFyjh4or8vXMxbu5Qi7cW/0CvcSkkb1YjMq+puIvIBSsaZTzerwkf3dWBotER3Pzat9z/\nwSp+PXbS62iFloq7iARMfM1yTLuvCyO71ePzlbvpnpjM9DX6YjYvqLiLSEAVjY7kgV4N+HxkJyqX\nLso976zgrreXs//ICa+jFSoq7iISFE2qluazezrxYO+GfL1pP90Tk/lg2U41IssnKu4iEjRRkRHc\nfWldvhzVhYaVS/Gnj1Zz02vfsvOQGpEFm4q7iARdnYqxJA1vz5P9m7Dih1/oNT6F1xd8z2k1Igsa\nFXcRyRcREcbQDrWYNbYrbWuX429frGfQiwvZuv+o19HCkoq7iOSramWK8fotlzDu+hZsP/gbfSfM\n57mvt6gRWYCpuItIvjMzrm51EXPGdqVHkzj+M2szVz07nzW71IgsUFTcRcQzFWKL8PwNrXlpaDyH\nfjvJgBcW8PSMjZw4ddrraAWeX8XdzHqb2SYz22pmD2Wz/hYzO2BmK33TsMBHFZFw1atJZWaP7crA\n1hfxYvI2+kyYx5LtP3sdq0DLsbibWSTwPNAHaAwMMbPG2Qx93znX0je9GuCcIhLmSheL5pmBzXln\nWDvSz5zh+pcX85fP1nL0xCmvoxVI/ly5twW2Oue2O+dOAklA/+DGEpHCqlO9CswcncDtnWvz3yUZ\njcjmbtzvdawCx5/iXg3YmenxLt+yrK41s9Vm9pGZVQ9IOhEplIrHRPGXKxvz8d0dKVEkilvfWMqY\n91dy6Dc1IvOX5fRRYDMbBPRyzg3zPR4KtHXO3ZtpTHkg1TmXZmZ3Adc55y7LZl/DgeEAcXFx8UlJ\nSXkKnZqaSmxsbJ62DaZQzQWhm025cqcw5jp1xjF12ymmbj9F8WgY2qgIl1SOxMw8zXUhLiRXt27d\nljvn2uQ40Dl33gnoAMzM9Phh4OHzjI8EDue03/j4eJdXc+fOzfO2wRSquZwL3WzKlTuFOdf6nw67\nq56d52o+ONUNe3Op23v4eEjkyosLyQUscznUV+ecX7dllgL1zay2mcUAg4EpmQeYWZVMD/sBG/zY\nr4iI3xpVKcUnd3fkz30bkrL5AN0Tk3l/6Y9qRHYOORZ351w6MBKYSUbR/sA5t87MnjCzfr5h95nZ\nOjNbBdwH3BKswCJSeEVFRjA8oS4zRyfQuEopHvx4DTe+uoQff1Yjsqz8+oJs59x0YHqWZY9lmn+Y\njNs1IiJBV6tCCd67oz1JS3fyj+kb6Dk+mQd6NuDWTrWJjMj5XnxhoE+oikiBFBFh3NCuBrPHJtCx\nbgWemraBayctZPM+NSIDFXcRKeCqlC7G5JvbMGFwS348dIwrJs5jwpwtnEwv3I3IVNxFpMAzM/q3\nrMbsMQn0aVqFcXM20++5+Ww/XHh71Ki4i0jYKB9bhIlDWvHqTW349dgpnlx0gn9M38Dxk4WvyKu4\ni0jY6d44jlljE+haPYqXU7bTe0IKi7YVrkZkKu4iEpZKFY3mliZFePeOdgAMeWUxD3+yhiOFpBGZ\niruIhLWOdSvw5agEhifU4f2lP9IzMYWvNuzzOlbQqbiLSNgrFhPJn/s24pN7OlG6WDS3v7mM+977\njp9T07yOFjQq7iJSaLSsXoYv7u3MmO4XM2PtHnqMS+HzlbvDsoWBiruIFCoxURGM6l6fafd1oUa5\n4oxKWsmwN5ex5/Bxr6MFlIq7iBRKF8eV5OO7O/LoFY1YsO0gPRNTeHfJj5w5Ex5X8SruIlJoRUYY\nw7rUYdborjS7qDR//nQNN7y6mB0Hf/M62gVTcReRQq9G+eK8M6wdT1/TjHW7j9BrfAovp2wj/XTB\nbWGg4i4iQkYLg8FtazB7bFe61K/IP6Zv5NpJC9m494jX0fJExV1EJJPKpYvyyk3xPDukFbt+Oc6V\nE+eTOHszaekFq4WBiruISBZmxlUtqjJ7bFeualGViV9t4apn5/Pdj794Hc1vKu4iIudQrkQM465v\nyeu3XMLRE+lcM2khT05dz7GT6V5Hy5GKu4hIDro1rMSsMQnc2K4Gk+d/T6/xKSzYetDrWOel4i4i\n4oeSRaN5akAz3h/enqiICG58dQkPfbyaw8dDsxGZiruISC60q1OeGaO6cGfXOnywbCc9EpOZtW6v\n17F+R8VdRCSXikZH8nCfRnzrfhomAAAIcklEQVQ2ohPlSsQw/O3ljHx3BQdDqBGZiruISB41vyij\nEdkDPS9m1rp9dE9M5tPvdoVEIzIVdxGRCxAdGcHIy+ozfVRn6lQowZj3V3HrG0vZ/au3jchU3EVE\nAqBepZJ8eFdH/npVY5ZsP0TPxGTeXvyDZ43I/CruZtbbzDaZ2VYzeyib9UXM7H3f+iVmVivQQUVE\nQl1khHFrp9rMGpNAqxpl+ctnaxn88mK2H0jN9yw5FncziwSeB/oAjYEhZtY4y7DbgV+cc/WAccAz\ngQ4qIlJQVC9XnLdvb8u/BjZn494j9JkwjxeT87cRmT9X7m2Brc657c65k0AS0D/LmP7Am775j4DL\nzcwCF1NEpGAxM65rU505Y7tyaYOKPD1jIwNeWMD6n/KnEZk/xb0asDPT412+ZdmOcc6lA4eB8oEI\nKCJSkFUqVZSXhrZh0o2t2Xs4jX7PzWfmjuB/8MlyesuOmQ0CejnnhvkeDwXaOufuzTRmnW/MLt/j\nbb4xP2fZ13BgOEBcXFx8UlJSnkKnpqYSGxubp22DKVRzQehmU67cUa7cCbVcqScd7208SdPSp+hQ\nM2+5unXrttw51ybHgc65805AB2BmpscPAw9nGTMT6OCbjwIO4nvhONcUHx/v8mru3Ll53jaYQjWX\nc6GbTblyR7lyJxxzActcDnXbOefXbZmlQH0zq21mMcBgYEqWMVOAm33zA4GvfSFERMQDUTkNcM6l\nm9lIMq7OI4HXnHPrzOwJMl5BpgCTgbfNbCtwiIwXABER8UiOxR3AOTcdmJ5l2WOZ5k8AgwIbTURE\n8kqfUBURCUMq7iIiYUjFXUQkDKm4i4iEIRV3EZEwlOMnVIP2g80OAD/kcfMKZHxQKtSEai4I3WzK\nlTvKlTvhmKumc65iToM8K+4XwsyWOX8+fpvPQjUXhG425cod5cqdwpxLt2VERMKQiruISBgqqMX9\nZa8DnEOo5oLQzaZcuaNcuVNocxXIe+4iInJ+BfXKXUREziNki7uZ/dvMNprZajP71MzKnGNctl/e\n7WtRvMTMtvi+vDsmQLkGmdk6MztjZtn+tdvMGpjZykzTETMb7Vv3uJntzrSub37l8o3bYWZrfD97\nWabl5cxstu94zTazsvmVy8yqm9lcM9vgGzsq0zqvj1d+n185Pg9m1i3L+XXCzAb41r1hZt9nWtcy\nELn8zeYbdzrTz5+SabmXx6ylmS3yPeerzez6TOsCdszOdb5kWl/E92/f6jsWtTKte9i3fJOZ9cpr\nhv/xp+m7FxPQE4jyzT8DPJPNmEhgG1AHiAFWAY196z4ABvvmXwTuDlCuRkAD4BugjR/jI4G9ZLw3\nFeBx4IEgHC+/cgE7gArZLP8X8JBv/qHsjnewcgFVgNa++ZLA5kzPo2fHy6PzK1fPA1COjDbbxX2P\n3wAGBvp45SYbkHqO5Z4dM+BioL5vviqwBygTyGN2vvMl05h7gBd984OB933zjX3jiwC1ffuJvJA8\nIXvl7pyb5TK+jxVgMXBRNsOy/fJuMzPgMjK+rBsyvrx7QIBybXDObcrFJpcD25xzef3All/ykCur\nzF9ynq/Hyzm3xzm3wjd/FNjA77+nN6D8PF75fn6R++dhIDDDOXcsQD//fPJ8jnh9zJxzm51zW3zz\nPwH7gRw/CJRL2Z4v58n6EXC579j0B5Kcc2nOue+Brb795VnIFvcsbgNmZLP8XF/eXR74NdOLQ3Zf\n6p1fBgPvZVk20ver4WuBuv2RCw6YZWbLLeM7bc+Kc87tgYxiC1TK51wA+H5NbQUsybTYq+PlxfmV\n2+chu/Pr777jNc7MigQoV26yFTWzZWa2+OztIkLomJlZWzKurLdlWhyIY3au8yXbMb5jcZiMY+PP\ntrni15d1BIuZzQEqZ7PqEefc574xjwDpwDvZ7SKbZe48ywOWy8/9xAD9yPje2bMmAU/68jwJ/B8Z\nL175lauTc+4nM6sEzDazjc65FD+3DWYuzCwW+BgY7Zw74lvs5fHK9/PL33349lMFaEbGt6Sd9TAZ\ntwFjyHi73YPAE/mcrYbvHKsDfG1ma4Aj2Yzz6pi9DdzsnDvjW3xBxyzz7rNZlvXfGJRzKjueFnfn\nXPfzrTezm4Ergcud78ZUFruA6pkeXwT8REbPhjJmFuV7dTy7PCC5cqEPsMI5ty/Tvv83b2avAFPz\nM5fvV1Kcc/vN7FMyfvVLAfaZWRXn3B7f/wD78zOXmUWTUdjfcc59kmnfXh6vfD+/zCw3z8N1wKfO\nuVOZ9r3HN5tmZq8DD/ibK1DZMp1j283sGzJ+E/sYj4+ZmZUCpgGPOucWZ9r3BR2zTM51vmQ3ZpeZ\nRQGlyfibiT/b5krI3pYxs95kvIL2O8/9xGy/vNv3QjCXjPuRkPHl3X5fQQbQELL8yuw7+c66Glib\nX2HMrISZlTw7T8Yfrc/+/Mxfcp6vx8t3z3EysME5l5hlnWfHC2/Or9w8D+c8v3zHdACBPV45ZjOz\nsmdva5hZBaATsN7rY+Z7/j4F3nLOfZhlXaCOWbbny3myDgS+9h2bKcBg37tpagP1gW/zmCPDhf6F\nOFgTGX9Q2Ams9E1n/8JcFZieaVxfMt5dsY2MX7fPLq/jOzhbgQ+BIgHKdTUZr7JpwD5g5jlyFQd+\nBkpn2f5tYA2w2veEVsmvXL5jsso3rctyvMoDXwFbfP8tl4+5OpPxK+jqTM93X6+Pl0fnV7bPA9AG\neDXTuFrAbiAiy/Zf+47XWuC/QGwgcvmbDejo+/mrfP+9PRSOGfAH4FSm82sl0DLQxyy784WMWzz9\nfPNFff/2rb5jUSfTto/4ttsE9LnQ46JPqIqIhKGQvS0jIiJ5p+IuIhKGVNxFRMKQiruISBhScRcR\nCUMq7iIiYUjFXUQkDKm4i4iEof8HqJQ/lyseqIMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112c96a58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plt.plot(-2,3, '-')\n",
    "plt.plot([0,-2], [0,3])\n",
    "plt.grid(True, which='both')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imagine taking the output value that comes out from the circuit and tugging on it in the positive direction.\n",
    "The tugging forces turn out to be the derivative of the output value with respect to its inputs (x and y).\n",
    "\n",
    "**The derivative can be thought of as a force on each input as we pull on the output to become higher.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial f(x,y)}{\\partial x} = \\frac{f(x+h,y) - f(x,y)}{h} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.00000000000189 -2.0000000000042206\n"
     ]
    }
   ],
   "source": [
    "x,y = -2,3\n",
    "out = forwardMultiplyGate(x, y)\n",
    "h = 0.0001\n",
    "\n",
    "# compute derivative with respect to x\n",
    "xph = x + h # -1.9999\n",
    "out2 = forwardMultiplyGate(xph, y) # -5.9997\n",
    "x_derivative = (out2 - out) / h # 3.0\n",
    "\n",
    "# compute derivative with respect to y\n",
    "yph = y + h # 3.0001\n",
    "out3 = forwardMultiplyGate(x, yph) # -6.0002\n",
    "y_derivative = (out3 - out) / h # -2.0\n",
    "\n",
    "print(x_derivative, y_derivative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "derivative -> a single input  \n",
    "gradient -> all the inputs (entire circuit)  \n",
    "The gradient is made up of the derivatives of all the inputs concatenated in a vector (i.e. a list).\n",
    "Indicates direction of the steepest increase in the function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5.87059999999986"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x,y = -2,3\n",
    "step_size = 0.01\n",
    "out = forwardMultiplyGate(x, y) # before: -6\n",
    "x = x + step_size * x_derivative # x becomes -1.97\n",
    "y = y + step_size * y_derivative # y becomes 2.98\n",
    "out_new = forwardMultiplyGate(x, y)\n",
    "out_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Need to compute circuit output for every input independently -> linear complexity -> expensive w/ large num of inputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analytic Gradient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial f(x,y)}{\\partial x} = \\frac{f(x+h,y) - f(x,y)}{h}\n",
    "= \\frac{(x+h)y - xy}{h}\n",
    "= \\frac{xy + hy - xy}{h}\n",
    "= \\frac{hy}{h}\n",
    "= y $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "note on third step  =>  f(x,y) = xy  \n",
    "** derivative wrt x is y  \n",
    "derivative wrt y is x **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5.87059999999986"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x,y = -2,3\n",
    "x_gradient = y\n",
    "y_gradient = x\n",
    "x += step_size * x_gradient\n",
    "x += step_size * y_gradient\n",
    "out_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gives the exact gradient (via calculus), only requires one forward pass!  Very efficient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recursion - circuit w/ multiple gates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New function: $ f(x,y,z) = (x + y) z $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-12"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def forwardMultiplyGate(a,b):\n",
    "    return a*b\n",
    "\n",
    "def forwardAddGate(a,b):\n",
    "    return a+b\n",
    "\n",
    "def forwardCircuit(x,y,z):\n",
    "    q = forwardAddGate(x,y)\n",
    "    return forwardMultiplyGate(q,z)\n",
    "\n",
    "x,y,z = -2, 5, -4\n",
    "f = forwardCircuit(x,y,z)\n",
    "f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the derivatives  \n",
    "1. multiplication gate: $ f(q,z) = qz \\implies q' = z \\hspace{0.25in} z' = q $\n",
    "2. addition gate: $ f(x,y) = x + y \\implies x' = 1 \\hspace{0.25in} y' = 1 $\n",
    "3. chain rule: multiply derivatives together (computing gradient of the final output wrt x,y) \n",
    "Final derivative for x:  \n",
    "$$ \\frac{\\partial f(q,z)}{\\partial x} = \\frac{\\partial q(x,y)}{\\partial x} \\frac{\\partial f(q,z)}{\\partial q} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x,y,z = -2, 5, -4\n",
    "q = forwardAddGate(x,y)       #=> 3\n",
    "f = forwardMultiplyGate(q,z)  #=> -12\n",
    "\n",
    "# derivatives of multiply gate\n",
    "derivative_f_wrt_q = z  #=> -4\n",
    "derivative_f_wrt_z = q  #=> 3\n",
    "\n",
    "# derivatives of add gate\n",
    "derivative_q_wrt_x = 1\n",
    "derivative_q_wrt_y = 1\n",
    "\n",
    "# chain rule\n",
    "derivative_f_wrt_x = derivative_q_wrt_x * derivative_f_wrt_q  #=> -4\n",
    "derivative_f_wrt_y = derivative_q_wrt_y * derivative_f_wrt_q  #=> -4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-11.5924"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# final gradient, from above: [-4, -4, 3]\n",
    "gradient_f_wrt_xyz = [derivative_f_wrt_x, derivative_f_wrt_y, derivative_f_wrt_z]\n",
    "\n",
    "step_size = 0.01\n",
    "\n",
    "# let inputs respond to the 'tug' of the gradient\n",
    "x += step_size * derivative_f_wrt_x  #=> -2.04, pulled down -.04\n",
    "y += step_size * derivative_f_wrt_y  #=> 4.96, pulled down -.04\n",
    "z += step_size * derivative_f_wrt_z  #=> -3.97, pulled up +.03\n",
    "\n",
    "# forward pass w/ new inputs\n",
    "q = forwardAddGate(x,y)\n",
    "f = forwardMultiplyGate(q,z)\n",
    "f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The local derivative of the forwardAddGate (q) is calculated as +1 (how to make value of q higher)  \n",
    "The gradient on q is calculated as -4, the circuit wants q to decrease by a magnitude of 4\n",
    "Accordingly, the chain rule applies the gradient according to the full circuit (+1 x -4 = -4)  \n",
    "**This is how backpropagation works**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> As we pull on the circuit’s output value at the end, this induces pulls downward through the entire circuit, all the way down to the inputs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single Neuron example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ f(x,y,a,b,c) = \\sigma(ax + by + c) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\sigma$ = sigmoid (squashing function between 0 and 1\n",
    "\n",
    "$$ \\sigma(x) = \\frac{1}{1 + e^{-x}} $$\n",
    "\n",
    "derivative wrt x:  $$ \\frac{\\partial \\sigma(x)}{\\partial x} = \\sigma(x) (1 - \\sigma(x)) $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# input to sigmoid gate\n",
    "x = 3\n",
    "\n",
    "#forward\n",
    "f = 1.0 / (1.0 + math.exp(-x)) #=> 0.95\n",
    "\n",
    "#backprop (gradient)\n",
    "dx = f * (1.0-f) #=> .045\n",
    "\n",
    "# note: sigmoid is really a series of gates (division, addition, exponentiation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Unit:\n",
    "    def __init__(self, value, grad):\n",
    "        # value computed in forward pass\n",
    "        self.value = value\n",
    "        # derivative of circuit output wrt this unit; computed in backward pass\n",
    "        self.grad = grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class multiplyGate:\n",
    "    def forward(self, u0, u1):\n",
    "        self.u0 = u0\n",
    "        self.u1 = u1\n",
    "        # backprop is auto initialized to 0.0\n",
    "        self.utop = Unit(self.u0.value * self.u1.value, 0.0)\n",
    "        return self.utop\n",
    "    \n",
    "    def backward(self):\n",
    "        # gradient in output unit chained with local gradients\n",
    "        self.u0.grad += self.u1.value * self.utop.grad\n",
    "        self.u1.grad += self.u0.value * self.utop.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class addGate:\n",
    "    def forward(self, u0, u1):\n",
    "        self.u0 = u0\n",
    "        self.u1 = u1\n",
    "        self.utop = Unit(self.u0.value + self.u1.value, 0.0)\n",
    "        return self.utop\n",
    "    \n",
    "    def backward(self):\n",
    "        self.u0.grad += 1 * self.utop.grad\n",
    "        self.u1.grad += 1 * self.utop.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class sigmoidGate:\n",
    "    def sig(self, x):\n",
    "        return 1 / (1 + math.exp(-x))\n",
    "    \n",
    "    # only takes one input\n",
    "    def forward(self, u0):\n",
    "        self.u0 = u0\n",
    "        self.utop = Unit(self.sig(self.u0.value), 0.0)\n",
    "        return self.utop\n",
    "    \n",
    "    def backward(self):\n",
    "        s = self.sig(self.u0.value);\n",
    "        self.u0.grad += (s * (1.0 - s)) * self.utop.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8807970779778823, 0.0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create inputs\n",
    "a = Unit(1.0, 0.0)\n",
    "b = Unit(2.0, 0.0)\n",
    "c = Unit(-3.0, 0.0)\n",
    "x = Unit(-1.0, 0.0)\n",
    "y = Unit(3.0, 0.0)\n",
    "\n",
    "# create gates\n",
    "mulg0 = multiplyGate()\n",
    "mulg1 = multiplyGate()\n",
    "addg0 = addGate()\n",
    "addg1 = addGate()\n",
    "sg = sigmoidGate()\n",
    "\n",
    "# forward pass\n",
    "def forwardNeuron():\n",
    "    ax = mulg0.forward(a,x)\n",
    "    by = mulg1.forward(b,y)\n",
    "    axpby = addg0.forward(ax,by)\n",
    "    axpbypc = addg1.forward(axpby,c)\n",
    "    return sg.forward(axpbypc)\n",
    "\n",
    "s = forwardNeuron()\n",
    "s.value, s.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to compute the gradient we iterate in reverse and call backward\n",
    "s.grad = 1.0\n",
    "# set gradient chain at output to 1.0 to start things off.  Increase value of function\n",
    "\n",
    "sg.backward()    # writes gradient into axpbypc\n",
    "addg1.backward() # writes gradient into axpby, c\n",
    "addg0.backward() # writes gradient into ax, by\n",
    "mulg1.backward() # writes gradient into b, y\n",
    "mulg0.backward() # writes gradient into a, x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.10499358540350662 0.31498075621051985 0.10499358540350662 0.10499358540350662 0.20998717080701323\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8825501816218984"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make inputs respond to computed gradients\n",
    "step_size = 0.01;\n",
    "a.value += step_size * a.grad # a.grad is -0.105\n",
    "b.value += step_size * b.grad # b.grad is 0.315\n",
    "c.value += step_size * c.grad # c.grad is 0.105\n",
    "x.value += step_size * x.grad # x.grad is 0.105\n",
    "y.value += step_size * y.grad # y.grad is 0.210\n",
    "\n",
    "print(a.grad, b.grad, c.grad, x.grad, y.grad)\n",
    "\n",
    "s = forwardNeuron()\n",
    "s.value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.10499758359205913 0.3149447748351797 0.10498958734506125 0.10498958734506125 0.2099711788272618\n"
     ]
    }
   ],
   "source": [
    "# check numerical gradient\n",
    "def forwardCircuitFast(a,b,c,x,y):\n",
    "    return 1/(1 + math.exp( - (a*x + b*y + c)))\n",
    "\n",
    "a,b,c,x,y = 1,2,-3,-1,3\n",
    "h = 0.0001;\n",
    "a_grad = (forwardCircuitFast(a+h,b,c,x,y) - forwardCircuitFast(a,b,c,x,y))/h;\n",
    "b_grad = (forwardCircuitFast(a,b+h,c,x,y) - forwardCircuitFast(a,b,c,x,y))/h;\n",
    "c_grad = (forwardCircuitFast(a,b,c+h,x,y) - forwardCircuitFast(a,b,c,x,y))/h;\n",
    "x_grad = (forwardCircuitFast(a,b,c,x+h,y) - forwardCircuitFast(a,b,c,x,y))/h;\n",
    "y_grad = (forwardCircuitFast(a,b,c,x,y+h) - forwardCircuitFast(a,b,c,x,y))/h;\n",
    "\n",
    "print(a_grad, b_grad, c_grad, x_grad, y_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.10499358540350662 0.31498075621051985 0.10499358540350662 0.10499358540350662 0.20998717080701323\n"
     ]
    }
   ],
   "source": [
    "# lets do our neuron in two steps:\n",
    "def sig(x):\n",
    "    return 1 / (1 + math.exp(-x))\n",
    "\n",
    "# forward pass\n",
    "q = a*x + b*y + c;\n",
    "f = sig(q)\n",
    "\n",
    "# backward pass; given df:\n",
    "df = 1\n",
    "dq = (f * (1.0 - f)) * df\n",
    "da = x * dq\n",
    "dx = a * dq\n",
    "db = y * dq\n",
    "dy = b * dq\n",
    "dc = 1 * dq\n",
    "\n",
    "print (da, db, dc, dx, dy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# calculate the gradient of complex function\n",
    "x = math.pow(((a * b + c) * d), 2) # pow(x,2) squares the input\n",
    "\n",
    "# split it up into chunks\n",
    "z = a * b + c\n",
    "y = z * d\n",
    "x = y * y\n",
    "\n",
    "dy = 2 * y * dx\n",
    "dd = z * dy\n",
    "dz = d * dy\n",
    "da = b * dz\n",
    "db = a * dz\n",
    "dc = 1 * dz\n",
    "# done!\n",
    "\n",
    "# + gate simply takes the gradient on top and routes it equally to all of its inputs\n",
    "# * gate acts as a switcher during backward pass; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# other useful gradients:\n",
    "# division\n",
    "x = 1.0/a\n",
    "da = -1.0/(a*a);\n",
    "\n",
    "# division in practice\n",
    "x = (a + b)/(c + d);\n",
    "\n",
    "# lets decompose it in steps:\n",
    "y = a + b\n",
    "z = c + d\n",
    "q = 1/z\n",
    "x = y * q\n",
    "\n",
    "# backprop\n",
    "dy = q * dx\n",
    "dq = y * dx\n",
    "dz = -1(z*z) * dq\n",
    "dc = 1 * dz\n",
    "dd = 1 * dz\n",
    "da = 1 * dy\n",
    "db = 1 * dy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# another useful one:\n",
    "# max gate acts as a simple switch between a or b\n",
    "x = math.max(a, b);\n",
    "\n",
    "# if a is the result of max -> a' = 1 else 0\n",
    "# if b is the result of max -> b' = 1 else 0\n",
    "da = 1.0 * dx if a == x else 0.0\n",
    "db = 1.0 * dx if b == x else 0.0\n",
    " \n",
    "# RELU\n",
    "x = math.max(a, 0)\n",
    "\n",
    "# backprop through this gate will then be:\n",
    "da = 1.0 * dx if a > 0 else 0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SVM - linear classifier:  $ \\hspace{0.25in} f(x,y) = ax + by + c $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVM 'Force Specification'\n",
    "\n",
    "- If we feed a positive datapoint through the SVM circuit and the output value is less than 1, pull on the circuit with force +1. This is a positive example so we want the score to be higher for it.\n",
    "- Conversely, if we feed a negative datapoint through the SVM and the output is greater than -1, then the circuit is giving this datapoint dangerously high score: Pull on the circuit downwards with force -1.\n",
    "- In addition to the pulls above, always add a small amount of pull on the parameters a,b (notice, not on c!) that pulls them towards zero. If a becomes very high it will experience a strong pull of magnitude |a| back towards zero. This pull is something we call *regularization*, and it ensures that neither of our parameters a or b gets disproportionally large.  If either of them is too high, our classifier would be overly sensitive to these features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVM for binary classification example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A circuit: it takes 5 Units (x,y,a,b,c) and outputs a single Unit\n",
    "# It can also compute the gradient w.r.t. its inputs\n",
    "class Circuit:\n",
    "    def __init__(self):\n",
    "        # create some gates\n",
    "        self.mulg0 = multiplyGate();\n",
    "        self.mulg1 = multiplyGate();\n",
    "        self.addg0 = addGate();\n",
    "        self.addg1 = addGate();\n",
    "\n",
    "    def forward(self,x,y,a,b,c):\n",
    "        self.ax = self.mulg0.forward(a, x) # a*x\n",
    "        self.by = self.mulg1.forward(b, y) # b*y\n",
    "        self.axpby = self.addg0.forward(self.ax, self.by) # a*x + b*y\n",
    "        self.axpbypc = self.addg1.forward(self.axpby, c) # a*x + b*y + c\n",
    "        return self.axpbypc\n",
    "  \n",
    "    def backward(self,gradient_top):  # takes pull from above\n",
    "        self.axpbypc.grad = gradient_top;\n",
    "        self.addg1.backward() # sets gradient in axpby and c\n",
    "        self.addg0.backward() # sets gradient in ax and by\n",
    "        self.mulg1.backward() # sets gradient in b and y\n",
    "        self.mulg0.backward() # sets gradient in a and x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SVM:\n",
    "    def __init__(self):\n",
    "        # random initial weights\n",
    "        self.a = Unit(1.0, 0.0);\n",
    "        self.b = Unit(-2.0, 0.0);\n",
    "        self.c = Unit(-1.0, 0.0);\n",
    "        self.circuit = Circuit();\n",
    "\n",
    "    def forward(self, x, y):  # assume x and y are Units\n",
    "        self.unit_out = self.circuit.forward(x, y, self.a, self.b, self.c)\n",
    "        return self.unit_out\n",
    "\n",
    "    def backward(self, label): # label is +1 or -1\n",
    "        # reset pulls on a,b,c\n",
    "        self.a.grad = 0.0;\n",
    "        self.b.grad = 0.0;\n",
    "        self.c.grad = 0.0;\n",
    "\n",
    "        # compute the pull(starting gradient) based on what the circuit output was\n",
    "        pull = 0.0;\n",
    "        if(label == 1 and self.unit_out.value < 1):\n",
    "            pull = 1 # the score was too low: pull up\n",
    "        if(label == -1 and self.unit_out.value > -1):\n",
    "            pull = -1 # the score was too high for a positive example, pull down\n",
    "\n",
    "        self.circuit.backward(pull) # writes gradient into x,y,a,b,c\n",
    "    \n",
    "        # add regularization pull for multiplication parameters: towards zero and proportional to value\n",
    "#         self.a.grad -= self.a.value;\n",
    "#         self.b.grad -= self.b.value;\n",
    "\n",
    "    def parameterUpdate(self): # update weights according to backprop\n",
    "        step_size = 0.01;\n",
    "        self.a.value += step_size * self.a.grad\n",
    "        self.b.value += step_size * self.b.grad\n",
    "        self.c.value += step_size * self.c.grad\n",
    "        \n",
    "    def learnFrom(self, x, y, label):\n",
    "        self.forward(x, y) # forward pass (set .value in all Units)\n",
    "        self.backward(label) # backward pass (set .grad in all Units)\n",
    "        self.parameterUpdate() # parameters respond to tug"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train w/ SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = [[1.2, 0.7], [-0.3, -0.5], [3.0, 0.1], [-0.1, -1.0], [-1.0, 1.1], [2.1, -3]]\n",
    "labels = [1, -1, 1, -1, -1, 1]\n",
    "svm = SVM()\n",
    "\n",
    "# a function that computes the classification accuracy\n",
    "def evalTrainingAccuracy():\n",
    "    num_correct = 0\n",
    "    for i,d in enumerate(data):\n",
    "        x = Unit(d[0], 0.0)\n",
    "        y = Unit(d[1], 0.0)\n",
    "        true_label = labels[i]\n",
    "\n",
    "        # see if the prediction matches the provided label\n",
    "        predicted_label = 1 if svm.forward(x, y).value > 0 else -1\n",
    "        if predicted_label == true_label:\n",
    "            num_correct += 1\n",
    "\n",
    "    return num_correct / len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy at iter 0: 1.0\n",
      "training accuracy at iter 25: 1.0\n",
      "training accuracy at iter 50: 1.0\n",
      "training accuracy at iter 75: 1.0\n",
      "training accuracy at iter 100: 1.0\n",
      "training accuracy at iter 125: 1.0\n",
      "training accuracy at iter 150: 1.0\n",
      "training accuracy at iter 175: 1.0\n",
      "training accuracy at iter 200: 1.0\n",
      "training accuracy at iter 225: 1.0\n",
      "training accuracy at iter 250: 1.0\n",
      "training accuracy at iter 275: 1.0\n",
      "training accuracy at iter 300: 1.0\n",
      "training accuracy at iter 325: 1.0\n",
      "training accuracy at iter 350: 1.0\n",
      "training accuracy at iter 375: 1.0\n"
     ]
    }
   ],
   "source": [
    "# the learning loop\n",
    "for iterator in range(0,400):\n",
    "    # pick a random data point\n",
    "    i = math.floor(random.random() * len(data))\n",
    "    x = Unit(data[i][0], 0.0);\n",
    "    y = Unit(data[i][1], 0.0);\n",
    "    label = labels[i];\n",
    "    svm.learnFrom(x, y, label)\n",
    "\n",
    "    if iterator % 25 == 0: # every 25 iterations...\n",
    "        acc = evalTrainingAccuracy()\n",
    "        print(f\"training accuracy at iter {iterator}: {acc}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NOTE:  SVM does not converge on the function with regularization!!\n",
    "\n",
    "** Convergence does occur w/ regularization in place and smaller pulls for the other two scenarios not handled in the forward pass (i.e. label == 1 and score > 1: pull -0.5) but will not remain steady... ** "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Condensed SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy at iter 0: 0.6666666666666666\n",
      "training accuracy at iter 25: 0.6666666666666666\n",
      "training accuracy at iter 50: 0.6666666666666666\n",
      "training accuracy at iter 75: 0.8333333333333334\n",
      "training accuracy at iter 100: 0.8333333333333334\n",
      "training accuracy at iter 125: 0.8333333333333334\n",
      "training accuracy at iter 150: 0.8333333333333334\n",
      "training accuracy at iter 175: 0.8333333333333334\n",
      "training accuracy at iter 200: 0.8333333333333334\n",
      "training accuracy at iter 225: 0.8333333333333334\n",
      "training accuracy at iter 250: 1.0\n",
      "training accuracy at iter 275: 1.0\n",
      "training accuracy at iter 300: 1.0\n",
      "training accuracy at iter 325: 1.0\n",
      "training accuracy at iter 350: 1.0\n",
      "training accuracy at iter 375: 1.0\n"
     ]
    }
   ],
   "source": [
    "a,b,c = 1,-2,-1 # initial parameters\n",
    "\n",
    "for iterator in range(0,400):\n",
    "    # pick a random data point\n",
    "    i = math.floor(random.random() * len(data))\n",
    "    x = data[i][0]\n",
    "    y = data[i][1]\n",
    "    label = labels[i]\n",
    "\n",
    "    # forward pass - compute pull\n",
    "    score = a*x + b*y + c\n",
    "    pull = 0.0\n",
    "    if(label == 1 and score < 1):\n",
    "        pull = 1\n",
    "    if(label == 1 and score > 1):\n",
    "        pull = -0.5\n",
    "    if(label == -1 and score > -1):\n",
    "        pull = -1\n",
    "    if(label == -1 and score > -1):\n",
    "        pull = .05\n",
    "        \n",
    "    # backward pass - compute gradient and update parameters accordingly\n",
    "    step_size = 0.01;\n",
    "    a += step_size * (x * pull - a) # -a is from the regularization\n",
    "    b += step_size * (y * pull - b) # -b is from the regularization\n",
    "    c += step_size * (1 * pull);\n",
    "    \n",
    "\n",
    "    if iterator % 25 == 0: # every 25 iterations... \n",
    "        # compute accuracy\n",
    "        num_correct = 0\n",
    "        for i,l in zip(data,labels):\n",
    "            x = i[0]\n",
    "            y = i[1]\n",
    "            true_label = l\n",
    "            \n",
    "            pred_score = a*x + b*y + c\n",
    "            # see if the prediction matches the provided label\n",
    "            predicted = 1 if pred_score > 0 else -1\n",
    "            if predicted == true_label:\n",
    "                num_correct += 1\n",
    "        percent_correct = num_correct / len(data)\n",
    "        print(f\"training accuracy at iter {iterator}: {percent_correct}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generalizing SVM into neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# compute accuracy\n",
    "def computeAccuracy():\n",
    "    num_correct = 0\n",
    "    for i,d in enumerate(data):\n",
    "        x = d[0]\n",
    "        y = d[1]\n",
    "        true_label = labels[i]\n",
    "\n",
    "        # forward pass\n",
    "        n1 = max(0, a1*x + b1*y + c1); # activation of 1st hidden neuron\n",
    "        n2 = max(0, a2*x + b2*y + c2); # 2nd neuron\n",
    "        n3 = max(0, a3*x + b3*y + c3); # 3rd neuron\n",
    "        pred_score = a4*n1 + b4*n2 + c4*n3 + d4; # the score\n",
    "\n",
    "        # see if the prediction matches the provided label\n",
    "        predicted = 1 if pred_score > 0 else -1\n",
    "        if predicted == true_label:\n",
    "            num_correct += 1\n",
    "    return num_correct / len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy at iter 0: 0.5\n",
      "training accuracy at iter 25: 0.5\n",
      "training accuracy at iter 50: 0.5\n",
      "training accuracy at iter 75: 0.5\n",
      "training accuracy at iter 100: 0.6666666666666666\n",
      "training accuracy at iter 125: 0.6666666666666666\n",
      "training accuracy at iter 150: 0.8333333333333334\n",
      "training accuracy at iter 175: 1.0\n",
      "training accuracy at iter 200: 1.0\n",
      "training accuracy at iter 225: 1.0\n",
      "training accuracy at iter 250: 1.0\n",
      "training accuracy at iter 275: 1.0\n",
      "training accuracy at iter 300: 1.0\n",
      "training accuracy at iter 325: 1.0\n",
      "training accuracy at iter 350: 1.0\n",
      "training accuracy at iter 375: 1.0\n"
     ]
    }
   ],
   "source": [
    "data = [[1.2, 0.7], [-0.3, -0.5], [3.0, 0.1], [-0.1, -1.0], [-1.0, 1.1], [2.1, -3]]\n",
    "labels = [1, -1, 1, -1, -1, 1]\n",
    "\n",
    "# random initial weights\n",
    "a1 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "b1 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "c1 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "a2 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "b2 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "c2 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "a3 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "b3 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "c3 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "a4 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "b4 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "c4 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "d4 = random.random() - 0.5; # a random number between -0.5 and 0.5\n",
    "\n",
    "# ... similarly initialize all other parameters to randoms\n",
    "for iterator in range(0,400):\n",
    "    # pick a random data point\n",
    "    i = math.floor(random.random() * len(data));\n",
    "    x = data[i][0];\n",
    "    y = data[i][1];\n",
    "    label = labels[i];\n",
    "\n",
    "  # forward pass:\n",
    "\n",
    "    #2 layer neural network w/ 3 hidden neurons\n",
    "    n1 = max(0, a1*x + b1*y + c1) # activation of 1st hidden neuron\n",
    "    n2 = max(0, a2*x + b2*y + c2) # 2nd neuron\n",
    "    n3 = max(0, a3*x + b3*y + c3) # 3rd neuron\n",
    "    score = a4*n1 + b4*n2 + c4*n3 + d4; # the score\n",
    "\n",
    "    # compute the pull on top\n",
    "    pull = 0.0;\n",
    "    if(label == 1 and score < 1): pull = 1; # we want higher output! Pull up.\n",
    "    if(label == -1 and score > -1): pull = -1; # we want lower output! Pull down.\n",
    "\n",
    "        \n",
    "  # backward pass:\n",
    "\n",
    "    # backprop through the last \"score\" neuron\n",
    "    da4 = n1 * pull;\n",
    "    dn1 = a4 * pull;\n",
    "    db4 = n2 * pull;\n",
    "    dn2 = b4 * pull;\n",
    "    dc4 = n3 * pull;\n",
    "    dn3 = c4 * pull;\n",
    "    dd4 = 1.0 * pull;\n",
    "\n",
    "    # backprop the ReLU non-linearities, in place\n",
    "    # i.e. just set gradients to zero if the neurons did not \"fire\"\n",
    "    dn3 = 0 if n3 == 0 else dn3;\n",
    "    dn2 = 0 if n2 == 0 else dn2;\n",
    "    dn1 = 0 if n1 == 0 else dn1;\n",
    "\n",
    "    # backprop to parameters of neuron 1\n",
    "    da1 = x * dn1;\n",
    "    db1 = y * dn1;\n",
    "    dc1 = 1.0 * dn1;\n",
    "\n",
    "    # backprop to parameters of neuron 2\n",
    "    da2 = x * dn2;\n",
    "    db2 = y * dn2;\n",
    "    dc2 = 1.0 * dn2;\n",
    "\n",
    "    # backprop to parameters of neuron 3\n",
    "    da3 = x * dn3;\n",
    "    db3 = y * dn3;\n",
    "    dc3 = 1.0 * dn3;\n",
    "\n",
    "    # phew! End of backprop!\n",
    "    # note we could have also backpropped into x,y\n",
    "    # but we do not need these gradients. We only use the gradients\n",
    "    # on our parameters in the parameter update, and we discard x,y\n",
    "\n",
    "    # add the pulls from the regularization, tugging all multiplicative\n",
    "    # weights (i.e. not the biases) downward, proportional to their value\n",
    "#     da1 += -a1; da2 += -a2; da3 += -a3;\n",
    "#     db1 += -b1; db2 += -b2; db3 += -b3;\n",
    "#     da4 += -a4; db4 += -b4; dc4 += -c4;\n",
    "\n",
    "    # finally, do the parameter update\n",
    "    step_size = 0.01;\n",
    "    a1 += step_size * da1; \n",
    "    b1 += step_size * db1; \n",
    "    c1 += step_size * dc1;\n",
    "    a2 += step_size * da2; \n",
    "    b2 += step_size * db2;\n",
    "    c2 += step_size * dc2;\n",
    "    a3 += step_size * da3; \n",
    "    b3 += step_size * db3; \n",
    "    c3 += step_size * dc3;\n",
    "    a4 += step_size * da4; \n",
    "    b4 += step_size * db4; \n",
    "    c4 += step_size * dc4; \n",
    "    d4 += step_size * dd4;\n",
    "    # wow this is tedious, please use for loops in prod.\n",
    "    # we're done!\n",
    "    \n",
    "    if iterator % 25 == 0: # every 25 iterations... \n",
    "        print(f\"training accuracy at iter {iterator}: {computeAccuracy()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2-dimensional SVM. We are given a dataset of N examples $(x_{i0},x_{i1})$ and their corresponding labels $y_i$ which are allowed to be either +1/−1 for positive or negative example respectively. Most importantly, as you recall we have three parameters $(w_0,w_1,w_2)$.  The loss function:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ L = [\\sum_{i=1}^N max(0, -y_{i}( w_0x_{i0} + w_1x_{i1} + w_2 ) + 1 )] + \\alpha [w_0^2 + w_1^2] $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: this expression is always positive (thresholding at 0 and squaring in regularization).  Objective is to minimize this function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "example 0: xi = ([1.2, 0.7]) and label = 1 -- score computed to be 0.560\n",
      "  => cost computed to be 0.440\n",
      "example 1: xi = ([-0.3, 0.5]) and label = -1 -- score computed to be 0.370\n",
      "  => cost computed to be 1.370\n",
      "example 2: xi = ([3, 2.5]) and label = 1 -- score computed to be 1.100\n",
      "  => cost computed to be 0.000\n",
      "regularization cost for current model is 0.005\n",
      "total cost is 1.815\n"
     ]
    }
   ],
   "source": [
    "X = [ [1.2, 0.7], [-0.3, 0.5], [3, 2.5] ] # array of 2-dimensional data\n",
    "y = [1, -1, 1] # array of labels\n",
    "w = [0.1, 0.2, 0.3] # example: random numbers\n",
    "alpha = 0.1 # regularization strength\n",
    "\n",
    "# def cost(X, y, w):\n",
    "\n",
    "total_cost = 0.0; # L, in SVM loss function above\n",
    "\n",
    "for i,x in enumerate(X):\n",
    "    # loop over all data points and compute their score\n",
    "    xi = x\n",
    "    score = w[0] * xi[0] + w[1] * xi[1] + w[2]\n",
    "\n",
    "    # accumulate cost based on how compatible the score is with the label\n",
    "    yi = y[i]; # label\n",
    "    costi = max(0, - yi * score + 1);\n",
    "    print(f'example {i}: xi = ({xi}) and label = {yi} -- score computed to be {score:.3f}');\n",
    "    print(f'  => cost computed to be {costi:.3f}');\n",
    "    total_cost += costi;\n",
    "\n",
    "# regularization cost: we want small weights\n",
    "reg_cost = alpha * (w[0]*w[0] + w[1]*w[1])\n",
    "print(f'regularization cost for current model is {reg_cost:.3f}');\n",
    "total_cost += reg_cost;\n",
    "\n",
    "print(f'total cost is {total_cost:.3f}');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}