mnist_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import numpy as np\n",
    "from tqdm import trange\n",
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the mnist dataset\n",
    "\n",
    "def fetch(url):\n",
    "  import requests, gzip, os, hashlib, numpy\n",
    "  fp = os.path.join(\"/tmp\", hashlib.md5(url.encode('utf-8')).hexdigest())\n",
    "  if os.path.isfile(fp):\n",
    "    with open(fp, \"rb\") as f:\n",
    "      dat = f.read()\n",
    "  else:\n",
    "    with open(fp, \"wb\") as f:\n",
    "      dat = requests.get(url).content\n",
    "      f.write(dat)\n",
    "  return numpy.frombuffer(gzip.decompress(dat), dtype=np.uint8).copy()\n",
    "X_train = fetch(\"http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\")[0x10:].reshape((-1, 28, 28))\n",
    "Y_train = fetch(\"http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\")[8:]\n",
    "X_test = fetch(\"http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\")[0x10:].reshape((-1, 28, 28))\n",
    "Y_test = fetch(\"http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\")[8:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# skip to the bottom for numpy only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "torch.set_printoptions(sci_mode=False)\n",
    "class BobNet(torch.nn.Module):\n",
    "  def __init__(self):\n",
    "    super(BobNet, self).__init__()\n",
    "    self.l1 = nn.Linear(784, 128, bias=False)\n",
    "    self.l2 = nn.Linear(128, 10, bias=False)\n",
    "    self.sm = nn.LogSoftmax(dim=1)\n",
    "  def forward(self, x):\n",
    "    x = F.relu(self.l1(x))\n",
    "    x = self.l2(x)\n",
    "    x = self.sm(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loss 0.15 accuracy 0.95: 100%|██████████| 1000/1000 [00:02<00:00, 397.91it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1314963a0>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAABdaklEQVR4nO2dd5gURfrHvzUzG9gFlpzDkgXJrpgREBVBMecsp6eenumnJ0bOHE49PcOJOefEAQqKAUVyzrrkzJKWJWyYmfr9UV3T1T2dp2dmZ6jP8+yz093V3dXp22+/9dZbhFIKiUQikWQ+gXRXQCKRSCT+IAVdIpFIsgQp6BKJRJIlSEGXSCSSLEEKukQikWQJoXTtuEmTJrS4uNj37a4u24/91WHDZb1aF2HxpvLYbz1WyyQSiaQ2MHfu3B2U0qZGy9Im6MXFxZgzZ47v271o7HTMWL3LcNmcJ0ag+O4Jsd9xdbJYJpFIJLUBQsg6s2XS5SKRSCRZghR0iUQiyRKkoEskEkmWIAVdIpFIsoRDStDLD9SkuwoSiUSSNLJW0JvUzY2bd+HY6WmoiUQikaSGrBX0+0/vge9vG4h6+Wpk5oqtFWmskUQikSSXrBX0nGAAXZrXQ4AQ0zJDnvkZ93y1OIW1kkgkkuSRdYJOwAQ8oOh4MGAu6KvL9uPDmetTUS2JRCJJOlkn6BRswA5CtMLuahty0A+JRJKBZJ2gc7iOW7lcJBKJJJvIOkFXXS7sv5XLxQxpoEskkkwk6wQ9oBwRN8y9WOhSzyUSSSZiK+iEkDcJIdsJIUtMlhNCyAuEkFJCyCJCSH//q+mcojo5AICDNREAqsBLJBJJtuNE7t4GMMxi+WkAuih/1wF4JfFqeaeoDutQtEfpFWpmob81bY3pNmSjqEQiyURsBZ1SOhWAcYJxxpkA3qWMGQAaEEJa+lVBt3ALvfwgE/SgiaA/8e0K021IOZckhZ2rZAONJKn44ZBoDWCDML1RmRcHIeQ6QsgcQsicsrIyH3Ydz0ndmwEA+rVroOzTfh1KKTbsOpCU+kgkAID1M4D/9AfmvJnumkiymJR6mCmlYymlJZTSkqZNDUdQSpgjixth5SPDcGynJgDMo1xEoY9EKc58aZpQz6RUTXIos+MP9n/T3PTWQ5LV+CHomwC0FabbKPPSRl4oGPtt5kMnUOeHoxS79lfHpql0umQOlALLxwPRaGr2t2URc524hVsJsl+EOWunAft3pLsWGY0fgj4OwBVKtMvRAMoppVt82K4vmAq6zkJ3wpTl2/DZnA32BSWpY9EnwCeXArNfS83+Xj2BuU5cw+8xKeiGUAq8PRx40yr+QmKHk7DFjwBMB9CNELKREDKKEHI9IeR6pchEAKsBlAJ4DcCNSautB5x0LApHtIJu5nIZ9c4c3Pn5Ij+qJfGLiq3sf3kaXrRjBwPvjHRWNhUW+n9KgE8ud15+9c/AmCJg91r7sjWVrOwsixfnC/1Zmac7s/8HlFiKTy5n58qKaJj93/mneZnZr7PtfnSJfX2NGFPE/naZR7gBYF9hY4qALQu97ceIn59g21zyBft/cLd/2xYI2RWglF5ss5wC+JtvNfIZJx1Fw6n6XJf4DxdIq4aPOW8BHU8EGnVUy057Huh/BbDsG6DDQKBxJ/f73jzPReEELPRtS4Gti4E+FwGLPwf2bQOOMXjkdv7J/jYvAFr1td/ugg/Z/3XTgYbF1mUr97D/U58GBlxrXGaX4orarwQ87FoNFDQClo9j09EIEAgarxuutK/v1H+x/ysn2JcViUaB359XpzfOBpZ+CbTsC+wsZfU6RrBDV05U/n8LtOwDbJoHlK0A+rp8kWxdzK5d8fHAz49rj2HnKqBNibvtOcBW0DMdYupDV3HqcpGkmYO7gdx6QDDELMZoGCA2H5nRCDD+VqCwKXBnKXBwD3vQfniQPdgrxgP5RcDd64H9O4HCxsmpd5R1dLO10KuVaKvcAnXeK8ey/z3PBb4YxX4ffaP5tsaeCIwp186LhIHqCiC/AatPXj2gej9bRpW6HdwD5BYCleVAQWPt9vkL0+zFGa4ymFfJtsnZtw2o15Ltv6CR+frV+1k9AKBqH3uZ1G8NVOg8ufyYAHYfBEJsPfE+oZRd4x/GqOv9MQlY8rl2Wz1GAkVttMe4ZwOry2vK10XPc4FQHntRFbUFgixEGgd2sTrWbaG9bv89nv1v3kudt1dpXoyobXZ+kvWC7sjlohN0GeVSC4nUAE8WA/2vBEa+ALx8FHMVnPIoW05NvrL4pzxvbHuyPRM1gAkMwATsz++BD84DLv8a6GTiHvByY0TCrN4xbO7HJ9qx/w8YNA5+eqWw3RogJIzKFbYRiHE3Aws/BE59DJh0DxOZbcpYANEwO7Yn2wNtjmQvutOeBo66TtifzfbLN8bPe/9creV9cDdzOUy+D7hlEdCwvVB/odyTHYD7t7PfjysR0IedHr/98bcA899Xp0N1gHs2s/Pd5xLg7FeAhR8DX1+vXU8v5oDx/bPgffbHKVvBztPYE4GjbgBOe4Jdh6c6qGX0L1JA/XIB2L0m/veZrO8Yb9axSLTc9Ra6jHLxQLga+PpGYPty7fyKbcBnVzFLK6HtKxbc/PeYJc39vnqXyy9PAysmqutFhHFkeRnuPtg4W1229lf2f8GHzMe56FP2WfzV9apY1hwUthu2ru/GucC7ZwLbl2rnz30LWPOrWp+JdwIbZqnLozXszwjR1SAK4LJx7IvDjG1LmZgDTMwBVcwBJmbc383PybJv2P8Ns4HxtwsWtMmzsXl+/Dy9G2Xmf9Vrs2cd+z/vXeCpjsBzh6vlIlXsntkn9FVZMV67rf07tGIOAOGD6rlb+CHz3evF3IyaSmDRZ8D0l83LTH2aiTkAzHwF+P0/7NzqKVsJfC24xGoM+riky4ee6ZjlchFlXm+hA8Cqsn1YsaUCI3qnrdNr8ijfyKwyO7+pHdUHmNXSuj+wZiqw4APmP730M7a8shz46EL2sHccBBxxlfd9cQuRRoFfnxEW8CupXMOfHmH/uaXELXRQ9qlsxkYlPnzxp+z/l9cCnYcCpT8Avc4DcgqAJl3V8lV7res7+zXW6Pi/W+OXvXM6q1+4Cpg1lv3dttS4cdLsRbh1EbNIm/cAPjVoCN22FMirDwRzVZeNGTtXARWbtfO4CH19A/PLNz2MTe8vY8cVzAVa9GLrtuoL7NWtb8S8d4HiE9hv7oIad7Nx2aVfMf+2Gf85wnj+6p/V39x374TwQeDLv7DfjUzaU5b/Tzs9+T5g4J26/f8CvOugobxOI/syHsh+QTex0EUJj+gaRSkFTnrmFwDAiN4jklW19MGtIaPPQzf87xYmgP/3J1CujPxU2Exd/t45quVm5+u2Q/SxiteUb5dSY5cIFw7AOtxw3W/x8/Lqsf+z32ANZcfdoi6zs7C479is4XTHn0BhE3VatFBFPjaJSXhbuS/7XWa83E7ERaa/CLQdoJ3Hv2watGOC/q0gXO+eyf7nFDDhv3ersQ/dCH69ojZfOABr6zCDf2Xp+fACZ/XQUyN8TYguEjumPq2ddiLm7Y4FuiUnPDPrBd3Mhy4m4IrzoSe1Rink/fPYA3n6s/Zl3z0LaNGTWYT7tgMXfxhfZtzf2acuX7ZNScC5b7uxD3fTHGHdm1lD22GKEL1/LrN+B90DDPqHff0igmBMf1H9zcW9qgJ4XOjf9vpQrUvFC3Uasv9bleOcJkRKiILy4pFqT9BLvwC6DGX1sWLa88ytY8bUp4EfH7Gvo97t4JXJ92mnty1mricruBU/4f+0vmYzcgpUQf/gPODK/1mXTyVvnpK6fQWTJ7tZL+hmUS77q1XLbe9BB9ZCprBzFfNjtzsGKP2ezXMi6Kt/Yn+cOW+yhqWcfGatzXuH/XG2LAK2L2O/qyqARR+z31x4F30Wv4/J97Gu72umqmL782PAiXexbefWZQJdvzXQqj/zlx82AvjxYeb+MES5vgt1L6BExRwAQvnmyyb8n/qbizkATL4XWPaVscUvMv896+VOxNxP9iQwtq5TMa85oL3H3jmD/c8rAqqS00ioYegYbbRLughIQfeMkzj0eeu1n88Zkz6XUuZSoFEW3xsIAu+dxR7Otkep5aJRJpTRsBpqJS4z+iYZfxvz6Z78ELOI9Q/Cqyeov/dtU10r4SrmbuD+SJFdq3X+b4XSKcx9IzJoNIvdnXA7mzazRCMOP/U9odw8ojWeW4+Fypm5UspWsL9soEVv5qv3gwbtzM/LOa8CH13kfFuXfwW8d7b7Ohx3q3tBP+u/zhtWnRLIsS/jddNJ23ItwSzKxapMhsg5MO3fwMONgUeaAg8pjSyVSmPdhplquU8vB35/AXi4Sbwr4PWT1HX1VChhfRtsrF0xsqV6P1DtMqLFyB+6b7uzdfWuAjc066H+7nxy/HL+shAbQPXx09lE/yu005cahPd5pait+bLCpkDjzup0y77a2G09nYYArU0aRa0ghLUbuWk76nsxcI9BJpMhyn2XZ+OWMkIM1/SZrBf0gAMT3W1v7KRY8BtmAc8cxroIT74f2L3OuNyiT1kyKgCY+4522dbFxuK4Yjzw/QPst9id+ecnrHs70ihrvNP3zNMfP3e3AMCqKfHRAHbwuml3YlyWx5D7QU4d9bdRaFlNZfy8giR0PKotiA3IQGLHeuzNwBkvAF1PY9M0YlzutKdYj8mrvwXOUsbGidQwVx8AHPt34/X4PdjrAuCWhUAfyw7t5oz6Xv2dU2hcxsj1xsvauYr+OjV+3qDRzurmgawXdCcWekgn+nZ6vaW80p2oH9yt5hwBWAzzDl1I1k+PsZ5wPz/OrOmxJ2orUraSTX95LUtGdWAXsFuXk2LOW/Z1WfKl+pt3RzZj1yrgRYPuyfqYY3243Xd329dDhPeeExFjvkUO8zHqKJjH/ufWA4Y9obXYgfgYcgCo38q//SeLXhcAnU5yv15OARPGoWOAI6911ninP2ecRp2AI65knW+6j1TFWk8fxdVSt5lqdUeq1Zdtx0HW++95Dgu/7eoxaqTtAOCqCUDJqPhIH04gwNqkRJoLx212bDdMZ18avS8C/vIjcMxNwMUfJ/UrL+sF3cmYovqG0zd+tYhXBnDsEz/i3ekmFrQR/+4NPNNNnZ58H/DiEcBe4VNOH9d8cDdLRgSwXowvDQAWCw2NYu80jlmeDBE3AyyY5e7mXaHtOPUx5/vSs+QL4/n5Hj5xzQgpgn7+W0DL3sDZ/1WX5dY17izD47HdcvHH5staWLgXjBisczP1OEs7Pehu4PIv4Zq2A9g5OP42YMS/4pcfc1P8PNFVYkTDYuDC94B6LbRx/Jy8+urvQmWMhG7DWIw9wNpkzh5rsGHF2OFtQok0NBYfzwIH+FcEj8QRXybXfKe6ahp3BtoerS7re4k2dj23LvvfvAcToHNeBdocAZz6KNDtNO/1dEDWN4qaRbmI6MMWX/jRokODwrTSHbjy4HusN9ypj1oX1ov1H9+y/zUHmGiMu9n4837zfGDKQ2pDYtlK6/04SXCUyhaCus29r2vW1dwq8sSIhh3iv2Ri21IEncdEhwQXjFk7gJnvlkdqXPwJ60wVzNUeg96dwek0BLjkU+Zie9HGL1zYlHXsqdNAOz9X5yqoZ9AZbvi/WC4Sbgjcvhx4tjv7fcdKVj+jr4+7NzCBK9/IkpuJIaMAa+w0gp9bkWt/Yl35g3nA3+ex8y0+nwWNWJ+GgsYsRBYAQIE+FzLRfc7ga4A3MOr7OYx8ERhn8AKygvdHufQzoEEx0MDA7/+Pdeza6o8vR8nhctNc9vIyS0WRZLJe0J24XPQdi5xAAeBXxYrRC/rutSxeW59NrWIrE+VYUiQKjB1kvpMNs7TpREW3jRFiIqTagJE127ynGr/uBVF0TrhDGzVTfAITmAUfsOmBd7GQQjNBDyq5UPiLUPSpc4b/C5gohCh2GgwM+CtzEQRzgIWfsIazuW8zQW/UERjxDND+OOBlxYo78R9al8DJD7Gvrz3rWUKoYA7QpDNw+nNMjH98VNs1n1PQmAm62I5w2RfsJcOPefi/1ARR577BxH7zAqDkGvYFd+V44MBOdh7PeJ5lE6zXwvj8AEC+YkE3U67l+W8zq/r9c6Dpb93hRGZNb5jBpnudH7+tvLqsfh0Gqomw9NRVOqad+ihQ1BrociqbLmoNXPKZtiMZoF5D/dcpn2/GOa/HCzYX4VAddj2M0L9MORe9z9qOGndK6yAm2S/oHpJzOcHShf58H/b//p3aG+3ZHuyzjjeo2IXc6XND28X7HtwNtOrHbsj1v1uXTQX84RTpNDgxQeefswBw0gNaQW9+OHDak+zirJoCDLkX+Pya+G207Mt6gXIri3eK0gt6YVOWKpYLen4DVmb4U2oZ3ns0rz7wv78D9ZoDR+pCNgcr+VNC+UD3M9g6OQVsu2IIW4lS126nxXfq6Xmu2nNSdDvx+Py2R7EIJzG1ba/z1O1xOgjhpl5SMRx+tnrzDxqtZjvsNIQJ2YYZzDWjD4/lmKXe1VOngXreOF0NOv/w/bTozf6XXMPcim1K2DXpcjLLf67/4utt8MI5chR7boxcQ3Y0LGYNwWkm6wXdrOu/iH6AC86gwAJgjJID+by3AKifWZpG0W//wRIPAczq4TzeWusG4T66GsVCd9M92wlrf2XWjx/53Vv1M/YhW3HpF8AH57Lfpz2lfobqGVMOPHs4sNcgQ59Zec5KxV018C72n7shANUqO1topBI/xfMbsCigv7K0DvhJ8fHzLv68vqF84L5t8fW426Ld5Igr2Z8V4ja5C8bM9zumXBX1Zj2A895U85foXSwAMGqy9b79hIf/ASwii1NP+XoyepH7jvL88etbv6Vap9OfY/9Huxz0pNd56kswQzkEBN2+jJmFfmnwB3VixssAbsPgwHzUIISh5UJnpJlCY9rvgo/RkU/bZ/LqG2eASwWdT2Iug4otzC9tJOjcGnYbq87pOoy9XLsrvQz/+ivwrOIOMPKvi4J+w+9a98vAO5lbiFuwuQVs2w2SFyccg2cFtGrMO+8t4POrVYuYvwSMXENphzIxDASA7membrfpHKP1up/NQx3TRNZHuTgbgs6BRaskH3or92m8n/s4Lt79X+Nydr67ZCB2CAnlaUezOfwc9Xdfk0ROfkEIcw0ALJbYKMSIu5lEQT/iKhbaVdSOhdxx6jRifuC4fZyjfmrXb6mGzvEoCU15pQ7BPOaHLT5eXRbMYdsSRaHnOSwiQeTMl72FAVrR8zzmbx9g0KOW02040OxwYLiSAOqMf7OImCZd/K1LIojpi/n1T2Kukhi1oTd3q35AUw/umSSS9YLuJMqF50M/oUsTzXwqNvqIebWtSMXNrKed4LqpPqD1UxYfx/53HAyc8nDy69KqH/tv5ofkFnqnIeq8M55noV23LQbOFcasHPW9s09gnlPGKAKllZJh8Yqv7bdjRr9LvYUBWlG/JfD3+eqweEbk5AM3/q76vTsOAq7/TW1HEM9h2kmXwMpBt0WkywWqy4WLfx6q8WbO0zguKLgunOYM8XNgWTtumsuiENYIvdH0roygEF6lCbUisHwI9fkmblnI3AM8zesNvxu3AfQ6D2jWnTVQAsCti1lUxbZlwDc3qvs8/x3W6GuVF9qsYc2MFj3j5w24lglis+7utlWbIYS9DOpaRKekCrOwwWTD7410ulxqIVkv6E5cLnUPbEIbsh0Bwj7ZTwrM04o5YD/EV7Jo0F4d3UVP/VbM7yu6efQx71zECdGKOwmojbRHXc+iJD6/mk0HQsC5rwPP91bL6wfD4IJthLisQTv21+xwYMsCtTEzt4CFzFlhFMtsxOVfAdtXGJcnJLvEnGNl2aeS425h0VUDrrMv6yfnvQnMes0658shSNa7XJxEudy27Dz8lndrrGwEBj0uI1UIwKfOAmIvMyPEhD/6EVFEuIDlCaF8og8a0Iq96A4SLarTnmS+Y85fp7IEQi37xu8zp8C+d6BhXXOZL7iugZ/bDKftEZ2GaEdtl6SO/Pqsl6VR5E0yaVjMYtWddAU/hJAWukAAFEvzrkYhMXCv7C/DN7kJZPYTadBO7YBhxF2rWRZFQDuKOAA06QbsUHqM8hj3Foql27yntkEUUCMi9K6Nhu3jh/i6fweLpOCJka79MT4T42iHoYZ+4NRCl0gkAA4BQa+b5/wQQyRqLOYKvQJrvVeEBFUXx5kvsdHVvzWxvkVLWh/6d8XXapdtTmFjFg3SdkC8T7H9ccDge9XGxQs/YI2I/S6L304wR+u3NsoNI867ZjJLX/DeWcbHkSjBDBf0y78yjryRSJJE1n+v3DDIaMBXihzEj1KUC4eRLF4Y9T0AwjK7hXKBkqvjy9RvzcQXYO6OY//OfNs8VnnQaOY37zZCG9kCMME2yquRV5eNCMR9rt1PZ9NOswZ2H8lC7Ixod5Q2DNAvRjwDFDRJT8SQn3Qa4j7xlkSSABn+xNiTnxPEgOJGmLV2V2zeVcFJGJPzLkoqX8EOqP7qpAn6//3Jes+N2aPOM+pQcvsy9TfvzQgAD+zUljMa7zNZXGgzVFoyhtM68i/x3eclEoktWW+hA9CEqrYqysfVhdMBAC3JTrQjanfsXAOr3RHnvG693Cz6IhvIluOQSLKAQ0LQxXbRu4d3R7MaNqBCFAQlRE1JGzIQ9EdqLgUtaosdjY8034FdbzG3KV8lEonEA44EnRAyjBCykhBSSgiJG46GENKOEPITIWQ+IWQRIWS4/1X1jhi6GEAEdSgbboyCoA5R48vzqTb3SufKd/F6ZAT+N2QySjbdZr6D+q2N5zdXOrqkIx2AQxZGa0k8s0QiSRhbByghJAjgJQAnA9gIYDYhZBylVHD44j4An1JKXyGE9AAwEUBxEurrCVHQc6Kqn/zO0Cf4Lao2WhVG1QGU7675C8LK6dm1zyTy5cR/sA4VhU2Au9bEjyJ03c9sUOZa6pboVfk6qhGCzbAZEokkQ3BioQ8AUEopXU0prQbwMQB9OjUKgI8lVQRgs39VTByup73JKhRV/BGbPzi4EPWIOjhw26pVsd8bqBpuZtpBngSYmANstBVhlPTVp7zFQgCdjh+YBrdMBQpQBR++Hs5+1XgwXIlEklKcCHprAGJi4Y3KPJExAC4jhGwEs84NM70TQq4jhMwhhMwpKyvzUF1v9GnTAAAwLu9+HP3jhZplt4bUpEutq9QBJcJU/XgxS+y2qEA3cGyXk2M/9zTuZ1+xvCI2YADARrHJVPpcZN+NXyKRJB2/GkUvBvA2pbQNgOEA3iMkPlsPpXQspbSEUlrStGnqOlzcdrKzFJdH7ZkY+10jdP+PGij6r5Ge+KHcYOxGhUo46BQzej0b0mtMOXDUXx3VUSKRSMxwIuibAIiD77VR5omMAvApAFBKpwPIB9AEtQQ33f85YaN8LgJRm1N3IGK9vlu+mr8RSzeX2xdMIfd9vRj3fGUw9qVEIkkLTgR9NoAuhJAOhJBcABcBGKcrsx7ASQBACOkOJuip86k4wG1iLVHQuYFeJbhhwggaZmJ+rM4d+DHSF/urPca0m3DbJwsx4oXffN1morw/Yz0+nLk+3dWQSCQKtoJOKQ0DuAnAJADLwaJZlhJCHiKEjFSK3QHgWkLIQgAfAbiK0towpIjKfSGbAZZ1iGJNlWbRDyJDY/PMLPTpBUNwTc1dOFgd0czfsa8K4xfVqrZiiUSSZTjqt00pnQjW2CnOe0D4vQzAcf5WzUeiEVwT+s508YDKlzArX5ulULTo+ZCjD4cvw3baAHfnfIxKGA++kJ/DhP6ATtBHvT0bCzeW47hOTdCwMPlx6X0rX0UQUcxN+p4kEkltIft6iu7fASz9CogILo8NMy1X2Y6GcfOCGkFnik4RwOuR4XgzPAxjaoxHeM/PYa6agzVaQd+w+yAAIEIppq/aiT0HEh8wY/LSrdi133g7e1APO4U8NRKJJPvJPkH/7m7gs6uA9dPVeW+dZlp8K2VivoPW18zfRNU2XdF5FEYID4WvMBVL3gC7v0rrQ+ceqHCE4uLXZuDKN2fZHooV5QdrcN17c3H127MT2o5EIskesk/QtypRFzUHrMspXFpzDwDgyKqXY/N6V45FGRp42j13z+gtdBr7z34t31qBROADW6/fuT+h7Ugkkuwh+wS9ShkkOSK4IlqXmBYPK/HiVDgV+hjyaNR5+y63xPVNwvppN4GUVu3LtarlWSKRpJXsE/QKJZIkIuQ2D+ZieVQ7+MNByhomowb5vKt1bcVuRJP72yO6l0BUJ/RutqnfFuDuhSCRSA4NskvQ95UBVGnMFAU9XBnzleuJEKNoFa1cmhnIRjm3uPjG9S6l2uVuiNSuCFCJRFJLyS5BP7BD/c2zKu7fCWyehxBUn/aYxs+AKApLHYy4Q13Y01yv9bpNY8s9CLrFS0BqvUQi4WSXoItWOfeh//AgAOCE4JLYojU56jijUWLfRd+NUc393Xq/OzVxxTjBWtClokskEkZ2CXpUFHQlbLBsBQDgAFUbOmtIDr6MsMGNRZfLujqHG2/XhWiqFrpO0HXL3TjRowZZC7zK+B/bKnyJgZdIJLWP7BJ0sTPRDw8CY4qAjSxOu4BUoZIy8Y6SIO4PX4Pxw6cjQlSXy387PI+elfHjg5qJJzFomow1iuoFXZk85+VpTo8mRthA0dXOTu445bmpGPmi+zpIJJLaT3YJumihG8Shj6h+DLuGPgcAiCCIcE6RZjSjMMnFPhTErefGq8FdLfFtomzG3sqwZtoJ4svh91U7tNv3YKqv33VAumokkiwkuwRd9KHr+E/0PKyirXGw58WxeYQARBB0M1e1XUPm0s3lGLdws2YbcS4Xj/q5a381Bjw6JTa9bS8b99TNC8EIqecSSfbhKDlXRjCmCGjWw3hZn4vx6oKzAYQR1MUaiqnSzUTSSvu+W7IF178/DwAwsk8r0zh0rwK6aOMezXRuMJjQ9mL1SWx1iURSC8kOCz2qhCRuX2ZahOdYCQRUMQwQgpygcApMVM4qDp2LeawqvOOQicvFLfoXQ06QaLbvVZily0UiyT6yQ9DDlebLmvcCBt4ZE3SxIZMQVSABc3F0I8axsEUblwufrg5HMeDRH/Dt4i2G2wvrBD03FHBdJ8N6JrS2RCKpjWSHoNeYCHqPM4EbfgMad4o1foqWKYHWQjezWs2MWe7PFuEWdZzLxaTquw9UY3tFFR4Yt9RwuX47XNDVLwFv0iwNdIkk+8gOQQ8f1E73V3KV16jzC/OY71nUR0JUgQTMRfftaWsN539gMPwat8wnL9uGDqMn4IAyFJ1eeLkrP6R8OZh1HtLP518Y1GPYIidRC18ikdQ+skPQ9RZ6xxPZ/91rY7PeuXoAbjmpC5rXVzsYEUBnoRtvvjribDzSaJRqtkEpsFEZ2CIuFYAyzaNswib70Megm2VzdIu00CWS7CNLBF0Xc96qP/vf+4LYrOImhbjt5K6aMEWnFrpTwlHqOpFWbOALEws9HNFnbeTraf+7xU9BH/bvqXjpp1L/NiiRSDyR+YI+503g1RO083LrAvfvBAbeabgK1zJCCHId+NCd8uf2ijgBtoOXNltP37j6PyXePfFGUf8UfcXWCjw9aaVv25NIJN7IfEGf/ED8vEAQCFqE2HN3B4A+bYv0sz1z+n9+Q43OdWKXtzwas9DNXC7aWn0yZwNWbq2QLheJRBJH5gt6QaP4eXn1HK1KCMFtQ7uqM3wQSTPXSVxZYR3AvJeqUWPp/uqwkMvFY5SLp7UkEkltJvMFvY5u4IrLvwaCRoNWxEMAhESXiw8yp2/cXLG1wnoIOZtdmrli9C8Et6SyY9Gu/dVxXy5+8970tVi7Q46vKjm0yXxBD+jymecWOl5VP+KQidfDFTz5Fufmj+bjSCEXix67PDFm4Yz61c5+eRrGL9rsrJJIrYXe/+HvcesnC5K2/apwBPd/sxTn/Xd60vYhkWQCmS/oVKfCOfHZEuNWUeQsoFP0ZMVm79hXZTj/k9nrMXPNTst1jVw4+6vCWLeTWaMUzNqev34PbvpwvuW2RKs8VQY63+eERcY9Yf3ZB/u/t9I8OZtEciiQ+cm59IJet5nzdXUWercW9TFp6bbE6+QASin+8cVi23JGFvzlb8wSNuR8RCXNplIm6KnZDyAHzpZIMttCX/UjsGWhOn3bMleCrheAq48txjd/O86fuvmEkyHrnA5rp9Xz1ChtKvYiI3YkEoYjQSeEDCOErCSElBJC7jYpcwEhZBkhZCkh5EN/q2nC5Pu100WtXa1O4lLpEvRp2yDBSjnDqQY5ESunA09H0+hySeo+lLOpbxORSA41bF0uhJAggJcAnAxgI4DZhJBxlNJlQpkuAEYDOI5SupsQ4sLvkQAJPsH6tUkGfq9QUOcWupiWIEn1idtnCvbBD99oSECJ5FDCiYQNAFBKKV1NKa0G8DGAM3VlrgXwEqV0NwBQSrf7W00TSNC+jNXquudfP/hFpuDUQhfdLE4t5+0VldheYZGe2G6fKVB0mdtdImE4EfTWADYI0xuVeSJdAXQlhEwjhMwghAwz2hAh5DpCyBxCyJyysjJvNRbZssDTauIAFyLBQOYJOqXOwy3tLPSt5ZWx7JCcAY9O0QyB57p+KbDRYxZ65l0+icRX/HIyhAB0ATAIwMUAXiOENNAXopSOpZSWUEpLmjZtmtgeq+MHgXaL/vnXC7yeYzo2TniffkMB1wnBAGPL+ejHp+CS12YmXimb/fgNt9ClnksOdZwI+iYAbYXpNso8kY0AxlFKayilawD8ASbwySMati9jh97lYmGh92xdH++NGpD4PpOANx+68ToLNuzxoUapRZ+KWCI5VHEi6LMBdCGEdCCE5AK4CMA4XZmvwaxzEEKagLlgVvtXTQN8EHR9I5qVx6UgN6RJE1BboJR68qHr9TxZXfOHP/9rUrYrEvVooUeiFL/+6YPrz4S/vDMb/5JZKCUpxFahKKVhADcBmARgOYBPKaVLCSEPEUJGKsUmAdhJCFkG4CcAd1JKrbtAJkqkOuFN6A26RCy8JnVzXZX30xXhWNAtfOgHqiP+VUhgdQryq8Q+UFxevrFTV+PyN2bhxxXJ6Uz2w/LteFHmiZekEEc9RSmlEwFM1M17QPhNAdyu/KWGiK6bd7cRrjfh5vm3K1s3L4Qd+6rRtF4eIlGKXfsTf+EAzhoVPXUs0q1SWcMEvTa0Cy/ZVI7Jy7bh9pO72heG94ZXnsxr+17j1AwSc75dvAX92jVEi6L8dFdFIlD7fAhOES307iOBi533ZeKPf8BH9aqbz96Ng7o2xfMX9fVtu3bGN4WbKBchbFEngtxCr5OTWCio2f7cMPLF3/DClD8Rddk24PZqZtq4qmOnrkLx3RNQkeacNdEoxQ0fzMP5r/6e1npI4slcQRd96Ge+6GkTich5/Xztx03dPDYdDBDPHVyMBPDnldYh/ZQCA5/+ydn2deuJ8HDFOrn+Cbo+sdhT363AngP2Xy58NafROzEfukeXWaa0pb4/gw1KvnOfP19/XuFXhY+XK6k9ZK6gixZ6fpF5uSRRL1+bc71uHpum1LvbQi+Aew5UY+HGcm8bM8DKh85dLnkh/wS9Oqz9dHj551V46H/LTEqr8PPnJXrHDZnWH0n/4qGU4psFm2LXLlV4bYSWJJ8MFvT0fnbm52hPXT3FYq+ORD3f6aIA7qsK496vlniunyGioOvUjA+k4WfnqqpwvC/IaJ4e3h/A6ehPqoXuonIQxpbNMGmiAF76qRRv/LYGt3y8AE98uyK1+6dqPSS1CynoDtHfvPk6XzN3uVSHo7YdlMwQXQyv/FyKCYsTzyF+2etqRyGr5FxcO9fvSrzDFkdvoTuFn7+IwwG3vfrQY2SInovVfHrSSjwyYTkA1sM3lWRa28OhROYKetS7oNs11r14ST/bbeSFtKeuUBH0qnDUsz6IjYA1DsXMjt9Kd8SO12qL4jmZu263L/s2EnQnYhBQTq3ZwNlx2zxEXC6cdOeuydTzdiiQuYKehDh0y7K6ab2FzqNDopR6jp7hev79sm2Y+od/HV46jJ6I0u37LEcsEr0bfDSkRKkKe/Ptxix01y4Xd+c9lnbX1Vrpgx+f0wFNjLjktRnoNWaSL/WRwl77yNwRiyJKlMtVE63LGeDlPrRzudTJZe/GSJRqBCI3FHDseuACdu27czzU0JoJi7bg0qPbxab1lrI4bSWk5Qdq8OSkFfhy3kbbfTrxlxvh2YfuaW+ZlzJAb6G7cYH8virx/n5SyGsvmW+h59VNYCPeH+QTu2qTi4kWuigQuS7SBTjt8emFnfurNA+iXivFaaNqUEpx/9dL0Oehyfhw5npU1tiLtVE6ASeHyE/fsU/8aF8YQuOm+0B0tp7L1dKN/hSmWmAPRR/6HZ8uxNkvT0t3NWzJXEHnPvSguy73XtE/9Fcc014zzS32SJRqhCUn6FwubvpwniPL147P5myIm7dzfzXmrN0Vm96ga/wUXyb895JNashkTYTivRnrXNXDq2vAbaSNarG6W89rdEy60b/4Uy2vibh8MpUv5m3E/PV70l0NWzLY5aIIeiDHupxP6O9h/Wc675Cjd7nkuLDQZ6/djdlrE2+QvPPzRXHz5q7djQmL1KiZK96cpVlONYLO/p/+n9/U5R5kw6jxzok16XagkUTzoWeKoPNqOu0ZnCzS3SgrMSdzLXQu6EH3gp6M+zE/JDSKCgrhRtCTSdk+63wlokgY9dD0cs68nmbXjZsGO4pEKR6dsAzb95qH9GWqLCXTNeeETD1vhwK1Q228wH3oHgSd46dlZuZyyQ3VjlNs58XQxqj7JOhGvngDOYhGKfZXqakc9HWdVrrDMr0vb8QVV5tWugOv/boGo79cbFtPq45FU5Zvw+n/+dVxxE1SUaqpP68p96HXglMhMaZ2qI0XeC6XJPjQrW7Yb285Ad/fNjBufp7SczRCtSMf1ZbPeTurVzxko6RYXqxCp+s8NH4ZDn9wkpDxUa3rvPW7cenrMy3zinP3kXiIXICt8sE4qd6tnyzAkk17sa/ShwFVfCL+vKa8VVRSS8lcQecWeiB5zQBHtG+IRoXaF0b3lvXRpXm9uLLctaIXQ6+9Rv3GrhZGPnQRv4a501NZE8Hbv68FoIY5io2iPBHVqrJ9ptvYYeBO4qKnP//z1u/Gd0tYW4Kj6Bhv7a2e+WbBJnQcPcEwP0vMh65vFJVRLhKFDBb05EW58Ae8Rf18PHVub0frhAJqZxiNhe577bxh914RRdzIsqYeGuKcNJ4ZZezz3LgpnG1+PHr3zTkv/47r35/nodOStzq55anvViJKgbIK8zaPdHt/0r1/iTlZIOj+R7mIOsS7oVs9z2ufGBGzKlkcurB+LbHQrSJHDlZHNCJeFY7GifG9X9v7ovUYPffxGq/O4FX0es7E1ex6jz4yYZmjF06qtYuY+MnZMqIsS2/Yooxyqb1krqDzOHQPLhfHtyMRHiKbokEzCz3Fel5+wDjHjZUPvfsD32HDLtVSfnrSSrw5ba2mzPhF7hOFGTeKajGy9vxI+EhjLhfj5V/M3Si4XMx3yF8MTr5Qyg/U4OyXp2H9Tu8JzmKCbnHH6c9ZqgVWynntJXMFPVLNYtATUEzbNalzl0ks/0iaLfR7TCzpfVXWjXqrdT7qh8fb5y03oqyiCos27gHgrFFUk6NdEU2jXDhONEs816rLxfj8i2kFLF3olG/PvgITFm/B/PV78PLP3scR5fW1cmukO+JGGui1lwwW9JqU9BJ1Ksi88XRgl6YagQik+AzvtIk3N8Mvkbjl4/kY+eI0lB+ocWTJiZboy78wIQz60AbBj6c6HMUbv62Ja6yuiUQdmZq8SKpiv/nxWlnd+rpUR6L4YVlyBro2QjaK1l4yXND9iXD54fYT8cPt8aGIgDtBn3b3ENw3orvmE97r4Ak9Wtb3tJ4+aZhTnCbCsoPn5l66pdyRCIodml79ZTUA43MubumbBZtw52cLrber7HvKiu14ePwyTFyidRnVRKiabdFBlIuT0+PHx1jAwsXHN69/+U4r3Ym/vDsH031IvOUIqee1lswV9Kh/FnrnZnXRuVl8KCKIO39u6wZ1EAoGdC4Xb3Vq16jA03q9Wxd5Ws8vC52P3HSgKmL44HONr4lEFSs+vpB4/hZs2BO3/JaPF+CzufE5b8T19O+Sg9XeUvny+vntpx63cLNxnZRjEPPo6DELIXUyXqsfSD2P57slW3HcEz9adoBLBZkr6NyH7gU3D6cHQdZYmB7NNq+5xOvkevtqcTqYhBWRKEWe8oWwvzps+Wl+x6cL0eehyYaXQjx/L/7E3DCOIlJEf7wD2VFHOrJqFNX+N+Oln0pRut08Vl5kztpd+PtH8/HP/y2NW8ZrcsvHC+KXKQuNOn6Jy5ONG/cTpRTbK1I7olI6uO/rJdi05yB2p+ilakYGC3o4YZeLWXQDjynPCwY8uUzENU7v1dJL1Ty7QLz6ev0wLO77ekksjfD+qohJEilWv3ELNwMAnv3+D+1SSj2Payp+Zej3bXStY4JusTt+PsXzWrq9AiWPfI9tSp6YypoInp60Em/8tsZRPSuUXqdbDIaOs3Lx8XvR/GsqNYru5hZ7f8Y6DHh0ClZurUhehWoBgdjLNs31SO/uE4BGktZL9OQezXH9iZ3wwBk9PFk94kP5lxM6eKqD53FJozRueDwn+NHo9/Hs9TFBP1AdtrSR+eH9uGK7rh7Gbiq+rRmrVT+xviFQdEXoj8fobDoZschoQOR3fl+HHfuqMWnpVk0ZtxhdYnGemO5YxOxapcpCd3O4v5XuABAfRQUA4UgUvyvL/WTl1gpDV10yEfuhpJPMFfRoGCDeGgDtCAUDuPu0w9CgINfTw6r1uFg/Zb3bePN5mxGJUhTkuj8vfjSKUgrkKzlt9ldFLN0kIRMrvCocsTxnYh73v+hGdhJdEfpdG23SyELfua8KxXdPwPhFm023rcfPh1h8kZ/33+maZXxRmt20vrUnPD/lT1zy+kzMWmP84nLKd0u2aL6OTv33VJz1UmoHozBLy5BqMljQI0AgOYKeKG4spVHHd8DLl/aPm+/1tqCUoqiO+7aFiE/filyMD1SHDf3O/H4PmcRz9nhgkmVDspU7Rnwp6RsOra+JuvCPbcySfHf6Ok0JK/+8lzw3iWC2P7NDrApHcPVbs7B8y15f9u+uCco8aoe3OVilOXDC9e/P89xvwi/UXrxsunT7Prz+6+qU18ORoBNChhFCVhJCSgkhd1uUO5cQQgkhJf5V0QQaTZqFLuLlM9bMwrz+xE6x360b1AEArN1xAK2U3yJeraAIpZ4aRsMRf0SJ+3dZoi0LC91iJCcr0bZaVn5Q7SXrqFOT4TxjN4zR9mIx4wbvwvemr7XYr3nd9PfOsY9PiRsw3Oxr4br35mqmN+w6gF37q7Fk0178tLLMUSphv7FOZZDauiQTbp/w++Tsl6fhkQnLEU7x55StoBNCggBeAnAagB4ALiaE9DAoVw/ALQBm+l1JQxKw0ENKZsRk3U/67fIb96TuzWLz7hrWDQBwVMdGrkfosSIS9fYy8CsOnd/QNZGosYWu/Lca+GOdQdd5fkh2Daal2yuUetjX1bBR1KSh1OoFoV+2ubwS938TH8Gix+hI9Ie3ubwSj0zQWp9OQ0xPeOonDHj0ByGdgD+Ih7tzXxVGvT3bNGTSKpVBIh82m/ccxLPf/2F5r89aswtrd+z3vhMX8GeYXxqe3z/VDhgnFvoAAKWU0tWU0moAHwM406DcwwCeBJCaGKVo2LOgP39RX/x1YEf0chCz7eWm0zdo8ostPqxn9m2NNY8Px9EdG5taKsd3buJ635RSTzHlc9ftdr2OEdzSr4lELc+dmQ8dALYbfIL/8kcZKmsiti+/ypoofv2zDJ/rxlV1Gq3Eq6wvbzWItt4FYue+sjovVr5+tS7Or29YzC3kk2tI3P9rv67BlBXb8eGs9Z6358WeuenDeXhhyp9YvsU8euaCV6dj0L9+9lwvN6gpG3hntfQ0kjoR9NYAxKdjozIvBiGkP4C2lNIJVhsihFxHCJlDCJlTVlZmVdQeGvHscmnTsACjh3c3zBniB/oblO9H/zlt1finvw9+uXOQo31HojStDTPfKZEfYk9MEW5RWQm6Ga/+stp+oA4KXP7GLCzcqO2YY7waja0jrm9U3tASTMJDa/Ti0W/f7f5iriFh3s0fzce6ncx6nbJ8G87/7++Ov+zEUrwu28orsXF3/JdVzIdu0Z7ihYM1Uc3+0436ztTWJ9XVS7hRlBASAPAsgDvsylJKx1JKSyilJU2bNk1sxylqFPXmQ9dOc6vSbFPVBn42vRi2b1xou9/cYAARSmtFvurNew5i217zxq6Qh7FWyw/WxL0IZpuE9ukx6qijIkTHmKQDEM/pMl3jop8PrdV7jr/MvLplxXr+b+Fm3PkZG0z87x/Nx+y1u7FXiY/fVxU2DDNUtyPG+7Pf70xfh+Of/Mmg0so63qpsi59++Pnrd+OU537BgWr3o1MFTK5NbbTQNwFoK0y3UeZx6gHoCeBnQshaAEcDGJfUhtFoFFjzC1LRkcJT2KKuXlyEzGLLq8P+NJyEggTRNFvonJlrdllGHlg1ippRE4nG+dD36NIFn/Hib463F4sxF07X6jJmtRIQzbB3XOinle7A/PV7NNtx6uJaXbZPkzzN8GvDJo8N4D6qRs0Po11vq9IxqkEBS6Gxaz/zg1/2+kwMeeYX0+2JW0l1hE8yeWzicvyxbR+WbLKPBtpbWYPiuyfgcyUFBT/HB2siqKgUG+eTU1cznAj6bABdCCEdCCG5AC4CMI4vpJSWU0qbUEqLKaXFAGYAGEkpnWO8OR+Y8TL7v2FG0nbhhPdHHYU3r4p/b+mtLC5eZoJulP+BUvfWRyhAEKX+p1fN9WBN2+HlVVwdjsa5yRLJnUF1/79bshUPjmONmXsra2JpBwC1B+D6XfFuBacv0CHP/IITn/7Z0kgw7FTF3UCxurh0uZi40Hm4YIMCFubKXzZ2nXKMXFScx79dbpiq2cqdU1uCXdy8m3YrLz+evoGf47NemoZeYyabJlJLNrbxbZTSMCHkJgCTAAQBvEkpXUoIeQjAHErpOOstJIHVBp92ScJKVI/vYtxoqbe8YlE1Jtvq3bpB3Dwvhk+O4nLx22jKDQUM3UJeoAC2763EqjL30QdG4l0TiYIQZ+frgW+WGM6vrGGdoH5YrvY8XaTzwatjlMavr9/3tFLzrIei2P24Yjs27zloGLaq3b6+0dXbBTZrXOX9FnabDI5isCXTuvCMmaNP6w7AphduLU3z5cSQ4ueSp3HQfzma+dSTjSPTi1I6kVLalVLaiVL6qDLvASMxp5QOSqp1DgDhxDoiJJs4Cz3WKMqmTzqsmWZ5UUEOOjXV+si93OzBAHO5RKIUww5v4Xp9q+36BaXAFW/O8rRuVSQaZw1XhaOOO1LFdxZi27r904V45ZdVaFxonr1TDZuMf2TcCqxY+q1pawz3I8I3H0vO5VEk9GvxL0Z+ffUx02ZiZDf+bE1YnWfXiM3KANsrKrF0c7lt2dqCPpmd/jBrlGivVLtckpMMJdnU8h4JcT50weWy4uFhhhEeeneMl2c2GCCIUuZDL8jzr8HYT0EHgBUeEzWFI1HDgSrcuiA44lpfzduEc49oY1qWC5foffKju7eT4eTW7zqA+79Wvy7cvkB4ef229Y+Rvi+CmduP2gi6eLs4uXMoBU565hdUVIax9okRbLtRimVb9qKnx3TQXnBzVvXnysydWhsbRWsfKeghmhDKteXXOEex6gKEID8naBjhESfoHnYbIASRKLuJcnwcKslPQf/lD+/hqgQkTgBrwtbx7o63TdRc7kZMXMwGyNAOc+fNCnthyp+x3/FuEON13puxDks3s8Y6tw2RZp3G+JGYxUyb7Uf8ejQKuXcSS6+Huy44Y39djdP/8xvmrkssz4sXnNzt+p7VZl8iUtCdUEtzuHC4/vFL7EQQ4+4HD/dBIAB8MW8jduyr9hRFYoaXmPFkYTT8mtdHJq6DkIUyvzo1fjSlB5TeoG4f2sXC4BVeYszdfpFwV4p+03ojQi9SZnURZxuJvmbELoueovoyIvwcbdpj3E8xUd80pRS/l+7AweqIpy88/UvS7HFLdTrdzBT0Wm6h8xuaPzDcIk/2aCZiL0o/RbhtQ2+jJ/kNBY17kGsi3sM0xfUIiKN8NkYvZz+tMCfa4vY2Uv25xi6XWESG/uUSZdE++qgVO5fL2KmrY6MxEYN1jLbjhPKDNSg/WIPu938Xc9t5HeLx87kbccnrM9H9ge/w2MTlSn2cV0jfG9ipy2XBhj34fZX/KYNj9UjalpMJqd3V5s88v8g5Qd7pwPyG2bFPmwvDS6OoGNLnpDHKCQ0KctC1RV1ftpUoU//Ygevfn6eZV+2jy8XJqE36B3fznoP4ZaV3N5L+gXciKs/98IdtGXEIO25IxDWK6l5Oeks1Sil6j5mMAY/+oK2jxuViXN8v57P4bKP7sCoc0aRBNnRy0Pglff45GX3+ORkHa7yN5iUi7t9oOEMz+PXRv/ydCvpZL03DJa8lL91VZjaKptDlwgeLaFTgfPxSbjXwaxyLIrAQDH3vNNYg5U6UxZvKL79324YFngfb8BujB3nOul2+WcjOEpRpy4x8cRp27PMedZVInhYrLntDFY2YoOstdP5f+aE/fm6xH9CNfaq10I33rxd6cZ1/fL4IXy/YjBNMwn5nr92FCUqbhd2t5zn00eM9zRuK9efKbHMZ1/U/LaTQQu/btgEePqsnnjy3t+N1+E0Wc7ko4lpj8Un/wV+O0m0jnhmjT7Lcr+hy8TJqkRFRSmuNoBsxrXSnkqo3cSI2Lpf9VeG4Rt1ExBwwaIhMQpxbtYmFXn6wBut3HjCN1hn23NTYbzFnuZ3LBVCPI+ZyEZZNVkaaMhu4+3xhYA87l4pXwRTtHbvoH83+lP8yysVPUmihE0Jw+dHtUVTgfNAIfnEHdGgEAOjfviEAWMY592vXUDNt9Ondoijfer/CXZqf4885ohRZOx6keIoJIdiy1zpR6B2fLsRHszZYlnFdB4s6JYIoL7G2G4OImoFP/xT7EgxHaCyaB2CpezlHPvpDLOJE43IxqXA4SvH+jHWoVAY7F+9nLvZ6y9/wOGxsCXH/bnzgXn3vfH9xPnQTJa2NXf9rIbXXYgSYmH57ywl45TI2EtGdp3TD+JuPR5fm9SzX+/Svx+CCEhYLLd4HYy8/wnafHZsWaqwOPhQcAHx2/TGG61x1bLHtdqOUYvpq856PfnCU8uITufzo9kndJ6AVJgLgw5nWKWB5JkmRnASjiUQR+nNbBdbu9D9/N/8ytNOW7RVVuPGDeabLecSJxkI3+Tj6eWUZ7vt6CSYujh93NRpz5TA3YyIfgNpOTvblLxo7Hd8t2aJ5VvZWhrFo4x5Hzht+HHofutnLSfzi+reDto9EyUxBNxoippbRvWV9FCgjB4WCAUcdJAZ0aIQLj2wXm+b3XI4D98mU20+MxSkDWgv9yOJGqJsX31wyZuThttutCkcxuFuCmTFtuHFw57h5XVtYv/xqC/mhxL6ERKE7+bmpCVt03AUktr/w5G9rTAZ7qFTaJqrC1hYzHwBcrKLRSw5AXGQM98d/OHN97AWzXxHBr+ZtghkErDftcU/8aLhctND3HKg2deMA7OU5Y/UuXP/+vLiXyF2fL4r9vverJaYjTvH96V0u9fONv+DFF/a/f/jTsIyfSEGvZXgZi+CMPq3iGlD1QuPVCqqqieCVy+y/EBLB6FPZa+9PN4jWpdfzk+9gQO6hz/5iOqKP3z7WOz9bGDfPLlz291XsC8yuKlz4vcSAR6Is5PSer9Rh8Lj4mr0UAHZdHhq/DJv2HDRcLtbliEd+wKB//WS6LfGW0j8vO/ZVxbJorthaYTvilCjo+6rCpuknpMvFCVzQi09Ibz2SgFEjkshfB3bEnad208y74+SuAIA+bdSvgLwc7aUd3I3lj5l3/8mu6lMZjiI/J4jxNx9vXucEPWBGx5qKxiQx6shrY6STxufS7fvws0loo9+HWVkTL97//J+zAZTtzjmPMnJSZX2QVZTSuPz4zvKOE7RvZN4PQn/ZrHLwa/od6OqnDxu224boQ+/54CSMW7jZdp+pIDMFPRoB6rcBLv083TVJHpocperP0cO7428GLgoAOKe/motE3yj69Pm9MfXOwWhk0TBrRJXyEFu5jBJOr2twz6ci7aiYh95oHFMnbNxtbDnqMXvpfTZ3I7aUO9uGE3gP4WSMK3uwOoID1WFH29ZbwNEoxcCntdaz4ZizBhEndS1SMrj5ktM0gjtoh+NfEPsF9xHfhlXEmqZ+lGLxxnIs1mXvTBaZKeg0AhQ2AXKsoz4yEf4gOL1NrzmuA9o3ZhaMtlFUK+h5oSDaNXbf47PSQUhgooLu9wDCThHDHf3orOKVWWv8y1cScHn/iNiNUP/ohOXo8cAkfLPA2Bq1IkKdDeSif5FHoxSFuRaC7uJARWvZSTcNHss/Rxhvl7dRODU4IlGKM178zdXAK4mQoYIerfW9Rb3i1nvxwBk91FQDYtiiT3HoTm5cJ422VhinjE2BhZ7kVAxOyUuwYVWE93mwi6k3ws7q5OdLn4bYiDiXi0MB1Bez+2pw8yWiDVO1L88HTq8RXkQ3KD2VnXVCYx3PUklmqmKKxhNNB2KjqFvftNi5wa84dNEvb4Y+dO/KY9yFHPIHbYiQJ96p5TWid0tX+xKpMvA3p4P8nABe/3W1L9sKBAjKD9agwmDUIDv8fMHFJT5zKLxGycqsvhIved15N3p97h4v622vYKGbdl8z6SIzBZ1Gan2CLq/ERkk3GLjYDiuXixe6t6yPd0cdZVsuR3C5NCzIwbGdjbt0m8GPjkC1MJ1a6C3qe3e71RYLPTcUwCMTlvuyLQJ49snX+NTjFoiPrHF6PXs+OEkzHYnSWDtOolg1ijpdb8e+arw3Y13Kh5ZzSmYKejSatRY6x4vHgWgsdO+Xtp4Ss969ZT1HowGJVnJuKGAak2sHIcCCB0/BojGnOH5gEgmw8WtwbidY5eXx07tEiHGkixOcuhGcUK1Pxetw23H5ZKI0Fi7pFX4viZt+4tsVjtfXV/3+r5f4eq78JDMFPZt96ILLpXNTluWwUWGeo3UTcbk0r6/u49UrWNz5YQ4795zRuxUuLGkLgFnrVgNFcFoqaQyuG9hR8IMS1M0LoX5+TpxFV2zSoJtIyKRRR5qWNukVksFqkw4/Xti2twobd3uL2PEzvbN+WzM9NvxGKU04V8/4RawRV/S3uxFko68L6XLxExrJWkHv3KwuDmtRD2NGHo67hh2G90cdhb5tGzhaVww2cduDceY9Q2O/j+3UBF/eeCz+cnxHTZmHzjwc1xzXITbNhTs3FIjFveeGAoa9UvU8eMbhWPvECNwzvHvcmJlAvFX06V+N0xckgpGFfvVxxXHz7j+9R8L72mmRxEscXi5RFmzYg5s+nO9pXT+/WPSC/uuf3nKAh6M04RdNMObG87a+0ddibbXQMzN9bhY3iubnBPHdrQNj08ebpBg1QrTQ9R2L3NJflywMAK44phgAMKBDQ7RrVIgbP5iLisowcoOB2L5zgwE0qx//RXFhSVt8Msc6sZVobMfHIxub4onkfXf6TB7Woh7euvpIXP3WbM/7ctq5J534aaE7GSzECX9u2+c45tuMBnVY3wvPg2unqZ+EFzLTzM3iRtFEEMXNr/S5Rgzr2RI9WtWPZZEsqpMTa8zMDQVQkBvC8xf11axzTKfGeO7CPiZbjH849A9MqjL4ig/v0R0bxfbNe9qa8eS5vZJZrZTgrw/dn5fD27+vxd7KmoS2wSNs/BjZilNbLfQMFfTsbxT1QofGhbHfVpZrxyaFqK+4S/59YV/cfdphnvb32Nm98P1tA9GwMDf2WcsjXtrqumtTmOdV52UHCFkX9c+L2brJ1HleByf54Hu3aeBq21cdW4zblZQNRpgN/pAMCpV8NIlawiJ+WvuJNhq/8dsaFN89wTJxlxmVNRHc+1W8SywciSbVaPJKhrpcsrdRNBF6tSnCq5cfEUsE9chZPQ1zsE+548TY77P6tfa8v/ycYCwlMO/UxHuN6l02lAJBk1Szh7cqwtQ7B6NtozqxeXHjX5pVwmdFF/fK3T5OduF2EJBrjuuACKV49nvjlKpPnNvbNMOg35zVrzWmle7A8i177Qs7xM7l8uAZPVLmhpqqDEpy4tM/u173hKd+MuxFHI5S5IYCbJBy4VDfuLIEo96Z47WqCZOZgp7FjaKJcurhLWK/LzPJKe7XeKMi3OVi1muUUuuBq/VpCfRhbukYNeme4d0x+svF6GXTuapJ3dxYH4CGBTmoDkdjqWHNyAkREIsiwRQebyhAfBuykGNnoeckmv8nRYgjNYlEohQ5wQBCAaL5sjmpe3M0rZdnul6yyYyzqieLG0Wd8utdgy0zIKYaLrhmn6EU7kQ5zkUprPrz/w0SZvsrRKK11a9dQ3x368BYXvsHzzCOdplyx6DYSzIUDCDPQchoTjBgKaJmI+Akg0AaBD23Fror3FAToQgGCEIGFyqdw+9k5lml0UO+UbRtowJHg2a4IZFelyGdy0UPpdSVaMQ1RAmTxU3EtgLndUyUYT3Vr59v/nZc7HdRnZyYhZ4bDDhKVsatOzNSaaEHCfH9C8iuzTDRkZ7SzUez1iMUIIbXMJ1D8DoSdELIMELISkJIKSHkboPltxNClhFCFhFCphBC3CXzcEu4Cgg562wjcc73tw/EzHusB6I2IxBrFDW+mym0ycPsuOa4DujSrG5s2ixCwe9nxyrNgmiNhUyOMxQkjkJGc+0s9BSqQiBATI8nWQRT+QniEw104woHTc6b31+NbrA9q4SQIICXAJwGoAeAiwkh+m/P+QBKKKW9AXwO4Cm/K6qhai+QlxlDlGUS9fJz0Nyjlc4tlZBgnWqsFxsfup52jQvw/e0nomPTQvvCKUKsv94HzMMsQwHi0EI3/lznuHn51c0LoUld7wYOIan9IgCc7S+dlq4R+vv3YHUEuw/Eh1TaXbqExw+wwMmWBwAopZSuppRWA/gYwJliAUrpT5RS3t94BoA2SBaUAlUVUtBrGfqwRQD49hZ1RCkKrculczNnQv3eqKPw8Fk90dBkYA6/H3qrEDkxSkf/cPO465xgwJGFHgwQ06gfvtwpv48egjn3DbUvaEKAxPvQT+nR3Ha9GwZ18rxPJ5pW2xpO9V9NO/cbj3JkFXTQvWX9pKaGdnLGWgMQu/htVOaZMQrAt0YLCCHXEULmEELmlJUZD8llS81BFuUiBb1WQWI9RdWbuUvzerighL3bKUUsJUCdnCA6N3N2/Vo3qIPLTaJ1AHeft7Pu9eZO4lhZ6DxML8ehD52QeP/raYKP3o3FnKh7JkDiXyBOtjm8p/fUxU62n5diQbfLXeRHw3FOkKRd0B1DCLkMQAmAp42WU0rHUkpLKKUlTZt6HEm+eh/7n1vXupwkpfCY7ZDuIVTTAauC7meDmBsta1YvH+0bFziyPo0IWgh6O6Vz1DXHFzvuZCRur1m9PNw7onts2omLma+f6Nk0stCd7F889+cf4e6j3ImgJzpwilt6tKpvudzpF4PVucsJBpI6cLSTGm4C0FaYbqPM00AIGQrgXgAjKaXJC8KsqmD/pYVeq6gRLFSR20/pipN7NMcZfVrFBP2Ahx57fvHLnYNxlUECLifkWDSKNizMxdonRuDsfm00wmy5PeFcvX5lCdo0VGPxnQget/ATdTsRA0F3Il51ctVIM7dtL06sXb9DKa342+BOuHGQ8Vi9HKOQXNGtyLH6auRfb17GfHWCE0GfDaALIaQDISQXwEUAxokFCCH9ALwKJubb/a+mQETxWwXdDXYsSS4ndGmCenkhDNR1WW9ePx+vXVGCunmh2GC/6c6DYeXO4A+a0WDaYkNljoUZ5sb327O1sVXoxOXCBc+pNhTVyUHX5vFftszloq2zk4yZYi9kJy+VkvYN0UfJHBoIELxxZYll+VS2iZ7ZtzVaN6hjWcaobaRb83jD0upc8Pj7ZD0CtncepTQM4CYAkwAsB/AppXQpIeQhQshIpdjTAOoC+IwQsoAQMs5kc4kTVay7QGZ2cs1WerYuwuJ/nmo5WlEdn4bFEzF6dsxyp3PsLL9FY07Bb/8YbFnGrzC/U3owv7newg0ECB4+q6flutyKd+qT/faWE9C2Yfy5CRAC/eE4eSmJA5k46X1c3KQQBco9ECQEJcWNLMuLm+RtMckiQNgXx18HdjQto28byQkSw2gkqzOhCnpyFN2RKlJKJwKYqJv3gPDbexO7W6LKWIlS0DMOQgjaNqqDK5U0vD5tVDM5576hqJMTRPnBGvznx1J8NGt93CpWIYGUwtGIS34J+k2DO+Oyo9sbfhFcfnR7PDFxuWkagSuPbY+XflplOch037YNsGDDHgBMuI2ExMhCD0ejyM8JWI5+JJ5HJ2cjNxSIZT4MBOzD+0S3k5tBLs4/og127q/GjyucOwv4C+kfww7DgeoI3puxLq6M/jx/dSPrXHZ85yb4rVTN9271clNdLo6r5oraFRfkhJiFfmj3FM1Ufr1rCP5ygrkVZMWc+4bGdXzSPzpN6uahMC+EVg3q4J7hh+Gkw+LT3nJ3RiK+ZyuXC6B+BYzo3dLStRAIEEMx51g99/93Sjesfmy4ZTf6z69XBwYJEOPtBQIEPyzfppm3rzIcG37w17sG4/vb1Bz9Z/RphXeuGaDdhoOTmRsM4ATlC65NgwJbq15MH+Cm3aVefg5G9mnluDyg1j8QICjINdYW8Tyf1rOFaU9tZy6X9PnQaxdUCvqhSpO6eaaNb40Kc/F/p2jT0dbLz8GzF/SNK8vFVnQB3TSYNYg5fczsOv7wBsv7R/TASd29RdUA5pbcJUe1AyHGn/ycc/u30UYdEePtGYkxIQQ9WjL/fiRKNft5/sK+OLFrU1159v+4zo1xXOfGhvXJCwXwt8GdMX30ELRrXGBroZcfVDvttG9k7UbTc1qvFrFhEZ3gpP1VzNEvuu30vYstXS7SQtchXS4SAS4klx3VDjcN6WJQIH4WF7Bk+PQ53AdtFKLZy0UOnptPMo68uGe4dSTNGX1a4ZkLtAOKEJi7XERK2jfEPcO74/mL++H5i/qiuElh7KumIDeoEfd7h3fHm1eVxLZhZannBAMIBAhaFtWJ1ccKMYvhncO6WZYVIYS5Rx47x/mgI5p6m1Rre0Vl7LdVr2erc7CvmunX53OtR+/ySuYK+iGenEvCEOPcDZcbPFvcuhIH0u6k9FwVE38lQixGXFeB03u3xMfXHe14OzcO6oy1T4yIm29nUT6rE3MAyM8JOLLQ7zilG5rWy0P9/Byc2Zf1IeTHoy977cCOGHJY89hxEkJMc6HrXUN2se7iyzAvFLRt7ObwtdxEPTpJtXB8Z/WrRN/fQrN/ZVOfXHc0LjmqnSbPf43SFuDnYCIiGSjoMspFomLnujVazD+dxTjqs/q2xpc3Hoszenvv/SjCXTjcH9teEaPixoUodBASaIedz9ooSqUgN4TTerWIm69/6RjFSPMiZrvl84PEfLxNvaDnhYKmovvCxf0w8e/aGG+nEtiobq5SJze9bcXfxuuNHn4YbhvK3HpWYaXcyKhfJwePnd1LM2A7Pzf5Sfo6lIIuyWqMHs7KMLuHRJcLIQT92zX0bfCPawd2xNonRsQe3AsUf65dY1i9vJBpo5yIl2oGAwSXDGiHFQ8P07gM9F4hIz02s9A5fH7d/JyY8OoF3CglwpJ/norHDVwjI/u0io2GxXEyMPOT5/bCtQ4b3Y/tZOzrNzu1OcFA7GUh5uHRX9KWDVg7Dz9+cXFNzJhIjvRmoKBzH3rmVV3iH9wPzR8rM5000h8uPq0aeM//7hZeDztNmnv/yVjwwClx888/og1eu0KNlvGaopUQgvycoKZzV5yFbmALc4vUzKLms+vnh2IWvv7FZBSNU5AbihP6L2881nAfZtf4vVFqxM2FR7aL+zrJN0mW1qxeHloWsXvA6GXRwcD9FhWyanL0/QWev7AfnruwDzo1ZZ24xJd4WIncybcINU2EzDNzqbTQJeoAEy//XArAPI+5kfD1aVOEh8/qiZG9W2HS0snJq6QAf4nYdfk2C0F8+nytT9xIWId2b4YflrvvqK3fVjeDJFUBGwudhxjyUEcAKMgJYg/USBWzY4vozok4Hu34m4+PfeWYnbujOhhb2gAw+96hKN2+Dxe/NiNuWZSqXx5RIcydH2KnpnWxZsd+zTr8RSieBy7cnKKCHJzdT+0IJVabty/kO/gK80LmmbkyykUCJjBiQ5YbC50QgsuPbo/6dVJ3D/Fq+BV/bCSsr195JFoV5eOKY9yNL0MIwfe3DcR1Azti3v0no1m9+C8XXm0zV88eJS94fUHQT9fFgnNrWI+VK6Vn6yJ0bsYtXeMyVu2ZTevladpKAGCoEkZKoeZnMbouRscaUZTfTW5/cds1yvoFSfKhZ54qyigXiYBTn7eZsANaq9KOSbcO9BTuqHbTd72qIWaH/fvo+BTBbRvVsRy0OEAIujSvZxkKya1js/PNLfTerYsweelWAMCRxY1wz/DuOOmZn7GqbL9prpT2DqNX9KJ734ju6NGyPkLBAG4d2sXUDaVvwOTHQinF61ceiY9nr9fUIRY5ZXCtuIVulctej7idLs3qYv76PWhSLzkjrmWgoEuXiyQeN2GLIo+f0wvHdDT/ZNdj5I5wgupD90fR3TTe/nTHoLjz07FpIVaXMXeCE2OTW943DzGOi79laFcc2aERjhEaGrmL5bUrSvDJ7A0obmwcEnpspyb4+LqjcdHYeLeIyHUDO+KRCctj04O6NY3l1b91aFez1eKa24qUoeRaN6iDDk0KMfo07YuMn1ojF4+RD90O0R34z5E9cVa/1nFuGr/IQJeL7CkqUVEfPuPlduF9Fw9o51vsuRWqDz3pu4ojFAzENRR+fv2xsQEdnMRg5+cEsfaJEbjCJA9Po8JcnN67FQghapSLss+OTeti9PDulvtx0tlKTBmx4uFhjgdJ0d8Dx3RsjP9e1h93nGLdWcno5XvBkW3Ru00RLj+62NG+AdU/P+SwZqiTG8SxncwT2CVK5pm5MR+6FHSJinmjaPKolxdCQZ6z+zBgYfW54YsbjsHkZdvsC9rQqDAXPVrVx4qtFUkbkNrNQCZW+WiMcBPHrT/llALDLEZbUts72P+6eSH8S2mUblYvH+NuOt5NVWM8dObhntZzQwYLeuZVXeI/3C9rlBYWcOeacMuCB+PDC83g1qk+osMtR7RvhCPaW6eddQqvSrLGkXBz6rkLIxkDKItJvgDjvOZG8CvVtXldDOsZ3yHLKdzST9aLUyTzVFGGLUoERvZphYYFuTihi/FnLH+Ezuvvfz5tNyPqEJ8bRf1gULem+Gr+Jtuh11IBIQQPnXm4q/YMp3BBP7xVfZzbvw3O6G2TiVEXYurEKGjXqMA0pXKz+vnYUl6ZkkGvM08VZaOoRIAQgoFdzcenDQQIFo05JWlhYk4J2Pj608GZfVvj5B7NUZBbO54lM/+8yIjeLdHVoe9cT8uifFxzfAfbcvoQUyev7al3DTZd9voVJfittAxNkxTZIlI7rqQbYmGLmdeeK0kPTgasSDaxsMXaZKIDSRHzbs3rYf76Pa7CQZ3y0iX9Xa9zRPuGuGtYN1x0ZDtX6/Vr2xDtGxfihhM7ud6nSNN6eZqORskkAwVdWuiSzINb6Mka2KA2MWbk4Ti9dyvHUSjJhhBiOwC0yLCeLfD8lD8xvFfLWuGOckPmqWLDYqDbcCCYfqtLInEKTzRVUtzQpmTmk58TxPEmbRqZQPeW9Q1TFmcCJNEwKq+UlJTQOXPmpGXfEkk62LTnIFoV5Sc18kaS/RBC5lJKDcc1zDwLXSLJUMy6vkskfiFbFiUSiSRLkIIukUgkWYIUdIlEIskSpKBLJBJJliAFXSKRSLIEKegSiUSSJUhBl0gkkizBkaATQoYRQlYSQkoJIXcbLM8jhHyiLJ9JCCn2vaYSiUQiscRW0AkhQQAvATgNQA8AFxNCeuiKjQKwm1LaGcBzAJ70u6ISiUQiscaJhT4AQCmldDWltBrAxwDO1JU5E8A7yu/PAZxEZP9miUQiSSlOBL01gA3C9EZlnmEZSmkYQDmAuEz1hJDrCCFzCCFzysrKvNVYIpFIJIaktFGUUjqWUlpCKS1p2tR8UAKJRCKRuMeJoG8C0FaYbqPMMyxDCAkBKAKw048KSiQSicQZTgR9NoAuhJAOhJBcABcBGKcrMw7Alcrv8wD8SNOVl1cikUgOUWzT51JKw4SQmwBMAhAE8CaldCkh5CEAcyil4wC8AeA9QkgpgF1goi+RSCSSFOIoHzqldCKAibp5Dwi/KwGc72/VJBKJROIG2VNUIpFIsgQp6BKJRJIlSEGXSCSSLEEKukQikWQJUtAlEokkS5CCLpFIJFmCFHSJRCLJEqSgSyQSSZYgBV0ikUiyBCnoEolEkiVIQZdIJJIsQQq6RCKRZAkkXVluCSFlANZ5XL0JgB0+VicTkMd8aCCP+dAgkWNuTyk1HCEobYKeCISQOZTSknTXI5XIYz40kMd8aJCsY5YuF4lEIskSpKBLJBJJlpCpgj423RVIA/KYDw3kMR8aJOWYM9KHLpFIJJJ4MtVCl0gkEokOKegSiUSSJWScoBNChhFCVhJCSgkhd6e7Pn5BCGlLCPmJELKMELKUEHKLMr8RIeR7Qsifyv+GynxCCHlBOQ+LCCH903sE3iCEBAkh8wkh45XpDoSQmcpxfUIIyVXm5ynTpcry4rRW3COEkAaEkM8JISsIIcsJIcccAtf4NuWeXkII+YgQkp+N15kQ8iYhZDshZIkwz/W1JYRcqZT/kxBypZs6ZJSgE0KCAF4CcBqAHgAuJoT0SG+tfCMM4A5KaQ8ARwP4m3JsdwOYQintAmCKMg2wc9BF+bsOwCupr7Iv3AJguTD9JIDnKKWdAewGMEqZPwrAbmX+c0q5TOR5AN9RSg8D0Afs2LP2GhNCWgP4O4ASSmlPAEEAFyE7r/PbAIbp5rm6toSQRgAeBHAUgAEAHuQvAUdQSjPmD8AxACYJ06MBjE53vZJ0rN8AOBnASgAtlXktAaxUfr8K4GKhfKxcpvwBaKPc5EMAjAdAwHrPhfTXG8AkAMcov0NKOZLuY3B5vEUA1ujrneXXuDWADQAaKddtPIBTs/U6AygGsMTrtQVwMYBXhfmacnZ/GWWhQ705OBuVeVmF8pnZD8BMAM0ppVuURVsBNFd+Z8O5+DeAuwBElenGAPZQSsPKtHhMseNVlpcr5TOJDgDKALyluJleJ4QUIouvMaV0E4B/AVgPYAvYdZuL7L7OIm6vbULXPNMEPeshhNQF8AWAWymle8VllL2ysyLOlBByOoDtlNK56a5LCgkB6A/gFUppPwD7oX6CA8iuawwAirvgTLCXWSsAhYh3SxwSpOLaZpqgbwLQVphuo8zLCgghOWBi/gGl9Etl9jZCSEtleUsA25X5mX4ujgMwkhCyFsDHYG6X5wE0IISElDLiMcWOV1leBGBnKivsAxsBbKSUzlSmPwcT+Gy9xgAwFMAaSmkZpbQGwJdg1z6br7OI22ub0DXPNEGfDaCL0kKeC9a4Mi7NdfIFQggB8AaA5ZTSZ4VF4wDwlu4rwXzrfP4VSmv50QDKhU+7Wg+ldDSltA2ltBjsOv5IKb0UwE8AzlOK6Y+Xn4fzlPIZZclSSrcC2EAI6abMOgnAMmTpNVZYD+BoQkiBco/zY87a66zD7bWdBOAUQkhD5evmFGWeM9LdiOCh0WE4gD8ArAJwb7rr4+NxHQ/2ObYIwALlbziY/3AKgD8B/ACgkVKegEX8rAKwGCyKIO3H4fHYBwEYr/zuCGAWgFIAnwHIU+bnK9OlyvKO6a63x2PtC2COcp2/BtAw268xgH8CWAFgCYD3AORl43UG8BFYO0EN2NfYKC/XFsA1yvGXArjaTR1k13+JRCLJEjLN5SKRSCQSE6SgSyQSSZYgBV0ikUiyBCnoEolEkiVIQZdIJJIsQQq6RCKRZAlS0CUSiSRL+H9JW+NUqDf9TQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# training\n",
    "model = BobNet()\n",
    "\n",
    "# protip: if you like accuracy like 96 not like 93, next time through the notebook, consider\n",
    "# CHAD MODE WEIGHT INIT WITH NUMPY\n",
    "# instead of virgin torch init mode\n",
    "# TODO: why is torch linear init bad?\n",
    "\"\"\"\n",
    "with torch.no_grad():\n",
    "  model.l1.weight.copy_(torch.tensor(layer_init(784, 128).T))\n",
    "  model.l2.weight.copy_(torch.tensor(layer_init(128, 10).T))\n",
    "\"\"\"\n",
    "\n",
    "loss_function = nn.NLLLoss(reduction='none')\n",
    "optim = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0)\n",
    "BS = 128\n",
    "losses, accuracies = [], []\n",
    "for i in (t := trange(1000)):\n",
    "  samp = np.random.randint(0, X_train.shape[0], size=(BS))\n",
    "  X = torch.tensor(X_train[samp].reshape((-1, 28*28))).float()\n",
    "  Y = torch.tensor(Y_train[samp]).long()\n",
    "  model.zero_grad()\n",
    "  out = model(X)\n",
    "  cat = torch.argmax(out, dim=1)\n",
    "  accuracy = (cat == Y).float().mean()\n",
    "  loss = loss_function(out, Y)\n",
    "  loss = loss.mean()\n",
    "  loss.backward()\n",
    "  optim.step()\n",
    "  loss, accuracy = loss.item(), accuracy.item()\n",
    "  losses.append(loss)\n",
    "  accuracies.append(accuracy)\n",
    "  t.set_description(\"loss %.2f accuracy %.2f\" % (loss, accuracy))\n",
    "plt.ylim(-0.1, 1.1)\n",
    "plot(losses)\n",
    "plot(accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9288"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluation\n",
    "Y_test_preds = torch.argmax(model(torch.tensor(X_test.reshape((-1, 28*28))).float()), dim=1).numpy()\n",
    "(Y_test == Y_test_preds).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1315a82e0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAABZCAYAAADFEWQMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdo0lEQVR4nO2dbagk2Xnff8+pqu6+b3Nn7szs7OzMarWyZQd/CLYibIkYY2IcJBHsfDBBSsBycFhILIjJh7BLIHE+BJJ8CIkh2BGJEgUS2Y7zYiEcFEU2GAyRtbIleSVlpY29kna9u7M772/3dledJx/qVN+6fbu7nnN3em7fnvOHpqur/1X1P0899dSpc55zSlSVhISEhITVgjtuAQkJCQkJDx4puCckJCSsIFJwT0hISFhBpOCekJCQsIJIwT0hISFhBZGCe0JCQsIKYiHBXUQ+ICIvishLIvLsIo6RkJCQkDAb8qDz3EUkA74J/CTwCvBF4COq+vUHeqCEhISEhJlYRM39h4GXVPVPVHUI/Brw0ws4TkJCQkLCDOQL2Ocl4Lut368APzJJEpFngGcApN/7C8XF8wuQ8oDgAZV6Oet40lGgkv3fmYLMZNdcpeaokS9a35YrATeHr4APHFrL8/Y/WY4uPQ287Ou3oJJ6v87AP9FaqH0nRgvY+I0Wp7ZqWqz2WXylLpNMWd/2fWmth1pj28aT/DYch/cPtT2R+lvYvxYeUQxffvUtVZ0aPBcR3E1Q1Y8DHwfoP31ZH/+ljx2XFBua4C6Wi67t4F03gwnvtfJF9y+wLn7DsfBnbWvhNroWxT+JWmL5y6RlHn/e+nmY3GYef5bOmOtrxfGdn3vu27P+W8Q971Xgydbvy2HdyUUlMJLZtYw2NHCHDiphbWsP6aqJDx2UdW1zsDmczy9DrQWQzFMMyvl6Sqk1qZD1K7J+1V2GUI58UCLWGqeH/mZHWSf4G6fv2/ghkGxa+ZUguWewObRpKYXexrDblo2WUtjY3iXreZMWKmHrzD2bLcOT3NaZe91lDb7mCs/G9q5N+0goBiXrpwx8X/MHm0OKtdHB9ZXU31P4vY0hkvv962boxtfDIT2lsHZqt67FN7zRjP0DIrC1c7f+v+y+Jl3uWdva6y7rCmIRNfcvAu8Wkaepg/qHgb++gOM8NLh+RZZ7vAp+mDGvD1oyT76+HySGe8VcfrZe4rIK7x3OeUaj2fsXgWxzhPeClo68V283U0vgV8MMRMmLktHe/FMuAqoguScvKqrSdv/P1ow3jXCMfGPEsMOWY37myfsl9+/3TPzGpsNdm3vnmyPUC+Uws2lZL9nbLfBl952m8Z27dwZTg9Uh7RsjBLh7Z9BZVnFKfmqE9477d3vd2vNaezXKqEoDv/DkGyXDvXzctCICFB6XKb4StBVcG36eV4goZe5xouPata8yquG+PzV6RJTe5vBAeX2VHdp/s83eXkGxtYd6Fzhupq20cuzdLzrLuop44MFdVUsR+RjwWSADPqGqX3vQx3mY8KXDG4OcemEU4UzV0FGFB6iK+cFFFcq9fc6oI3gd4KuwZwgAzUWilbB7p5vfoBo6qqGNr9qt/QD/bdjUgrZNH7SWGN8BqGK0xNqxitM+ja+hnbyaUmNu+O1t5t3yp/G74EeO4chuT1XQCPuvEhZSalX9bVX9PlX9HlX9J4s4xkNHV1viJFcFSkfvz3rd2zb/72UUr3XwW//J3Zzsqv3CyK/0kNvGYKBC8XoPrMHGC/3v9G1NVwBDx9qf9M12lXs5g5cj+LcK8jeMNycVitd6yC2jLb0w+HYfuWe0ZSW1dmtQGjoG3+7buCrIrYLeK/ayuutF7WdGfvF6z+5nvi5rdq0Ybz/+TEPpWP9WH+5n3VyAvazmj5zNF4au9uNHEMfWoXqi4EP7n2DIlmm1A3oYbXbwq+DMYZNyQ+d3EnnqZ2OnaKb4QYf2JttBhWqgaM/eAVUN1J7loVBueFsWRsDwlF2LOmV0yts60LxArlQb9v6Cal3RvqENPZzf4SmP5gZ+8J3htrdl4wTfGW4b9h342vOMThluHEG7H3g0N/YfVVKf12xifZPh1T4fY77iC9337QaHsmXq/qPhqZAVU01wJ/dPvf1wW0MmGONrYSYypdw02nLFkIK7BU5tFybUzphPph7Od779XDFfdxLNQzvYZkqnqoYvihYRTi6KFh3a23DgC+zBPQ/7tyJT1NpiEW58YhWTKZpHnN9C0SyzlbVJPW2ClUELgFqfqXMPVWY7T41vVkbtDd85tO3/01IUm/WFUvVDJeKAb8/ig++F41huli7cOPKYa9JGWzWk4L5IOEUHEQHVYas9NohxcIBeXM1aB5W94U4UXa/sNwMBH8PPteZbUXg0wru1b6xZN/y1av8m3gUJ2q22d2H/VhQezSJObCRf+8Ynpoa/Vtmf+ETxmxF8R823nqvYa2qF8Gj2NDwseCG7kdvb60shu2XvUGPokHt2vruTw579lLtbefeTRAMV8mu5KSMEAA/F1QjbDB3FtYhovedq/UZktzJk12jLpqxDoy291Nqt/RFV2L8Ve478up0v9zKym3a/yW5ldj8LtjHzvVC8FWHLUui9GWHL2GtqhZBq7gZIpoir2/nmpV1BaAJsNX/48zr3ydTlHnGgHqTn8f1qZoAUqVPB1NcpYm5jhK5Nzwc+wG+yBc7tIWW3ftU6zU4u7CGj+fzxdrnHX9rt3P9Y16CivORNF6kIyOaIcq003Txc7uGUR9fdXNuP+YXHnx8eSrubpcX1KqqL3qRFMkUKT/n4fD84oL3wlBe6a5vj85sppaGmL05xueK3SqrNbi1j/tmQ464hFVLq60G9oC0b1HyPPlnnKmkYRdrk66vnQOaQOMX1PP6pw3li6jm0f6j9ZvRkRZZpnQnjD6dLHkDhqXYezZp7Cu4G6JR825lc5UAubxdi0uRUQVsZF13bTvLVkK0xToX0QjWMeHSPtJFFy1H5MTaFOr0uRkvM+Y2xC7w9f+jkx57TKXxVQA8H3Qe1/85tgi2npWImHEQK7ga0RwpaaqWT7ZPTLoRD/NZw7k4++7VrC7/RLJmCF1tNXKjbNSP51kAmoaNxEfz2OZhnm0N8NZa16SQ12OaAPxj2f2TtBv6kr5n5E9pnXQ8HavXaeqKcUf7Jp4ADCM11k/Ya+6XO5kwrg8WWq4bU5m5FV9ZLG9L6WObakNZ3F39Sh3EuD/O0AEdF7BwfgqmZApoLFHt7fkRwPMC33sSMgX1/I8w3jujA7nRmTXqmFuP+x/xp2medb6EO7A2nuRFOS2ts89vXzCxuAzVwGqpGlHXFkGruBjSj8h40F+Idr11ztWzbcKJ1PcRyHLuWCP4y2XHhWmbwm6aZh61n3rETDiMFdwuaiZKE7lzcZuKwBl254qWrB2M46u+u9MaRq//PtN62Kwd/5PZT9kqpl60zW46kW3+DZqInay59uxwWLaXUXEsKXBk6UhelZST2NNTGd6xaYrTH2iX2nM7iNwP1son1Xg5mVzlC80nrd/v6mbxW2nAc3j/s3wya69Fa9kcQKbhbEDuIKWIUqGngRhvti96ybZsfM2gothyZMThO02XREqP97djUoiXGLjG+A3HaY+0Sq30Wvz1Qrw33gPY/D42PxfjaI4oU3A2ISoUM6WMNfEcq4TgVUus2VF/Obj89lApZ+Dobw5gK2SxbUyFdrp362+VA6Nx/Wxdqyw5p0kvn2eaoWqDed2dK3RG1NL5j1hK0WzJ4xlqMGTmx57TNh30fnZ8Kub9j9SCtYkxNhZwxEGxmKmSmiOj4mrGet0cRKbhb0HSeWTrFVPCGacHH9MqhPnSihe1ncps2yiZLYHLujhn8sWbDRTDmHqEciJqCxmQ5LHxfRvAjtNR8+74hTkvjO2YtkW3uUVrCOTV3BLd8YJwiqyDMCKhTfEb9wf+7+DO5DbygyPiaSe3vs5GCuwExnTiT3K4slWh+6+JXL1H8mAthfBFbg1Lgm/dv0H5UfqOlnQZq2beF35yvGC0N36oFbFoa/lG1x/Db+585d/rb2H+Un0ncTTBm/6uEFNwXjFinWiR/mbQsmr9MWhbNXyYtq8BfFaQ894SEhIQVRAruCQkJCSuIFNyt6HpDzDSuhd/mWvnTlh8Uf9a2D5ofY89l2vei+SdF+7z18z4x/C5dqTN1LlKbuwUHBjF19RJNDMwomD9gpJL9QUxK/cabeXnRpewP8Kjk8Nttpu0/V8YDXvIOPe1yxPDHg5iMDZylhIFDBu54sA72txkpkVqIHMRk1DIexGTU0mShWOeKj7HLeFASkYOYJvjjQUwT67sGMWUT57tzENMUnT6c2/EgJtIgphlIwd0AKTxuoOOX7c7NE888buDHU6f4YTY/z32tHOcMiws5yHPy1t1GOc4XdmtlZ567Wy/rPGVR3EbVmeM8znMP5ejSPy5Hv0Jcd17/tHJ0QZziNsu5tjmgZa2sxwwY87mzGC2NXYxaGt8xjxcI2i0zT4qA2xzhS2fLc488p+2yNmnA9Xw2HunpId+T3JOt1bmPSjuTpz6Y93Igf799rTQYZ+KG/PVDee6Fr/Pcw6sjrTn+jyJScDdAZ7ztfSrXS9Qb7N/OdLNd207yLQHjwJS/MeWInKo2atpcZWE2hUgtkXaJ8R04gj/EaInVPoXfNeVvuUA9UNtTEYgYg/GoIrW5JyQkJKwgUnBPSEhIWEGk4J6QkJCwgkjBPSEhIWEFkYL7AiGZcuHCTfM8JFm/4tz522Z+sTZi8/R9s55TZ+4x2ByauCKwc+42Wb/7xctQZ7Q8cfF6ncVggCs877z81vhVgV0oBiVPXbpqts1gc8iZs3dMXBE4d/42a1t7Nr5TLl28TrE2MvOfunT1wIvT58EVnksXr0eV9fHHb9jIwMb2Lucfu2Xm75y7bfYzEXjyiWtsbO/a+Jnyve+4Qm70s6zn+b53vFHPnmnk75y7beKuGlK2jAFZz5MXJb5ylHt5Ryqk0l8bIVK/R/LNq1tz+cWgJC8qytKRZcr1mxsz+SJ1QK8qhx/WWQb37/VnaxHIByPKvfo0D0c5e/eLuWVtp0Le3+uZs2CywnPt9obpVXgiUPRKXr26bUsndEqWe15584wphS/vV6jCjesbNi1rI27eXhvbaS4/nN/Xr26bMlsa3/nO6zsm2xSDOjX2z66c7k4pdUpvbcRolPHGlW2z9nt3+ty7PdtvGrjc0xuU3LixcWDiNsk8LlN8JQf8w+We/tqIa3fXUa3L4jKPCzfw0TA/kE3T8F+7eYqiV1L06hQYVaEcZYf23xz/T9/coTco8V6oymxumqkfOa5f2+ws6yoiBXcDqqGjGvZMXK2E3Ts2LsBoN2e0W5+GrnqgKgzv7QfnUUegVj3IsehqLhJfOu4bAkCDci8zp8Gpwt5du43Ux9k0Vkvbpp38yPMb4zvA2BdMWrzE2TFSuy/dVL6WbupUvbE+E8tvUmgrXOe10t7mUR3JmpplEhISElYQKbgvECKY26CPzI8Yei1Oo+ZQj+YvuqyLtk0MP4u3jZW/cNs4jefH2mZBthQJby+LseUjOj1BCu6LhFMeO3fL7IiuV3F2546Znw9GbJyydVwBbG3fp79h71A9s3MH17N3qF587IY5aEjmecfj18wXXt4va77RNv2NIafP3LVpETh79g6DCNs88dgN8oG9Q/Udj1+rXy1o4eeeJx67EVXWC+dv2sjA+tYe587aOxnP7Nwx+5kIXL5wnXVr53SmfM+lN8mMfuYKz/c+8SaS2Tunz+zYOtZXDaJLMJN9/+nL+vgvfey4ZSQkJCScKHzn5577kqq+d9p/qea+QDSZGOZHyEwpBvZJM1zuzamKUGeRWNPxIGRuWGviAr31iLI6pb8xtD/V5D6OX3hzeh2EzA6jbZqyWtPxROratfUpRZzSW7d2GdZltaZlQp3BE+NnxaC0p8QG22Q9u20Gm0O7nzmt+RG2jCnrKiFlyywQk9kqnfxKGFX2U+JLFzWBUsykThCZuRGbdRKZ6eFLx14ZwR857LexBZf1CNlBMfv3Ixc14ViTcWLFom0Tk8ETmzmlXqL0rxI6z7CIfEJErojIC611OyLyORH5Vvg+E9aLiPyyiLwkIl8VkfcsUnxCQkJCwnRYbt//AfjAxLpngc+r6ruBz4ffAB8E3h0+zwC/8mBkJiQkJCTEoDO4q+rvAdcmVv808Mmw/Engr7bW/0et8X+A0yJy8QFpPZkwjEo8Mj/2VWP+CPwYxPJjXrLQvP0nhh+j5yi2ieHHvlBikWVdtG2qh8CPQaxfrgiO2qF6QVVfC8uvAxfC8iXguy3eK2HdIYjIMyLyvIg8X922paydOKhAzIsjfCyfg68160IZGSBHLu7CGLq498xGvCQDT5xtqkhbjlycbYaOuEb9SNsssqzlEWwT42cx/MYPrEl7XpC9zG7L2GtqhfC2S611LmV0PqWqflxV36uq7822uucAOU40AyEsPfST3PzWfEdsuOIUvJDdnt/peWBgUemQ+938Bu5+hkQEVHcvLuDVZTWSFYobtotUBKgkjj8Ssru2sopAdsche7ZX24lAcTMzBY3mXBXXM5Mtaz8ItjFAnMJIyA38sW/uuvpcGfnZHYe7lx1cLxzKXGrW5TdrP2v79rQBcSKAKL1gmy4+AF4ortU3A9Mgu0rI7jyawf2o3chviMhFVX0tNLtcCetfBZ5s8S6HdScaMfNTHOCKUp6fPzDmwMRZuac6F8HvV2hHilqb77ft6XIA/kwcv3zMNggIAKeMLtr4qkDPM3o8gr9WUa3Z0vdUoTprK2szLGR0IUIL2MvqBTK1l9ULrFWUhrKOfXOrpNwy7DvwJ20za2hMs75sbNPx1Nfwh0/UfMskcuSe4eUIfuGpzsY8Yq0OjnpL+zTw0bD8UeC3Wut/NmTNvA+42Wq+ObnwAqWxFts8UjefrptC80hdhWN08ctWU4mlrbKZVU+N+2+XI4bf6LfCas9l1BLTxOIjtRxFu7XpLFb7LH7T/zFtfdv3m7K0f8/jT247TaeX/WsmpuyPIDpr7iLyKeDHgXMi8grwj4B/CvyGiPw88G3grwX6bwMfAl4C7gF/cwGaHz6Uun3VCaZ2h6aGJBZuq3NLqW+38/zVt/734Rjz+O0nCQ9YU92bMpv5Etc4F2PPMZ/5ZT2qFjXut823btPY0bzvCO0qB/3BtE0kdxq/8Smd8D3lcLA98CQ7IUAn/m9DOLz/A8du/U6Yis7grqofmfHXT0zhKvALb1fU0iHT+mOBKPQiPC73cY1j7ZF/lhGVbU6MLneEcsTAOIIROJpNYxAxajdaS4zvQJz2WC2x/Fk+4LT+TFsfMSo42scg2DJFdAsezZ6GBWPc2eQFd73ofAxud5C6G/bRfexlyF37nUFu57BrH6Xqbhb2JgIVsquF/TG5EvI3e/YmgqGr+VbsZsgtuy3dzQI6OqfbyN7qgXXEr4ayWpt+Kqn3b8VuRnbNVlYRkHu52c9EatvIHbufZW/1kHv7/Gmdr2OokL/Rg6Gb2VF7AKWjeCPClqWrz+0jiEdzXO6CMe5wcoo/XXY2z4z5ucefjqi59SvU/q4DdCtujo2oDlhRqp3uso6RKeW5kZ3f8zXfikGFDuy1yNjO5upshHaJLGum5g5eAAYVVd/mN6rAeomu2RME9G3aZu7chKKUj9V80xyGuWf0WIQtc4/fTh2qCYtA7BPkIgcxHYUfg9iyLpK/cNsQz49BdFkXzT+htoF4P14RpJq7BbsZ2T2Hz0Ptd16tYeTIb+TjDrdqw6Prs2vMcjsnu+/wfcXtCdWan13DViG7nuMHiq5V9XJf0Y0ZfC+4Wzn+VAkKxVsFo9PV/HZRL3Vb6NBRXM0ZnStNbdJyJye/4+paVdd4AC/jZoSu1M+aJORXC6qBoqe6a5FyO8ftOaqtjrIGuBsF4qE6VUHeob105NdzqsbuXS0zuxnZXUe17mHgO2uccqvAlQR+h3Yv+z5QGNq7g2+WpwKviz90ZLdy/LpHs7B/FRgJMnRozx/sO9nLyG9mZHvgcyg3FDcCCU0o1bpHN1u+Oqptme1KXYYmBgtUA4/2/aG+GbmX1+MXFHyh+DUP/Tl2DXqj2/ZXAKnmbkHIArA+CdLyNXd6/jS10uzXU19AXY/Afr/W5XtzAvt4/4z55YZHIpoqyi3fHexaGJ2uugN7A1c3PZim8FXBF4o7t2fn9z35KePLN0Jgz+bchPf3XX9kZw9n6TwM5uid3TW9YEI8qIPejuHlGFr7j/Y9vTM2Ph5kvaQ4bXz5hgfZHpJtTfjljCdMURie8VQDrbcN/i2zMmmA4bkKdbR4zKxta6ZUjw2hSUbrcDfJPcW27cUhq4ZUc7eg76kKbXnfHORKeTYECVGczm9z9JslfmM/pdE5nT04Q5TqzKhODxNFNyokU3RW55JTqu1yP1ujeeqY95jaBOdCoV+OM9K6oOsVUnjw0s13SnW6xPUq21S1ucdvK5lVy2ZZ93cYm7iq7RLpebylky5Xyp2SzGHj9z1Vz0Nlyy/3W2Vdc60MdsmU8kyJFJ6qNHTuFrVvCuArI3+nxAHa3JdEoQDNy8Npij3P6Kwi/aq2TSX4dpknb/yFUp4d4foV5SA7GPzdjBTf8E6CcifcbMK1MBOKzTYriKV4E5OI3AZePG4dRpwD3jpuEUacFK0nRSckrYvCSdG6bDqfUtXz0/5Ylpr7i7NeFbVsEJHnk9YHi5OiE5LWReGkaD0pOiG1uSckJCSsJFJwT0hISFhBLEtw//hxC4hA0vrgcVJ0QtK6KJwUrSdF53J0qCYkJCQkPFgsS809ISEhIeEBIgX3hISEhBXEsQd3EfmAiLwoIi+JyLNLoOcTInJFRF5ordsRkc+JyLfC95mwXkTkl4P2r4rIex6izidF5HdF5Osi8jUR+btLrHUgIn8gIl8JWv9xWP+0iHwhaPp1EemF9f3w+6Xw/zsfltZw/ExE/khEPrPkOl8WkT8WkS+LyPNh3dKd/3D80yLymyLyf0XkGyLy/mXUKiLfH+zZfG6JyC8uo9ZOqOqxfahn5vh/wLuAHvAV4AeOWdOPAe8BXmit++fAs2H5WeCfheUPAf+Tepzc+4AvPESdF4H3hOUt4JvADyypVgE2w3IBfCFo+A3gw2H9rwJ/Oyz/HeBXw/KHgV9/yD7w94D/DHwm/F5WnS8D5ybWLd35D8f/JPC3wnIPOL2sWluaM+B14Kll1zpV/7EeHN4PfLb1+znguWM3CrxzIri/CFwMyxepB10B/BvgI9N4x6D5t4CfXHatwDrwh8CPUI/0yyd9Afgs8P6wnAeePCR9l4HPA38J+Ey4aJdOZzjmtOC+dOcf2Ab+dNI2y6h1Qt9fBn7/JGid9jnuZplLwHdbv18J65YNF3T/XbCvAxfC8lLoD80BP0RdI15KraGp48vUL1P/HPUT2w1VbWbrausZaw3/3wTOPiSp/xL4++y/HO/skuqEetqs/yUiXxKRZ8K6ZTz/TwNvAv8+NHf9WxHZWFKtbXwY+FRYXnath3Dcwf3EQevb89Lkj4rIJvBfgV9U1Vvt/5ZJq6pWqvqD1DXjHwb+3PEqOgwR+SvAFVX90nFrMeJHVfU9wAeBXxCRH2v/uUTnP6du6vwVVf0h4C5108YYS6QVgNCv8lPAf5n8b9m0zsJxB/dXgSdbvy+HdcuGN0TkIkD4vhLWH6t+ESmoA/t/UtX/tsxaG6jqDeB3qZs3TotIM79RW89Ya/h/G7j6EOT9ReCnRORl4Neom2b+1RLqBEBVXw3fV4D/Tn3TXMbz/wrwiqp+Ifz+Tepgv4xaG3wQ+ENVfSP8XmatU3Hcwf2LwLtDNkKP+jHo08esaRo+DXw0LH+Uun27Wf+zocf8fcDN1qPbQiEiAvw74Buq+i+WXOt5ETkdlteo+wa+QR3kf2aG1qYMPwP8TqgtLRSq+pyqXlbVd1L74u+o6t9YNp0AIrIhIlvNMnX78Ass4flX1deB74rI94dVPwF8fRm1tvAR9ptkGk3LqnU6jrvRn7q3+ZvUbbD/YAn0fAp4DRhR1zh+nrod9fPAt4D/DewErgD/Omj/Y+C9D1Hnj1I/Gn4V+HL4fGhJtf554I+C1heAfxjWvwv4A+Al6sffflg/CL9fCv+/6xj84MfZz5ZZOp1B01fC52vNtbOM5z8c/weB54MP/A/gzBJr3aB+AtturVtKrfM+afqBhISEhBXEcTfLJCQkJCQsACm4JyQkJKwgUnBPSEhIWEGk4J6QkJCwgkjBPSEhIWEFkYJ7QkJCwgoiBfeEhISEFcT/BzfH2bYvZyYeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAA8CAYAAABPePC9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAASqklEQVR4nO2dW6wkR3nHf19Vd8/Mue/FF7xrmSU4JMYCAxYQBUUJCYkhCEdKHkyQQhQkvyQKiXIRiKe8JQrKTSJEFhBIRCCEQHAQcbhKyUMAY4McY3vxAo7t9a53l91z9pwzZ6Yv9eWhu8/OuXfPmZkzc1w/aTSX7q76qr7qf1dVV38jqorH4/F4xhdz0AZ4PB6PZ3e8UHs8Hs+Y44Xa4/F4xhwv1B6PxzPmeKH2eDyeMccLtcfj8Yw5lYRaRO4SkdMickZE3jNsozwej8dzDdlrHbWIWOB7wJuAZ4EHgber6mPDN8/j8Xg8VXrUrwXOqOoPVDUGPgncPVyzPB6Px1MSVNjnBPBMz/dngdftmujclIbXL+zDLI/H43lhkVxYJL3alu22VRHqSojIvcC9AMF185x6/719peOc4FxuqzGKMf094q4KqlLYpsim4meZIUsNqoINMoLA7TvNfujHjoOg0wnJlkNQwcwkNFvxQMp/mCjbR9lGSoxxQ6urftpkHFuS1QgywbRSmlPD92V7pQFXovzLQszUbHd9myp01iLcagAGwpmYqJEO16ABEseWtJtLadBIiaJsx31Lf3XXQnQxAgXmE1ozXZ76o/t2PK7K1MdZ4Oae7yeL3zYZoPep6p2qemcwN7X+e954qoutc0KaBKSpXRfsfnFOyDKDc1uLmWVCshaStgPSJKBqyJPyQrL5ZOyXDXakFhHFmPET62w1pHk2pPVMgFsOD9qcsSXLDFkmG16DaivboSq7tvPtSOOA8EJI87kQtxwO1b7cRmAxZP60Yf60gcVoy/nmlvP2FZ0PSNb6a191taY8prSxfNUl7QbIlQgWI7LE7miHKjhncv1Yipj9vmXuSQuL0Z55VOlRPwjcKiKnyAX6HuDXqxQgF7S88dTuVRQ9k7Li+r3il5Vfiv6WXkefjTRv3Ns7I8+nboKCSw3dboAIWOv6Hk3UyrYni11tVpDiNSqGMYI5CHZqK4NNv84BxcuBDFmkr+UpiOp6O9qwSQWJDeEqZJEQp/VsujaSMesiWaWtlMIJ1Bb4rYmBAJoZ0tQCbOhwlfY4J6gTyARx5K8KWe8p1KqaisjvAP8JWOAjqvrdvY+DteUG5kqY99uPdWlNx7vuD/l0RxCmecU7Ie6GiHEEQX3hMiZ3WJpY4tiC6PrQxFpFmylKLopVRaC0YTvHOiekae54axVr9+4Zl3Y4J8jzDaILTbIGrL04ZnphrXJZoYbo9uyfJBbnDEaUIMx2rGM7k7B2opiSmklq2dUPqhB3Q9JOgBglbCWEYba+Dfq/eA8Ta922wrljR2GfiFxrZ1XFJmikJNcbkkwwU+n+RWoPRICFmKWX5UN9PbJxqkVVaD1vuP7hDslMwNljFo5XT985Q2exibQt2syIFrrrbWXnY/LpB9cJIHA0puNaOtBL0EhJj+YiTdvSXZpGI4eZSTBWsUGGtY4sM8RXG0hswCjLpxxq8rrZM48qhqjqF4Av1C2ArAZMnTc4C+1WALsIdUk5L12eqFlqMFZQW3YDKuYtZcNV4k4AqwFqFBcokFdcEJQnfjXvXEtze5wTXGZRBZEMa/dO01q37kS50uL4IwnxvOX8cQsLlczqG9Xc3iw2aOCwgWOnOm61YmSqu37csIfLAFlskVWLWiWLMsIw23Axyut56GZUprfN9ZJPS8ie7Wd/eVYnijKax1eB0flyeq6Dmc87Hm5TnqrQ/JESfutJGseOEL7mZK20s0ywVy2NHxniOSGbTfYUalXBtQPsssU1Da6Z9n1/KIoyGo2UJLEki9M0LxnSKSE2ijbyNK11uMxgVi3BipAsOBo3rxAG2Zb62I6B3UzcDo0c8axBLRDWb6BiHMZKX3NPG9KxiovyMcYwew8i+XBHVWrlU5YvmVFWbgpIpwWNhn8zpczXBA6xe9dN2ZRGNfshgcv9ZsBIOZLpbx7xIMnrtV6bOIyUPnSbREkEOkeF9I6XkswGJNNKhT7OhuNdU0lmlWzKEVSoZxGFyOFaBg0rzj/sklY5UnJNRzIrZA2FUJHAIcUUiBjFNR2pM3mP2yhGdEt9bJvHMP44oPXSm7Rc9XFtsn6Muj5jSu8FqeqNoVHifXl4OAhf7panMQ5bbE+dqW1X70WwzrEiOtA62MuO7W8y5vv98A/vY+3Mc8Ndnrcd81Nr3DC1jFPD+dVZVjqNWsdb47BFD7Uf55VEQUorSnAqdOKQJKtzva6OiBIUV89MpZbYiiinjl7m9rnnWMkaPHzpZhbbraHY2Utos/VRQJLZXet41ALdCBNaYUo2ZL8Nm35FZJgchB075SmivObGZ7n72LdZzKb41Lk7efrKkcrpGuM4OrXGXKPDahJxaWWabI9zT0SZbsQ0g5TEGVY6jT2PqWNHNw1Y6jZJMkOaWTJnsMYx3+rQChJWk4irnUZljRiqUMdpwFLcwqmQZPUrQYHMmfwm9T4aVuYMcZoXNRtyA81U8ru/feRztdvkqfYxOllAPCJRylTAmUrDr1HjnCEuLh7D9tswqTsV9kLk+bVZvrnyEtouYiWu16FTFTppgEiDThpUmprTQpNELGlmBzKd12tHklmSYslkmbYCnTTAqay366pUEmoReQpYBjIgVdU7qxzXSQLidBrYOi9VhUEN/zNn6CR5/sNe09pv+qrClXaLq8WoY1S9R+cMOuDh36BInSGN8zYwjvbVYdLtHyaqwrmrc1xqvwxVIU7rtX1VoR2HrMVhvuqwom7EabA+ihyEfzbbUaZZvjtn6CQBnSSonWedHvXPqeqlGvuPVU9oEk6UzJl9D7/6YVzrZlzt8gyevAfaf+ekn07dMFa87GVHv53P8btj5fF4PJ4NVBVqBb4oIg8VMT08Ho/HMyKqTn28QVXPisj1wJdE5AlV/a/eHTYHZfJ4PJ5e/PLOjdSpj0o9alU9W7xfAD5LHqN68z7bBmXyHB6uLez3KxgmnVH7sje/7fIsg5GNY0CyKtSty73qYzN7CrWITIvIbPkZ+EXg0coWjTnjIjrjYsdOeJE+XByEL4VrT7fuut8LpI1VrQ+oNvVxA/BZyYMqBMA/qeoDfdq2JyK64VHT/QyTdnvQwBhHYBwiur4gvU6agxq+iSihzWrbMWqiIGWu2cWIcrXTZC32oU63Y7MAjiKWRt02GdqM2WYXaxyrcUS7u3eYzf0y3Yi5fnoFgAurM1sefmtFCfPNDqkzLK011597mARCm9GMkjzOdBLsunql9NV0I+Zoq40R5VJ7es+HAatEz/sB8Mp6pvePESUqgiXFqe17eV8p+EaUbJuTJTCOuWYXEa3cWHsvIio6kHXegXHMNOKRnjT9sNDq8LpjTxGajAcv38LTcfUnx14olE+mbhbquk+p1s2zt51XyWeqEfOTR84zF3R4/OqN/F98ZOgXk5tmlvjVGx4G4HMX7+Cxzo3r20SUG2eWedXCM7SziAcv3cKllckR6qlGzImZJZwKZ1fmSdZ2F2oryo3Ty9x13aM0TcLnL76CJzo37JrH0Guj7tXeiGKLeSojht1jYO2dlojuGHPXGkcgjrWaQ63ypBgEZY86EEenmKMbVUSzOoQm40i4SigZkdmPVw4/W4buQ/alqdl+BZgLOhwJ2jRtMvB4F9sxFSScCK4A0LRbw+Q2bcKLwkWWbZPQ9te+RlGO7bCi62UKiwv1TnE+pHifCmJuCq8QScZUEO853TNUoW5FCXPNDk6l8nAm2/RkkjWu1tNGJWVPRlSwxhEFGU7zhfWueLBkNY4Q0cpPQqkKukMPHTYOe6uKbWmHNY75RodTc5dppxHPrcyNpHddBsNxKrtOuyzHDR5auoVAHJc7o7lZHNqMoIi13E3tWAaq6mW997zN0HeYIlJ2Gqqm300DTi/fQNMmXFqbGYm4nWvP8e+LdxSft64KOxK1ua15lsvZDP8d3For7d64Hd3MstqNKrWVKEiJgozMmX21r7Uk5Fx7HkPe6To23SZ1hnYcbhhRiShTjZiGzWinEV9eejmQ18eBhjmdaXR5yeyP6DpLkh2vJNTOGeKiUEEhsJkTkj56mWXFR0HGdJTHwr7aaRCXQl0IYZ10d3Nmb1CmqkGkSjuMKD82f4k3HnmCi+ksDyQvH7pQl/YG1pE52fWewNW1Jo8Xw7NRxQUJg4zZRrd4YjP327izXfvYfAEfJP2MvjpJwFNXjiDs/z5QVS6uTPPV1R8HtsbbEVGui1Z4ZbTG89kqM0F3uyR2xIiy0Fzj+tYyl7vTdJNgz7YiorSihNmoSzcLSLMW/a436SQBz6czBMZx4+wyx6dXWEkaXGA2j9mTWhx5z3smipkNO6wmDf7n/CnSzFQanQ83HrUKqRqcmtpBTwbZeJyW7xvTHJfphbKnnjpLx4Ukbvzm5w4iHIBqfvEYx4BRk8xBTK05Z3YVwq4LuJxlLLoGqfb3OHjqbF8Xnv3WRVmfGXnHy/VoXm/aZTt2aoidrdWLH6oiLHcjvr90HKfS1yqBTAWX2n1Hz0syy3I3Wg+XOix60++nsTy9fIQH0tuJnWVxrTkME7fkmRaji/3W8TCIU8tVze+GD9Nvw2ZzcB7PRlSF71w5yZ+mbyJ2AWdX6j0w51S40mmxmkTEWbUFCFpoUpJZMjcYXXAqXF6bYjWJyIoVIFlPZEqnwuJaixUbEadDiJ7XL90kpJv0v4xrUHOSzhm6IzjR9xs9b7HdGkkM6l7Ged73oIJUDQMv0jujKpxbmuPc0lzfx7e7Ee2ax8VpQDzAP1IqxX+nTqmud1jra+LhOAs8Ho/nEDOUv+ISkYvAKlArLOqYc5zDVR44fGXy5RlvDlt5YLBlukVVr9tuw1CEGkBEvlX1DwYmgcNWHjh8ZfLlGW8OW3lgdGXyUx8ej8cz5nih9ng8njFnmEJ93xDTPggOW3ng8JXJl2e8OWzlgRGVaWhz1B6Px+MZDH7qw+PxeMacgQu1iNwlIqdF5IyIvGfQ6Y8CEblZRL4mIo+JyHdF5N3F70dF5Esi8mTxPlGxPkXEisi3ReTzxfdTIvKNwlf/LCLjGWN1G0RkQUQ+LSJPiMjjIvJTh8A/v1+0t0dF5BMi0pwkH4nIR0Tkgog82vPbtj6RnL8pyvWIiLz64Czfnh3K8+dFm3tERD4rIgs9295blOe0iPzSIG0ZqFCLiAU+ALwZuA14u4jcNsg8RkQK/IGq3ga8HvjtohzvAb6iqrcCXym+TxLvBh7v+f5nwF+q6kuBK8C7DsSq/vhr4AFV/QnyeOmPM8H+EZETwO8Cd6rq7YAF7mGyfPRR4K5Nv+3kkzcDtxave4EPjsjGOnyUreX5EnC7qr4C+B7wXoBCH+4BXl4c87eFHg6EQfeoXwucUdUfqGoMfBK4e8B5DB1VPaeqDxefl8lF4AR5WT5W7PYx4FcOxMA+EJGTwC8DHyq+C/BG4NPFLhNTHhGZB34G+DCAqsaqusgE+6cgAFoiEgBTwDkmyEfFH15f3vTzTj65G/gHzfk6sCAiLxqJoRXZrjyq+kVVLR88/zpwsvh8N/BJVe2q6g+BM2zz37L9MmihPgE80/P92eK3iUVEXgy8CvgGcIOqnis2nSf/m7JJ4a+AP4b1IGbHgMWeRjdJvjoFXAT+vpjK+VDxf54T65/iD6TfDzxNLtBLwENMro9KdvLJYdCK3wL+o/g81PL4m4m7ICIzwL8Cv6eqV3u3ab5cZiKWzIjIW4ELqvrQQdsyIALg1cAHVfVV5OEKNkxzTJJ/AIq527vJL0I3AdNsHXZPNJPmk90QkfeRT5F+fBT5DVqozwI393w/Wfw2cYhISC7SH1fVzxQ/P18Oz4r3CwdlX01+GnibiDxFPh31RvI53oVimA2T5atngWdV9RvF90+TC/ek+gfgF4AfqupFVU2Az5D7bVJ9VLKTTyZWK0TkN4G3Au/Qa+ubh1qeQQv1g8CtxZ3qiHxy/f4B5zF0ivnbDwOPq+pf9Gy6H3hn8fmdwOdGbVs/qOp7VfWkqr6Y3CdfVdV3AF8Dfq3YbZLKcx54RkReVvz088BjTKh/Cp4GXi8iU0X7K8s0kT7qYSef3A/8RrH64/XAUs8UydgiIneRTyG+TVV7I6veD9wjIg0ROUV+k/SbA8tYVQf6At5Cfjf0+8D7Bp3+KF7AG8iHaI8A3ylebyGf1/0K8CTwZeDoQdvaR9l+Fvh88fklRWM6A/wL0Dho+2qU4w7gW4WP/g04Mun+Af4EeAJ4FPhHoDFJPgI+QT6/npCPet61k0/I/2P3A4VO/C/5apcDL0OF8pwhn4sudeHvevZ/X1Ge08CbB2mLfzLR4/F4xhx/M9Hj8XjGHC/UHo/HM+Z4ofZ4PJ4xxwu1x+PxjDleqD0ej2fM8ULt8Xg8Y44Xao/H4xlzvFB7PB7PmPP/jZPebZ2of6QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# compute gradients in torch\n",
    "samp = [0,1,2,3]\n",
    "model.zero_grad()\n",
    "out = model(torch.tensor(X_test[samp].reshape((-1, 28*28))).float())\n",
    "out.retain_grad()\n",
    "loss = loss_function(out, torch.tensor(Y_test[samp]).long()).mean()\n",
    "loss.retain_grad()\n",
    "loss.backward()\n",
    "imshow(model.l1.weight.grad)\n",
    "figure()\n",
    "imshow(model.l2.weight.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9288"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# copy weights from pytorch\n",
    "l1 = model.l1.weight.detach().numpy().T\n",
    "l2 = model.l2.weight.detach().numpy().T\n",
    "\n",
    "# numpy forward pass\n",
    "def forward(x):\n",
    "  x = x.dot(l1)\n",
    "  x = np.maximum(x, 0)\n",
    "  x = x.dot(l2)  \n",
    "  return x\n",
    "\n",
    "def numpy_eval():\n",
    "  Y_test_preds_out = forward(X_test.reshape((-1, 28*28)))\n",
    "  Y_test_preds = np.argmax(Y_test_preds_out, axis=1)\n",
    "  return (Y_test == Y_test_preds).mean()\n",
    "\n",
    "numpy_eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x131624460>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAABZCAYAAADFEWQMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdpklEQVR4nO2dbagk2Xnff8+pqu6+b3Nn7szs7OzMarWyZQd/CLYibIkYY2IcJBHsfDBBSsBycFhILIjJh7BLIHE+BJJ8CIkh2BGJEgUS2Y7zYiEcFEU2GAyRtbIleSVlpY29kna9u7M772/3dledJx/qVN+6fbu7nnN3em7fnvOHpqur/1X1P0899dSpc55zSlSVhISEhITVgjtuAQkJCQkJDx4puCckJCSsIFJwT0hISFhBpOCekJCQsIJIwT0hISFhBZGCe0JCQsIKYiHBXUQ+ICIvishLIvLsIo6RkJCQkDAb8qDz3EUkA74J/CTwCvBF4COq+vUHeqCEhISEhJlYRM39h4GXVPVPVHUI/Brw0ws4TkJCQkLCDOQL2Ocl4Lut368APzJJEpFngGcApN/7C8XF8wuQ8oDgAZV6Oet40lGgkv3fmYLMZNdcpeaokS9a35YrATeHr4APHFrL8/Y/WY4uPQ287Ou3oJJ6v87AP9FaqH0nRgvY+I0Wp7ZqWqz2WXylLpNMWd/2fWmth1pj28aT/DYch/cPtT2R+lvYvxYeUQxffvUtVZ0aPBcR3E1Q1Y8DHwfoP31ZH/+ljx2XFBua4C6Wi67t4F03gwnvtfJF9y+wLn7DsfBnbWvhNroWxT+JWmL5y6RlHn/e+nmY3GYef5bOmOtrxfGdn3vu27P+W8Q971Xgydbvy2HdyUUlMJLZtYw2NHCHDiphbWsP6aqJDx2UdW1zsDmczy9DrQWQzFMMyvl6Sqk1qZD1K7J+1V2GUI58UCLWGqeH/mZHWSf4G6fv2/ghkGxa+ZUguWewObRpKYXexrDblo2WUtjY3iXreZMWKmHrzD2bLcOT3NaZe91lDb7mCs/G9q5N+0goBiXrpwx8X/MHm0OKtdHB9ZXU31P4vY0hkvv962boxtfDIT2lsHZqt67FN7zRjP0DIrC1c7f+v+y+Jl3uWdva6y7rCmIRNfcvAu8Wkaepg/qHgb++gOM8NLh+RZZ7vAp+mDGvD1oyT76+HySGe8VcfrZe4rIK7x3OeUaj2fsXgWxzhPeClo68V283U0vgV8MMRMmLktHe/FMuAqoguScvKqrSdv/P1ow3jXCMfGPEsMOWY37myfsl9+/3TPzGpsNdm3vnmyPUC+Uws2lZL9nbLfBl952m8Z27dwZTg9Uh7RsjBLh7Z9BZVnFKfmqE9477d3vd2vNaezXKqEoDv/DkGyXDvXzctCICFB6XKb4StBVcG36eV4goZe5xouPata8yquG+PzV6RJTe5vBAeX2VHdp/s83eXkGxtYd6Fzhupq20cuzdLzrLuop44MFdVUsR+RjwWSADPqGqX3vQx3mY8KXDG4OcemEU4UzV0FGFB6iK+cFFFcq9fc6oI3gd4KuwZwgAzUWilbB7p5vfoBo6qqGNr9qt/QD/bdjUgrZNH7SWGN8BqGK0xNqxitM+ja+hnbyaUmNu+O1t5t3yp/G74EeO4chuT1XQCPuvEhZSalX9bVX9PlX9HlX9J4s4xkNHV1viJFcFSkfvz3rd2zb/72UUr3XwW//J3Zzsqv3CyK/0kNvGYKBC8XoPrMHGC/3v9G1NVwBDx9qf9M12lXs5g5cj+LcK8jeMNycVitd6yC2jLb0w+HYfuWe0ZSW1dmtQGjoG3+7buCrIrYLeK/ayuutF7WdGfvF6z+5nvi5rdq0Ybz/+TEPpWP9WH+5n3VyAvazmj5zNF4au9uNHEMfWoXqi4EP7n2DIlmm1A3oYbXbwq+DMYZNyQ+d3EnnqZ2OnaKb4QYf2JttBhWqgaM/eAVUN1J7loVBueFsWRsDwlF2LOmV0yts60LxArlQb9v6Cal3RvqENPZzf4SmP5gZ+8J3htrdl4wTfGW4b9h342vOMThluHEG7H3g0N/YfVVKf12xifZPh1T4fY77iC9337QaHsmXq/qPhqZAVU01wJ/dPvf1wW0MmGONrYSYypdw02nLFkIK7BU5tFybUzphPph7Od779XDFfdxLNQzvYZkqnqoYvihYRTi6KFh3a23DgC+zBPQ/7tyJT1NpiEW58YhWTKZpHnN9C0SyzlbVJPW2ClUELgFqfqXMPVWY7T41vVkbtDd85tO3/01IUm/WFUvVDJeKAb8/ig++F41huli7cOPKYa9JGWzWk4L5IOEUHEQHVYas9NohxcIBeXM1aB5W94U4UXa/sNwMBH8PPteZbUXg0wru1b6xZN/y1av8m3gUJ2q22d2H/VhQezSJObCRf+8Ynpoa/Vtmf+ETxmxF8R823nqvYa2qF8Gj2NDwseCG7kdvb60shu2XvUGPokHt2vruTw579lLtbefeTRAMV8mu5KSMEAA/F1QjbDB3FtYhovedq/UZktzJk12jLpqxDoy291Nqt/RFV2L8Ve478up0v9zKym3a/yW5ldj8LtjHzvVC8FWHLUui9GWHL2GtqhZBq7gZIpoir2/nmpV1BaAJsNX/48zr3ydTlHnGgHqTn8f1qZoAUqVPB1NcpYm5jhK5Nzwc+wG+yBc7tIWW3ftU6zU4u7CGj+fzxdrnHX9rt3P9Y16CivORNF6kIyOaIcq003Txc7uGUR9fdXNuP+YXHnx8eSrubpcX1KqqL3qRFMkUKT/n4fD84oL3wlBe6a5vj85sppaGmL05xueK3SqrNbi1j/tmQ464hFVLq60G9oC0b1HyPPlnnKmkYRdrk66vnQOaQOMX1PP6pw3li6jm0f6j9ZvRkRZZpnQnjD6dLHkDhqXYezZp7Cu4G6JR825lc5UAubxdi0uRUQVsZF13bTvLVkK0xToX0QjWMeHSPtJFFy1H5MTaFOr0uRkvM+Y2xC7w9f+jkx57TKXxVQA8H3Qe1/85tgi2npWImHEQK7ga0RwpaaqWT7ZPTLoRD/NZw7k4++7VrC7/RLJmCF1tNXKjbNSP51kAmoaNxEfz2OZhnm0N8NZa16SQ12OaAPxj2f2TtBv6kr5n5E9pnXQ8HavXaeqKcUf7Jp4ADCM11k/Ya+6XO5kwrg8WWq4bU5m5FV9ZLG9L6WObakNZ3F39Sh3EuD/O0AEdF7BwfgqmZApoLFHt7fkRwPMC33sSMgX1/I8w3jujA7nRmTXqmFuP+x/xp2medb6EO7A2nuRFOS2ts89vXzCxuAzVwGqpGlHXFkGruBjSj8h40F+Idr11ztWzbcKJ1PcRyHLuWCP4y2XHhWmbwm6aZh61n3rETDiMFdwuaiZKE7lzcZuKwBl254qWrB2M46u+u9MaRq//PtN62Kwd/5PZT9kqpl60zW46kW3+DZqInay59uxwWLaXUXEsKXBk6UhelZST2NNTGd6xaYrTH2iX2nM7iNwP1son1Xg5mVzlC80nrd/v6mbxW2nAc3j/s3wya69Fa9kcQKbhbEDuIKWIUqGngRhvti96ybZsfM2gothyZMThO02XREqP97djUoiXGLjG+A3HaY+0Sq30Wvz1Qrw33gPY/D42PxfjaI4oU3A2ISoUM6WMNfEcq4TgVUus2VF/Obj89lApZ+Dobw5gK2SxbUyFdrp362+VA6Nx/Wxdqyw5p0kvn2eaoWqDed2dK3RG1NL5j1hK0WzJ4xlqMGTmx57TNh30fnZ8Kub9j9SCtYkxNhZwxEGxmKmSmiOj4mrGet0cRKbhb0HSeWTrFVPCGacHH9MqhPnSihe1ncps2yiZLYHLujhn8sWbDRTDmHqEciJqCxmQ5LHxfRvAjtNR8+74hTkvjO2YtkW3uUVrCOTV3BLd8YJwiqyDMCKhTfEb9wf+7+DO5DbygyPiaSe3vs5GCuwExnTiT3K4slWh+6+JXL1H8mAthfBFbg1Lgm/dv0H5UfqOlnQZq2beF35yvGC0N36oFbFoa/lG1x/Db+585d/rb2H+Un0ncTTBm/6uEFNwXjFinWiR/mbQsmr9MWhbNXyYtq8BfFaQ894SEhIQVRAruCQkJCSuIFNyt6HpDzDSuhd/mWvnTlh8Uf9a2D5ofY89l2vei+SdF+7z18z4x/C5dqTN1LlKbuwUHBjF19RJNDMwomD9gpJL9QUxK/cabeXnRpewP8Kjk8Nttpu0/V8YDXvIOPe1yxPDHg5iMDZylhIFDBu54sA72txkpkVqIHMRk1DIexGTU0mShWOeKj7HLeFASkYOYJvjjQUwT67sGMWUT57tzENMUnT6c2/EgJtIgphlIwd0AKTxuoOOX7c7NE888buDHU6f4YTY/z32tHOcMiws5yHPy1t1GOc4XdmtlZ567Wy/rPGVR3EbVmeM8znMP5ejSPy5Hv0Jcd17/tHJ0QZziNsu5tjmgZa2sxwwY87mzGC2NXYxaGt8xjxcI2i0zT4qA2xzhS2fLc488p+2yNmnA9Xw2HunpId+T3JOt1bmPSjuTpz6Y93Igf799rTQYZ+KG/PVDee6Fr/Pcw6sjrTn+jyJScDdAZ7ztfSrXS9Qb7N/OdLNd207yLQHjwJS/MeWInKo2atpcZWE2hUgtkXaJ8R04gj/EaInVPoXfNeVvuUA9UNtTEYgYg/GoIrW5JyQkJKwgUnBPSEhIWEGk4J6QkJCwgkjBPSEhIWEFkYL7AiGZcuHCTfM8JFm/4tz522Z+sTZi8/R9s55TZ+4x2ByauCKwc+42Wb/7xctQZ7Q8cfF6ncVggCs877z81vhVgV0oBiVPXbpqts1gc8iZs3dMXBE4d/42a1t7Nr5TLl28TrE2MvOfunT1wIvT58EVnksXr0eV9fHHb9jIwMb2Lucfu2Xm75y7bfYzEXjyiWtsbO/a+Jnyve+4Qm70s6zn+b53vFHPnmnk75y7beKuGlK2jAFZz5MXJb5ylHt5Ryqk0l8bIVK/R/LNq1tz+cWgJC8qytKRZcr1mxsz+SJ1QK8qhx/WWQb37/VnaxHIByPKvfo0D0c5e/eLuWVtp0Le3+uZs2CywnPt9obpVXgiUPRKXr26bUsndEqWe15584wphS/vV6jCjesbNi1rI27eXhvbaS4/nN/Xr26bMlsa3/nO6zsm2xSDOjX2z66c7k4pdUpvbcRolPHGlW2z9nt3+ty7PdtvGrjc0xuU3LixcWDiNsk8LlN8JQf8w+We/tqIa3fXUa3L4jKPCzfw0TA/kE3T8F+7eYqiV1L06hQYVaEcZYf23xz/T9/coTco8V6oymxumqkfOa5f2+ws6yoiBXcDqqGjGvZMXK2E3Ts2LsBoN2e0W5+GrnqgKgzv7QfnUUegVj3IsehqLhJfOu4bAkCDci8zp8Gpwt5du43Ux9k0Vkvbpp38yPMb4zvA2BdMWrzE2TFSuy/dVL6WbupUvbE+E8tvUmgrXOe10t7mUR3JmpplEhISElYQKbgvECKY26CPzI8Yei1Oo+ZQj+YvuqyLtk0MP4u3jZW/cNs4jefH2mZBthQJby+LseUjOj1BCu6LhFMeO3fL7IiuV3F2546Znw9GbJyydVwBbG3fp79h71A9s3PH3AkoTrn42A1z0JDM847Hr5kvvLxf1nyjbfobQ06fuWvTInD27B0GEbZ54rEb5AN7h+o7Hr9Wv1rQws89Tzx2I6qsF87ftJGB9a09zp21dzKe2blj9jMRuHzhOuvWzulM+Z5Lb5L1bB2qrvB87xNvIpm9c/rMjq1jfdUgugQz2fefvqyP/9LHjltGQkJCwonCd37uuS+p6nun/Zdq7gtEk4lhfoTMlGJgnzTD5d6cqgh1Fom1Jg4hc8NaExforUeU1Sn9jaH9qSb3cfzCm9PrIGR2WJ9SQlmt6Xgide3a+pQiTumtW7sM67Ja0zKhzuCJ8bNiUNpTYoNtsp7dNoPNod3PnNb8CFvGlHWVkLJlFojJbJVOfiWMKvsp8aWLmkApZlIniMzciM06icz08KVjr4zgjxz229iCy3qE7KCY/fuRi5pwrMk4sWLRtonJ4InNnFIvUfpXCZ1nWEQ+ISJXROSF1rodEfmciHwrfJ8J60VEfllEXhKRr4rIexYpPiEhISFhOiy37/8AfGBi3bPA51X13cDnw2+ADwLvDp9ngF95MDITEhISEmLQGdxV9feAaxOrfxr4ZFj+JPBXW+v/o9b4P8BpEbn4gLSeTBhGJR6ZH/uqMX8Efgxi+TEvWWje/hPDj9FzFNvE8GNfKLHIsi7aNtVD4Mcg1i9XBEftUL2gqq+F5deBC2H5EvDdFu+VsO4QROQZEXleRJ6vbttS1k4cVCDmxRE+ls/B15p1oYwMkCMXd2EMXdx7ZiNekoEnzjZVpC1HLs42Q0dco36kbRZZ1vIItonxsxh+4wfWpD0vyF5mt2XsNbVCeNul1jqXMjqfUlU/rqrvVdX3Zlvdc4AcJ5qBEJYe+klufmu+IzZccQpeyG7P7/Q8MLCodMj9bn4Ddz9DIgKquxcX8OqyGskKxQ3bRSoCVBLHHwnZXVtZRSC745A926vtRKC4mZmCRnOuiuuZyZa1HwTbGCBOYSTkBv7YN3ddfa6M/OyOw93LDq4XDmUuNevym7WftX172oA4EUCUXrBNFx8ALxTX6puBaZBdJWR3Hs3gftRu5DdE5KKqvhaaXa6E9a8CT7Z4l8O6E42Y+SkOcEUpz88fGHNg4qzcU52L4PcrtCNFrc332/Z0OQB/Jo5fPmYbBASAU0YXbXxVoOcZPR7BX6uo1mzpe6pQnbWVtRkWMroQoQXsZfUCmdrL6gXWKkpDWce+uVVSbhn2HfiTtpk1NKZZXza26Xjqa/jDJ2q+ZRI5cs/wcgS/8FRnYx6xVgdHvaV9GvhoWP4o8Fut9T8bsmbeB9xsNd+cXHiB0liLbR6pm0/XTaF5pK7CMbr4ZaupxNJW2cyqp8b9t8sRw2/0W2G15zJqiWli8ZFajqLd2nQWq30Wv+n/mLa+7ftNWdq/5/Ent52m08v+NRNT9kcQnTV3EfkU8OPAORF5BfhHwD8FfkNEfh74NvDXAv23gQ8BLwH3gL+5AM0PH0rdvuoEU7tDU0MSC7fVuaXUt9t5/upb//twjHn89pOEB6yp7k2ZzXyJa5yLseeYz/yyHlWLGvfb5lu3aexo3neEdpWD/mDaJpI7jd/4lE74nnI42B54kp0QoBP/tyEc3v+BY7d+J0xFZ3BX1Y/M+OsnpnAV+IW3K2rpkGn9sUAUehEel/u4xrH2yD/LiMo2J0aXO0I5YmAcwQgczaYxiBi1G60lxncgTnusllj+LB9wWn+mrY8YFRztYxBsmSK6BY9mT8OCMe5s8oK7XnQ+Brc7SN0N++g+9jLkrv3OILdz2LWPUnU3C3sTgQrZ1cL+mFwJ+Zs9exPB0NV8K3Yz5Jbdlu5mAR2d021kb/XAOuJXQ1mtTT+V1Pu3Yjcju2YrqwjIvdzsZyK1beSO3c+yt3rIvX3+tM7XMVTI3+jB0M3sqD2A0lG8EWHL0tXn9hHEozkud8EYdzg5xZ8uO5tnxvzc409H1Nz6FWp/1wG6FTfHRlQHrCjVTndZx8iU8tzIzu/5mm/FoEIH9lpkbGdzdTZCu0SWNVNzBy8Ag4qqb/MbVWC9RNfsCQL6Nm0zd25CUcrHar5pDsPcM3oswpa5x2+nDtWERSD2CXKRg5iOwo9BbFkXyV+4bYjnxyC6rIvmn1DbQLwfrwhSzd2C3YzsnsPnofY7r9YwcuQ38nGHW7Xh0fXZNWa5nZPtOnxPcXtCteZn17BVyK7n+IGia1W93Fd0YwbfC+5Wjj9VgkLxVsHodDW/XdRL3RY6dBRXc0bnSlObtNzJye+4ulbVNR7Ay7gZoSv1syYJ+dWCaqDoqe5apNzOcXuOaqujrAHuRoF4qE5VkHdoLx359ZyqsXtXy8xuRnbXUa17GPjOGqfcKnAlgd+h3cu+DxSG9u7gm+WpwOviDx3ZrRy/7tEs7F8FRoIMHdrzB/tO9jLymxnZHvgcyg3FjUBCE0q17tHNlq+Oaltmu1KXoYnBAtXAo31/qG9G7uX1+AUFXyh+zUN/jl2D3ui2/RVAqrlbELIArE+CtHzNnZ4/Ta14QXy9jWYKXY/Afr/W5XtzAvt4/4z55YZHIpoqyi3fHexaGJ2uugN7A1c3PZim8FXBF4o7t2fn9z35KePLN0Jgz+bchPf3XX9kZw9n6TwM5uid3TW9YEI8qIPejuHlGFr7j/Y9vTM2Ph5kvaQ4bXz5hgfZHpJtTfjljCdMURie8VQDrbcN143MyqQBhucq1NHiMbO2rZlSPTaEJhmtw90k9xTbtheHrBpSzd2CvqcqtOV9c5Ar5dkQJERxOr/N0W+W+I39lEbndPbgDFGqM6M6PUwU3aiQTNFZnUtOqbbL/WyN5qlj3mNqE5wLhX45zkjrgq5XSOHBSzffKdXpEterbFPV5h6/rWRWLZtl3d9hbOKqtkuk5/GWTrpcKXdKMoeN3/dUPQ+VLb/cb5V1zbUy2CVTyjMlUniq0tC5W9S+KYCvjPydEgdoc18ShQI0Lw+nKfY8o7OK9KvaNpXg22WevPEXSnl2hOtXlIPsYPB3M1J8wzsJyp1wswnXwkwoNtusIJbiTUwicht48bh1GHEOeOu4RRhxUrSeFJ2QtC4KJ0Xrsul8SlXPT/tjWWruL856VdSyQUSeT1ofLE6KTkhaF4WTovWk6ITU5p6QkJCwkkjBPSEhIWEFsSzB/ePHLSACSeuDx0nRCUnronBStJ4UncvRoZqQkJCQ8GCxLDX3hISEhIQHiBTcExISElYQxx7cReQDIvKiiLwkIs8ugZ5PiMgVEXmhtW5HRD4nIt8K32fCehGRXw7avyoi73mIOp8Ukd8Vka+LyNdE5O8usdaBiPyBiHwlaP3HYf3TIvKFoOnXRaQX1vfD75fC/+98WFrD8TMR+SMR+cyS63xZRP5YRL4sIs+HdUt3/sPxT4vIb4rI/xWRb4jI+5dRq4h8f7Bn87klIr+4jFo7oarH9qGemeP/Ae8CesBXgB84Zk0/BrwHeKG17p8Dz4blZ4F/FpY/BPxP6nFy7wO+8BB1XgTeE5a3gG8CP7CkWgXYDMsF8IWg4TeAD4f1vwr87bD8d4BfDcsfBn79IfvA3wP+M/CZ8HtZdb4MnJtYt3TnPxz/k8DfCss94PSyam1pzoDXgaeWXetU/cd6cHg/8NnW7+eA547dKPDOieD+InAxLF+kHnQF8G+Aj0zjHYPm3wJ+ctm1AuvAHwI/Qj3SL5/0BeCzwPvDch548pD0XQY+D/wl4DPhol06neGY04L70p1/YBv400nbLKPWCX1/Gfj9k6B12ue4m2UuAd9t/X4lrFs2XND9d8G+DlwIy0uhPzQH/BB1jXgptYamji9Tv0z9c9RPbDdUtZmtq61nrDX8fxM4+5Ck/kvg77P/cryzS6oT6mmz/peIfElEngnrlvH8Pw28Cfz70Nz1b0VkY0m1tvFh4FNhedm1HsJxB/cTB61vz0uTPyoim8B/BX5RVW+1/1smrapaqeoPUteMfxj4c8er6DBE5K8AV1T1S8etxYgfVdX3AB8EfkFEfqz95xKd/5y6qfNXVPWHgLvUTRtjLJFWAEK/yk8B/2Xyv2XTOgvHHdxfBZ5s/b4c1i0b3hCRiwDh+0pYf6z6RaSgDuz/SVX/2zJrbaCqN4DfpW7eOC0izfxGbT1jreH/beDqQ5D3F4GfEpGXgV+jbpr5V0uoEwBVfTV8XwH+O/VNcxnP/yvAK6r6hfD7N6mD/TJqbfBB4A9V9Y3we5m1TsVxB/cvAu8O2Qg96segTx+zpmn4NPDRsPxR6vbtZv3Phh7z9wE3W49uC4WICPDvgG+o6r9Ycq3nReR0WF6j7hv4BnWQ/5kZWpsy/AzwO6G2tFCo6nOqellV30nti7+jqn9j2XQCiMiGiGw1y9Ttwy+whOdfVV8Hvisi3x9W/QTw9WXU2sJH2G+SaTQtq9bpOO5Gf+re5m9St8H+gyXQ8yngNWBEXeP4eep21M8D3wL+N7ATuAL866D9j4H3PkSdP0r9aPhV4Mvh86El1frngT8KWl8A/mFY/y7gD4CXqB9/+2H9IPx+Kfz/rmPwgx9nP1tm6XQGTV8Jn681184ynv9w/B8Eng8+8D+AM0usdYP6CWy7tW4ptc77pOkHEhISElYQx90sk5CQkJCwAKTgnpCQkLCCSME9ISEhYQWRgntCQkLCCiIF94SEhIQVRAruCQkJCSuIFNwTEhISVhD/H6DP2bUbrPVnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAA8CAYAAABPePC9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAASr0lEQVR4nO2da6wkR3WAv1P9mJn73ocfeG2ZJTgkxgIMFhAFRQkJiSEIR0p+mCCFKEj+kygkykMgfuVfoqC8JEJkAYFEBJI4OHEQcXhKyY8AxoAcg714Acf2etfr9e59zqO7q05+dM/duffOvbd77szcmev6pNFMT3dXnapz+nRVddVpUVU8Ho/HM7mYwxbA4/F4PHvjHbXH4/FMON5Rezwez4TjHbXH4/FMON5Rezwez4TjHbXH4/FMOKUctYjcKSJnROSsiLxv1EJ5PB6P5yqy3zxqEQmA7wFvAZ4BHgLeqarfHb14Ho/H4ynTon49cFZVf6CqCfBp4K7RiuXxeDyeLmGJY04BT/dsPwO8Yc9EF2Y0unbpAGJ5PB7Pi4v04jLZalP67SvjqEshIvcA9wCE1yxy+oP3DJSOc4JzuazGKMYMtsRdFVQFVcEYh2wrvrUGmxlUhSC0BMHOY3ZLE0BE9z2+DNvlCEN38ERHQLsdYdciUMHMpdQbyVDKf5Totble+tnfsPOE8jaZJAHpRgxWMI2M+szoddlcr8GVON9YSpiZ72zuU4V2K8ZthGAgmkuIa9loBeqDKgPVQ5IEZJ3clYa1jDi2e+ahKnRaEbocgwKLKY25Dk/+wb27nldm6OMccFPP9o3Ff9sE0HtV9Q5VvSNcmNn8Pzee8s7WOSFLQ7I03HTYg9J1+s7tLKa1QtqKyJp5XlXT3H4xDsoWObIAEcWYyXPWdiOifi6i8XSIW4sOW5yJxVqDtbLlMyxb6Ydqbo/Wmr523o8sCYkuRtSfjXBr0Ujly2UEliMWzxgWzxhYjtn+aMyt5fYVXwhJW4PZV1Vfs0PGnu8q52WdELkSw3KMTYNd5VAF50zuP1Zi5s8GLDwRwHK8bz5lPNRDwC0icprcQd8N/GqZQuQOLTeeQVoVeesk/z3oHT+/g7Hp9He0OgY00ty4+yujSyWZVXCZodMJEYEgcAP3JqpQWl4FKT7jYhQ9mMNgN1sZbvpVTig+DmTETvpqnoKobtrRll0qSGKINsDGQpJVk+lqT8ZsOskytrLdvg6EggBqDVkWAGxpcHXlcU5QJ2Alv55cuWtqX0etqpmI/Bbwn0AAfExVv7P/edBaq2GuRHm7/USHxmyy5/GQD3eEUZZXvBOSToQYRxhWd1zG5ArL0oAkCUB0s2sSBIrWM5TcKVZJE/or1jkhy/IbUxBoqXS7cjgnyHM14ot1bA1aL02YXWqVlguq3yRUIU0DnDMYUcLI7lrHwVxK61QxJDWXVpJrEFQh6URk7RAxStRIiSK7uQ8Gv3mPkiBwfR3nrg2FAyJy1c7KOpuwlpFea0itYGaygzupfRABlhJWXpF39fXY1qEWVaHxnOHab7ZJ50LOnQjgZPn0rTV0VupIM0Drlnips2kru+FcPvzg2iGEjtpsUmr4s1/ZwlpGdjx30jQDOiuzaOwwcykmUEzgCEOLtYZktYYkBoyydtqhJq+b/SjV51fVzwGfq1YEkI2QmQsGF0CzEcIejnrzHFHCUDcvVJsZTCBo0G0GlMxbuoarJO0QNkLUKC5UIB+TzhWjpVskV9Psj3OCs0Ex1mUJgv3T7MphrUGuNDj5SEqyGHDhZABLpcQaGNVcXpsYNHQEoWO3Om40EmSms3neqLvLADYJkI0ADRQbW6LIbrkZDTqmOCp6ba6XfFhC9rWfg+VZnji21E9uAOPT5exCG7OYNzzctjxVof6CEn3jCWonjhG97sZKaTsnBKsBtRcMyYJg59N9HbWq4JohwVqAqxtcPRv4+VAcW2q1jDQNSJdnqV8yZDNCYhStOYiB0OKswWwEhOtCuuSo3bROFNod9dGPoT1M7IfGjmTeoAEQlTOm3gtPjMMEcqCxJwAJFBfnfYzedEZx0RiTt6iqpN0tXzqnrN8Qks0KGo/+YUo3XxM6JNi/jruqGdfoh4Qu15sBI92eTPVxxMMmr9dqNnEU6erQbXNKItA+LmSveTnpfEg6q5Ro42w539WVdF6xM46wRD2LKMQO1zBoVHL8Ya+0NuVwpPOCrSlEioQOKYZAxCiu7sicyVvcRjGiO+qjbx6jeHFA4+U3aHfWR7cQ47hrTzu9N6SyD4bGidfl0aJKb3JY+UF/+zHGERT7M2cqy9V7E6xy7rDrYD85+j9kzI/74e/fS+vss6OdntePxZkW182s4dRwYWOe9Xat0vmBcQRFC3UQ5XWJw4xGnOJUaCcRqa1yvy6PiBIWd0+r/Web7HXu6eOXuW3hWVazBt9+4RTLzcZI5OwlCuxmLyC1wZ51PG4HXYtSGlGGHbHeRs2gTmTUjFuW3fITUV53/TPcdeJbLNsZ/un8HTx15VjpdI1xHJ9psVBrs5HGXFqfxe5z7Ykos7WEepiROsN6u7bvOVXk6GQhK506qTVkNsA6Q2Aci402jTBlI41ZbddK+4iROuokC1lJGjgVUlu9EhSwzuQPqQ9gVNYZkiwvqh2xcVqV/OnvAPmsduo82TxB24YkY3JKVgWcKTVONm6cMyTFzWPUehsl3aGwSavfSeK51jxfX38ZTReznlRr0KkK7SxEpEY7C0sNzemmTwo3fcxB6ZUjtQFpMWWym7YC7SzEqWzadVlKOWoReRJYAyyQqeodZc5rpyFJNgvsHJcqw7C6/9YZ2mme/6jntA6avqpwpdlgteh1jKv16JxBJ9SJZM6QJbkNTKJ8VZh2+UeJqnB+dYFLzVfki0GyaravKjSTiFYS5bMOS/qNJAtJrQ7tgep2Obppdr+dM7SSiLaElfOs0qL+GVW9VOH4iWoJTcOFYp05cPdrECa1biZVLs/wyVuggzdOBmnUjWLGy35yDJrn5D2x8ng8Hs8WyjpqBT4vIg8XMT08Ho/HMybKDn28SVXPici1wBdE5HFV/a/eA7YHZfJ4PJ5e/PTOnZR9yFyqRa2q54rvi8D95DGqtx/TNyiT5+hwNY7Ci3vhxlFg3Lrsza9fnt1gZNNqW1Xrsre8Zc7b11GLyKyIzHd/Az8PPFpaoglnUgxjUuTYDe+kjxaHoUvh6urWXY95EdlYmfroUmbo4zrgfsnXdofAP6jqgwPKti8iumWp6UG6SXstNDDGERZ3tO6E9CppDqv7JqJEga0sx7iJw4yFegcjymq7TivxoU77sd3RjCOWRlWbjALLfL1DYBwbSUyzs3+YzYMyW0u4dnYdgIsbczsWvzXilMV6m8wZVlr1zXUP00AUWOpxmk8tTMM9Z690dTVbSzjeaGJEudSc3XcxYJnoeT8AXl1N9MExosRhHlAlyYKBp/d1Hb4Rxfa5WAJRFuodRLS0sW62KgEVHco879A45mrJWC+aQVhqtHnDiSeJjOWhyzfzVFJ+5diLhe7K1O2Ouuoq1ap59tp5mXxmagk/fuwCC2Gbx1av5/+SYyO/mdwwt8IvX/dNAO6/eDuPt6/b3CeiXD+3xu1LT9O0MQ9duplL69PjqGdqCafmVnAqnFtfJG3t7agDUa6fXePOax6lblI++/yrttRHP0ZeG1Xv9kaUoFiGbcSwdwys/dMS0b4xd6XIJxRHq8rYUpHusOaHd1vUoTjaxm0u5560By6RsRyLNojEEpuDaOXos6PrPmJdmgr2C7kNL4RtjoVN6kE6llWTM2HKqfBK8XtnFM16kPKSaJm1oE4UDGZfh7X6MxClHuShfyOze0TObiNPRJkJE26IrhCLZSZM9h3uGamjbsQpC/U2TqV0d8aqkPSsTAqMq7TaqEu3JSMqBMYRhxan+cR6Vyws2UhiRHRLfvulqbu00GFrt7ess+3KERjHYq3N6YXLNLOYZ9cXxtK67gbDcSp7DrusJTUeXrmZUByX2+N5WBwFlrCItdzJgokMVNXLZuu5T9d3lE6k22gom34nCzmzdh31IOVSa24szu18c4F/X35N8XvnrLBjcZNb6+e4bOf47/CWSmn3xu3o2ICNTlzKVuIwIw4t1pkD2VcrjTjfXMSQN7pOzDZJi1WIvT0qEWWmllALLM0s5osrrwTy+jjUMKdztQ4vm3+BjgtI7clSjto5Q1IUKiwcrHVCOkArs1vxcWiZjfO7+Gq7RtJ11IUjrJLuXsrsDcpUNohUVw4jyo8sXuLNxx7n+WyeB9NXjtxRd+UNA4d1suczgdVWnceK7tkg4QAGIQot87VOsWIz19uk088+tt/Ah8kgva92GvLklWMIB38OVJbn12f58saPAjvj7Ygo18TrvDpu8ZzdYC7s9EtiV4woS/UW1zbWuNyZpZOG+9qKiNKIU+bjDh0bktkGg74Ar52GPJfNERrH9fNrnJxdZz2tcZH5PGZPFuDIW95zccJ81GYjrfE/F06TWVOqdz7aeNQqZGpwWj3oyTCNx2n3e2uakzK80G2pZy6g7SJSN3njc4cRDkA1v3mM68bwYuEwhtacM3s6wo4LuWwty65GpoMtB89cMNCN56B10a1PS97wcj0+rzftrh07NSQuqNSKH6lHWOvEfH/lJE5loFkCVgWXBQeOnpfagLVOvBkudVT0pj+IsTy1dowHs9tIXMByqz4KEXfkmRW9i4PW8ShIsoBVzZ+Gj1Jvo2Z7cB7PVlSFb1+5kT/O3kLiQs6tV1sw51S40m6wkcYkttwEBFWh2YlJbYB1w/ELToXLrRk20hhbzACxRWTK7v7lVoP1ICbJRhA9b1A6aUQnHXwa17DGJJ0zdMZwoR80et5yszGWGNS9TPK472EFqRoF3knvjqpwfmWB8ysLA5/f7MQ0K5530EBQ/eRoFdHz9toP1X3i0bgKPB6P5wgzkldxicjzwAZQKSzqhHOSo1UeOHpl8uWZbI5aeWC4ZbpZVa/pt2MkjhpARL5R9gUD08BRKw8cvTL58kw2R608ML4y+aEPj8fjmXC8o/Z4PJ4JZ5SO+t4Rpn0YHLXywNErky/PZHPUygNjKtPIxqg9Ho/HMxz80IfH4/FMOEN31CJyp4icEZGzIvK+Yac/DkTkJhH5ioh8V0S+IyLvLf4/LiJfEJEniu+pivUpIoGIfEtEPltsnxaRrxW6+kcRmcwYq30QkSURuU9EHheRx0TkJ46Afn63sLdHReRTIlKfJh2JyMdE5KKIPNrzX1+dSM5fFeV6RERee3iS92eX8vxpYXOPiMj9IrLUs+/9RXnOiMgvDFOWoTpqEQmADwFvBW4F3ikitw4zjzGRAb+nqrcCbwR+syjH+4AvqeotwJeK7WnivcBjPdt/Avy5qr4cuAK851CkGoy/BB5U1R8jj5f+GFOsHxE5Bfw2cIeq3gYEwN1Ml44+Dty57b/ddPJW4Jbicw/w4THJWIWPs7M8XwBuU9VXAd8D3g9Q+Ie7gVcW5/x14Q+HwrBb1K8HzqrqD1Q1AT4N3DXkPEaOqp5X1W8Wv9fIncAp8rJ8ojjsE8AvHYqAAyAiNwK/CHyk2BbgzcB9xSFTUx4RWQR+CvgogKomqrrMFOunIAQaIhICM8B5pkhHxQuvL2/7ezed3AX8neZ8FVgSkZeMRdCS9CuPqn5eVbNi86vAjcXvu4BPq2pHVX8InKXPu2UHZdiO+hTwdM/2M8V/U4uIvBS4HfgacJ2qni92XSB/Tdm08BfAH8JmELMTwHKP0U2Trk4DzwN/WwzlfKR4n+fU6qd4gfQHgafIHfQK8DDTq6Muu+nkKPiK3wD+o/g90vL4h4l7ICJzwL8Av6Oqq737NJ8uMxVTZkTk7cBFVX34sGUZEiHwWuDDqno7ebiCLcMc06QfgGLs9i7ym9ANwCw7u91TzbTpZC9E5APkQ6SfHEd+w3bU54CberZvLP6bOkQkInfSn1TVzxR/P9ftnhXfFw9Lvor8JPAOEXmSfDjqzeRjvEtFNxumS1fPAM+o6teK7fvIHfe06gfg54AfqurzqpoCnyHX27TqqMtuOplaXyEivw68HXiXXp3fPNLyDNtRPwTcUjypjskH1x8Ych4jpxi//SjwmKr+Wc+uB4B3F7/fDfzbuGUbBFV9v6reqKovJdfJl1X1XcBXgF8pDpum8lwAnhaRVxR//SzwXaZUPwVPAW8UkZnC/rplmkod9bCbTh4Afq2Y/fFGYKVniGRiEZE7yYcQ36GqvZFVHwDuFpGaiJwmf0j69aFlrKpD/QBvI38a+n3gA8NOfxwf4E3kXbRHgG8Xn7eRj+t+CXgC+CJw/LBlHaBsPw18tvj9ssKYzgL/DNQOW74K5XgN8I1CR/8KHJt2/QB/BDwOPAr8PVCbJh0BnyIfX0/Jez3v2U0n5O/Y/VDhJ/6XfLbLoZehRHnOko9Fd/3C3/Qc/4GiPGeAtw5TFr8y0ePxeCYc/zDR4/F4JhzvqD0ej2fC8Y7a4/F4JhzvqD0ej2fC8Y7a4/F4JhzvqD0ej2fC8Y7a4/F4JhzvqD0ej2fC+X89uud/A9MfwQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# numpy forward and backward pass\n",
    "\n",
    "def logsumexp(x):\n",
    "  #return np.log(np.exp(x).sum(axis=1))\n",
    "  # http://gregorygundersen.com/blog/2020/02/09/log-sum-exp/\n",
    "  c = x.max(axis=1)\n",
    "  return c + np.log(np.exp(x-c.reshape((-1, 1))).sum(axis=1))\n",
    "\n",
    "def forward_backward(x, y):\n",
    "  # training\n",
    "  out = np.zeros((len(y),10), np.float32)\n",
    "  out[range(out.shape[0]),y] = 1\n",
    "\n",
    "  # forward pass\n",
    "  x_l1 = x.dot(l1)\n",
    "  x_relu = np.maximum(x_l1, 0)\n",
    "  x_l2 = x_relu.dot(l2)\n",
    "  x_lsm = x_l2 - logsumexp(x_l2).reshape((-1, 1))\n",
    "  x_loss = (-out * x_lsm).mean(axis=1)\n",
    "\n",
    "  # training in numpy (super hard!)\n",
    "  # backward pass\n",
    "\n",
    "  # will involve x_lsm, x_l2, out, d_out and produce dx_sm\n",
    "  d_out = -out / len(y)\n",
    "\n",
    "  # derivative of logsoftmax\n",
    "  # https://github.com/torch/nn/blob/master/lib/THNN/generic/LogSoftMax.c\n",
    "  dx_lsm = d_out - np.exp(x_lsm)*d_out.sum(axis=1).reshape((-1, 1))\n",
    "\n",
    "  # derivative of l2\n",
    "  d_l2 = x_relu.T.dot(dx_lsm)\n",
    "  dx_relu = dx_lsm.dot(l2.T)\n",
    "\n",
    "  # derivative of relu\n",
    "  dx_l1 = (x_relu > 0).astype(np.float32) * dx_relu\n",
    "\n",
    "  # derivative of l1\n",
    "  d_l1 = x.T.dot(dx_l1)\n",
    "  \n",
    "  return x_loss, x_l2, d_l1, d_l2\n",
    "\n",
    "samp = [0,1,2,3]\n",
    "x_loss, x_l2, d_l1, d_l2 = forward_backward(X_test[samp].reshape((-1, 28*28)), Y_test[samp])\n",
    "\n",
    "imshow(d_l1.T)\n",
    "figure()\n",
    "imshow(d_l2.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loss 0.01 accuracy 0.98: 100%|██████████| 1000/1000 [00:02<00:00, 481.98it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x13165cd00>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1lElEQVR4nO3dd3wUZf7A8c+TSho99BJ6E2kRQVFQUdFT9E5PxXrWUw/19E7P3s+z3Hnq/aznWdA7xS4iiqKIDZAAinRDDzUECOnJZp/fH89MdnZ3NtlAwmY33/frta/dmXl25pmd5DvPPGVGaa0RQggR/eIinQEhhBANQwK6EELECAnoQggRIySgCyFEjJCALoQQMSIhUhtu3769zsrKitTmhRAiKi1evHi31jrTbVnEAnpWVhY5OTmR2rwQQkQlpdSmUMukykUIIWKEBHQhhIgREtCFECJGSEAXQogYIQFdCCFihAR0IYSIERLQhRAiRkhAF0KIGCEBXQghYoQEdCGEiBES0IUQIkZIQBdCiBghAV0IIWJEnQFdKfWSUmqXUmp5iOVKKfWUUipXKbVMKTWy4bMphBCiLuGU0F8BJtWy/BSgn/W6Cnj24LMlhBCivuoM6Frrr4E9tSQ5A5imjQVAa6VU54bKoBCiidmdG37a8kIo2tl4eYm0/duhoijSuajREHXoXYEtjuk8a14QpdRVSqkcpVROfn5+A2xaCHFI5c6B/xsFP78TXvqnRsI/+jduniLp8YHw3DGRzkWNQ9ooqrV+QWudrbXOzsx0fYKSaE4+nApLpkVm27NugflPh15esA5eOQ3K9jbO9mfeBPOfqX37LxwHpdbF8a7V8J+ToXx//bf17T/hi/v95xXthOePhcK88NaxfRm8eCK8fpY1/ZN7umVvwcwbfdOlu817SQE8Nw6WvwfTzoSqcshfa9ZZXhi8nnevhBXvu2/j5VPh3lb+x6aiGF6aBDtX+KetroL//hY2zYc9680+lxSEtcsA/PI5/O9c0Dp0mr0bgueV7jH7tmqmyW9lSfjbPAgN8Qi6rUB3x3Q3a17ztn4edBoKqW2Dl1UUw+YF0G9i7evY9D206wvpHXzzdq6A+CRo3+/g81i4FfZvg+5HHPy6DsTS18xr5MWwehb0nQgJSf5pcudAt9HQomXtv9uGr6HjYe6/t5sfnjfvY/9g3ld/DH1P9G3/k7/Axm9g47cw6PT675vWsGYWJKVD+/7QMqAWMuc/5n3ImdCyS/D3v34Mti0x+Rp5EXxxH2xZAOu/gsGT65eXOfea9/YDoOtI2P0LfPu4CcrTL4IL3w39uy1/FzIHwrtXwO41vvnxib7PezbAloWwdYnvd514nzlmthXvwY6f4Z1LzfSiF2HzfMj7AdZ9CUN+bU5ihXlQVQY/v2VeQ37tn5+S3bDpO/N59SwYcYH5vOk7s77P7oKL3vOl374MfvnMbDspDQpyYeUH0Hk4pLUzee8yHFLaQGUprPrI/L+17QVLXoNv/m7WU14IKa19692x3OyTTWtQCtbNNX+LpQVm36Zb+dvyA7TrA/s2w9bF0OcE6HSY+29+EBoioM8Apiql3gSOBAq11tsbYL3Rq6oMpk02geiKz4OXf3wTLJsOUxdD+77u69AaXj4F2mTBDY7S0LNHmfd7XUo19fWvUeApa5h1HYz18+DNKTDuRph4r2/+/u2mRDjgVJjyBnx4Laz8EG5YBm16+tJ5KuHV06HLCLjqq/pvf9P38Ob5MHYqnPxXa9vbzHt88oHt07ovzDrBBIu/bHRP98xYuNXlEZFej7V9K3DGxZt3XX1g+QF4/6rgeduWwPQL4dJZ4PUC2retwjx45zJIbR9cTxznCOhPDQ9eb/5q6OYoKNjrtH12B/Qabz4ra9m/XDrIVXsgPsH3eddK3zLt9X22f684K63X+p0KfjHvRY6QVFUKLx7vm+5zgjkJfH6XOdG4KdruH9D/c6JZj23lh+bk/NqZ7t9XCv59PJRYVc0tWkUmoCul3gAmAO2VUnnAPUAigNb6OWAWcCqQC5QClzZ4LqNNdaV53+na09OUCsC6HA0R0D0V5n3vxobMWcA2yvync7+A138DN60ypcb72sCxN8Nxt5vly98zJaxbNoRfEg7Hlh/Mux1EbZXF5j1/tXnfaf0z27/Nf04yVyvn/ddMb1ta/21/+SBkjbO+/6MJbkU7AOsSu8Kq4ri/nSnNn3i/21qMZ8aagDPyYv+qpLK9porATfk+UyUy7kb/+XaAev/35t0OVF89AjNuMHle87EvfXwyVFdA6x6mpHnc7abgcOeuOn4ATOnWzl9CC7h4Brx0km+5XW3iNO9hSGsfukpq/1aY9qBvWsUHp7FPEm9dFDpvD7QLvWzGVPjqb2ZbNduJC/1b2z6703963Rd1f2fh87D4Zfjdx/DKr4KXv30JfNIp9Pfn3OsL5gApDfj/41BnQNdaT6ljuQb+0GA5aipWfACtu0PXUfX/brX1z2iXIEr3mH/wo28wJaI8K4A5z/CeSvjuSThqKiSm+IIZmEDb94QD2g0Wvwq5n8NhZ5vLzWP/7J5u5Yfw1sXm866VkN7R5H/eIzD+VoiLgx9e8C3fs8GUwDoMNPM2fmuqRAY4erh6Kk2wGno2rJoBrbqb4NH/FN+lOfhKUYkp1vcqzG9hB1oVb65Y7HRKmfctC837zJt861r4PBz5e5O/hc/BEVZd7LgbfSU9+4QApmqj59Hm86ZvffMzrf3auQIW/ccE2O+ehIQUmHCruaxe/xV0HAItu0JVia/0WN92gTn3wi9zoHgHHHsLDDsX1s72LbeDOkD+KvPuDOZggjmYS3owwRyCT5J18ZT7B/PazHL5WxryG1MVsWk+bJjnm//xTcFp7ZPlwdgfULu79pODX6ebxS+bd7dgbiveEXpZYGEjpc3B58lFQ1S5xKa3LzHvdnVE+X5ISDYvp2qPL/gmpZugYZfQ7YA+4zpYPRO6HwkvOwJelaOEvORVmPugCRzH3eb/x/76b0JXi1RXmT/qjC6mxF1eCMktfZfqH11v3ld9ZN4PPwcyOvvXgYIvmIMJxHZgAFOX3Hu8WS/ArlXmnzkuAe4uMAHS/kO/e68J/mCC+FcPmZdTzkv+03Ypr7LUvL58ABY8A217+5Y7/yGKdvi3ISx39Lj45Bbodaxp8Fz6mgnqYC6X+xxvSvSBjXCf30WQUqvh7NvH/efPe9jU6751sSldNxT7ZPL+VTDoNP+T/cFYM6th1hOOkx+CMdeabTpP2OC74nAqWHfg2xp9la+AEY0koEfYw92hx1i47FP/+R9cYxpvAIZNgV8/FxzQ7eDgrPMD/5ZvT7l5ty9Dw+3b+sktJkD2nWhKY3YpMSEluIER4ImhMPS3cJajrtBZYgVYP9fUadumTYbbt0Oqdflrl8zsf9IXHY2U3z7uuwqwf4e6lO0z73ZDmG3PevNeuhv+fZxv/qunwR21lIaeGRM8z600advxc/C8iuLgeTXrPzL0MqdWPaBwc93pAv2te91pwjX7dv/p27fBQy6NsA0hKd1cPWV0du/5ESSg50haJtyca3r3bFsSnLxrNmzNMZ/bR7grZFyC+0nKZleBhZJaS1XSQZB7udTFWUe4eX7wcmcA+ukN815dZd7tAG5/74Or/b/rLCmqOP/v1BXQnzsG7m/vK+3mzvFvMLJL625WzvCftrui2dbNDf7Oq6fBT/8Lnv/e72HHMt/0lw+Y7nVvXgBf/rX2fbAdSEn3n0Pq/536CGxfOBCtXIdj1M1u+Bx2PmQOOvh82MGvVXfT06OxxFsFiJYHuN8trHrsS2a4N247G1lHXgyjAprrWnar3/bOfBZa96w7nZtb1pteVQDZl8GlnwanGVRLb6TAXk8NJPYDutamF0Nt/Uhrs7Ael3WJ1j+Ls4S+4Rvf8n0BpTW3gO4pg80LTf9Xp+oq//rQHcvAWxV+3pxatIJlb/umN37jv9yuq3bauth9XcveDJ63ZYGpYtofZh/n3WvDS+dkX/U0Zcf8CUZe4h+UR1wIvwuzGqRbtn/bSf/a7sDhosMQGHgaXDITRv8efjezft//1ePBXTbTO4ZOb1fjZV9q2iUSUmDwGeFvb9LD5j05w3QrDGwoTmlt0pz7uqn6tNtYbB0H16+Lac+jTXtI/0lw+Zzg5RNug6QMcyK58F3f/PG3Wr1UDjfTHQZDz7GmN5BT6x7u271qnvv8BhD7AX3p66b738oPDuz7tV1WBQ6BTko1786qhldPC/19t4C++BXTKPXdE/5pv3wQHm+A0hpAyS5474qGWVdTcv7bdaepj3EuDXn1kd4BJj/lG0dw4v1wxtOQdbSvNFubuARfV7kRF8Kpf/ctS7AakAef6ZuXHNBT49rvTQ+gjI5w6qOmCyyY+tsMRwkxIcUEfPCVkgFGXARHXmPti9WDY+TF0HtC6PyCaQS/dBbcuQNatA5Ol+UYWTniQt/nfif6Pivl34UVTBvImGt8QbujdZVml4RPetA0/geyG71tR11n3jM6w/Dz4fzpZixG4NVQ7wlwex5cMcdUabbJMtVox91mltsnMLuR3u5tZc/rczyuugx3n98AYr8O3a7Le/t3sOYT+I2jxL1rtakPvW6J6fQPpuriDUf9sa72L93/a5TpLdL9SNMI5FSSD/8YBGcHNPqF8v1TpmdHwTr/hj03gQG+sXQeFnoU4MG4xToOj/Zq+HXb0sKsl7x9uynheSrgIUdgu2GZ6dVitx9MuDW4UbQ+7L7anYaaHh92EAZTh+rsG+3GUwGJViFBY3pd3b7dqpJRJmgkploNqNa0pwIeqaMa4c/WFdgDVonyti2mNxKYoDjzJnP1l5Dkq/7pcxyc9oT53bzVsPNneGGCGYyV0dEUnJwDiWyB7Sh37ITinfCkVbodf6v5bih35pur1rhEX4HJ1mGQ+T0SU8xvkJQGmQOg33b/43rJRyYff7VOShPvhwm3B7cxXf0toH2/i12lYpua4x8LktLNe6XVgN1jjBlv8EiWqXbqe4LJ37QzfD3bGlnsB3RnSWjZdHN27jTUTP9onVFXfgjHWKWxuQ/5/xFu+cF0UbMVWKXyLQtNn+VARdv8+5vWZd4j4ac9FDK6mICekGJKY4G9FQ7Euf+tX7/11j1hnzXY5oJ34b9n1Z7+uDvMe6dh7suP+ZO5AmqTZU5YdmBISoUrvrCqfJQZrOQMsgnJpt8xWCMN003fZ9vk/zNBVMWZxnGnvhNNwAHzN5cYUP1w6Sx4aoT53HmY6dFz7M2+gWNHXGl+f/tv1G5bCQxq4F8vnpgSvDxQYA+n+ETTpTa5pam3zxrnO6n3HAeTHjEl6cQWVvoEUyVy+pOmSich2Uz3Po4gdmP/xHtNHX5iC1OlUpPfVLj4Q1O14SYhyb1x31ZzLNOC5wGc85oZ1BSXAtcuMOM64uLcf0e7W+s500wbVnJ6wPKA381e7uxinNIGfvNv6HmULy/nvma6Hn94bej9aCDNIKAHHITnxpkugN5q32gy5yi2wLr2jd8E1zHbQpWwihvw7nJpmXWfILKO8eVx9O8PLgjb1QMprWHcH0Ovq10/97p2gH4nm8EadnXVIEe107G3wNePmtJPqIFXWcfAj1ZAdxvm32GIqbNc9KI5YR99Q3B3Uqfsy0M3UHbLNi9b5gDzfoF1xWTX09rvdkBPaWMuqe31fvuE/9D4o67zXYpndILjAwaztO1tguHqmaau2B7ifvZL8P41cNIDJvjVFEjq0QY08mLYt6XudGOn+gorCckw5mpf3uwuo3FxvvlOSsGo3/mmR1/pvg17HMCwKeZ3AN9VB5h9DFWFczAGnGqCvPM2CR0G+U6ytQm33t8uoQf2iDr8HP/pjE7mFgXzHjFVPI2oGQT0EGf3h3tCpdWTJM75M4T5j9OidejeGbV1kWsMZ/wfPGmVTt0ue+vDvjop2m5GDYIJPJMehiccl6C11QFfYPX8cRt9d/wd5gXm5Hlf6+A0pbvNSLqyEHdt7joSfvUP8wrllEdNl07wLxHWJaVNeLdCCDWU3766SA7jOHQebgK6c9TgYWeZl61myH89Avrkf4WX7uQweyEdjKyjg39P58k3IYwrigMx5Y3GWa9TTe+hMHv1/HFZ3WkOUuwG9FUzTT9vtyHH4Avm4J8msK94KB2H+G4SFMopj8Kc+8wowgPl9o981HXwveOf1g68YErHXz8W3rp7Hu3bh3OmmWDkHBad2hYu/9zsq3MQFARf+YD5/iUfhbdtMKW8k/5q7uvhtG8z/OEH38i7q+aZtpD2A8xJtEstD8Wyr1bsqy/wlaQawh8WuVdr2KXx0580g8I6D697XcfcZBrjeo+vJZG13vqU0Js6+7cC3yC0aDRgkrlNQmBvmwiK4l+zDtMvMKPuwhncYpeC8teEf+8U+5K0NoPPrF8wP+VRUxq2nfaEe0nj+Lt8vRLAv7TceRhcvxT6nWR6PfQcZy6NL54RtBp++4rvc4vWpvU9sFdP99Hm0tV50gjcpi0x1f+mWeHoG1ClkpZpgmJ6pq+to8twUyXRcbCpm0xsEbSaGvZ3dLWpm82+rGGDRmZ/0zgZ6KwXzfHOOsZcsoezzbj4uqsb7OB3oN1uRePqPT74xmMRFLsB3RY4CtKNijO38Hx6dOjBOIHaWr01MmoZdZfRse6+w2c47sndvp/p+jTMqmcbcZEZHed0wj3mkvXUR33znME2IcmcbC54G27bDJd+bAKkWykwvYPvZlOtrEEZbg1bEFwqdQvovVxu9O/sHufGefndspsZKdh9dO3fqY3dGNXpcBMsT/vnga+rPjoNhXNe9TWsNRg7oId55RhNQvXTFgcsdqtcbHYru5N9t0Ob242D6pLSBv6ca7ojvenS0HHFl+b9t6+aEZzOoevDL4SNX5uqBXtwAvgC8+SnTKOYW3BwC6R2UDyQqoWjrofDz/U1WB375+B7r4Aphdy8zvQI+vim4CqXP+f6314U4C+b3KtmnOwSaJssuMZlJG59DTodblrdaCPxDjn7asStV0Y0+8vGA781sQgp9gP6L58Fz5t9R/C8+kpoYaoF7ADb6XATTOZaDU121UNiC/8HVIApiY+70fR5t6sIwPcHHp9obk3qxhkgL3zX1BXHxZtBJ71qq4sFrv4OnrMGWdhXAUr5gjmYdZ37untDYlp730knLuBPJ93lCVSBAd5N657mYQhDz264oBUrwRxMFdz4W82AmljSSDenau5is8rFvpcKuHeNC7z1aCi13Tq3jVXlYl8BtO4B42/xLXdWgwTWfyZnmAdbjLnaBFT7fhK1db2zOQNp34nQ/2TzefSVpn63Np0O842mnHBr6HSDTg9dt9tzrHk/4nLfvH4n177d2ihlukfaVT7CX1y8GZkYzslRNHuxWUIP9y5/dTn/bXjM0fjZfoDpa/z7r03jI5hnI0JwMHbWOQdWkwSVfq2AH9jw6MbZQ+BA9D/p4J5Q1La37/uRftKREMJPjJbQGyigp7WDE+72Tf/2ZVMdYQdz8JXQA/vTOlu+Mzqak4M9Gi4woNsl+Lrqm4UQohaxF9CL883DjxvKQMfd21p2Db6b24BToG0fM1oR4MznzP0tAvU/CU5/wtwAqMNg/2V2D4awuj8dZAldCBGzYqvKJX+N6XoYyr2F8MX98E0tIwwDORvq3Eb/pbWH6x034x8+xbzcDD3bvAL1Gm/uNd6Y96oWQsS82CmhL3mt9mBuC+xP/RuXp3z/aa3pogf+/cwba1Tb6U/A1MXhtfwfbB26ECJmxU4J3XkXPDenPWHeA+u6e4yBtA7mHuFgHj6Q4biJf1ycudF/mku3vIaSkGx6vQghxEGInYBem/YDzFNUwPcgCVt6R/PA2bkPwm157v2vnV30hBCiiYqNKhdviGHR2ZeZd2dVhn2HxGNvNjfPT0gyoyPvzK/fXfkOpbsKYPgFkc6FEKKJi42AHqqbov2gZecTtkutW7KmtPXdOF+p2m+iH2nxCY66c6lDF0K4i+2A3jXb3Bp33B9980ZcaObV52GyTYF9tRHqOYVCiGYvNurQQwX0tPZwT8BDEroMD54XDbqOkpGZQohaxXYJ3fmQAyGEiHGxEdDd7nken+z/LEshhIhxYQV0pdQkpdQapVSuUiroNn1KqR5KqblKqaVKqWVKqVMbPqu1cN5d0Xb+mzLyUgjRrNQZ0JVS8cDTwCnAYGCKUirgZiTcCbyltR4BnAc809AZrVW1Swk9Tm50JYRoXsIpoY8GcrXW67XWlcCbwBkBaTRg3+ikFbCt4bIYBrc69MAHMAghRIwLJ6B3BbY4pvOseU73AhcqpfKAWcB1bitSSl2llMpRSuXk5+cfQHZDcKtykYAuhGhmGqpRdArwita6G3Aq8JpSgWPsQWv9gtY6W2udnZnZgPdGcWsUbUJP4hZCiEMhnIC+FejumO5mzXO6HHgLQGs9H2gBhHgoZiNwK6HLwyKEEM1MOAF9EdBPKdVLKZWEafScEZBmM3ACgFJqECagN2CdSh1cG0WlykUI0bzUGdC11h5gKjAbWIXpzbJCKXW/UmqylexPwJVKqZ+AN4DfaR34ZORGJI2iQggR3tB/rfUsTGOnc97djs8rgaMbNmth+v5fsObT4PlShy6EaGaif6ToZ3fCpm/N58PP882XfuhCiGYm+gO604n3+z5LlYsQopmJrYDuvKe5BHQhRDMTWwE93hnQpQ5dCNG8xFhAT3Z8ljp0IUTzElsB3VkqlyoXIUQzE90BPfDh0MrxvE0J6EKIZia6A7rbCFGbBHQhRDMT3QHdUx56mbO0LoQQzUCUB3SXEnrwTR6FEKJZiO56CbcS+rULIC/n0OdFCCEiLMoDustNuTIHmJcQQjQz0V0/UVsduhBCNDNRHtBr6eUihBDNTJQHdCmhCyGELcoDuqOE3qZX5PIhhBBNQJQ3ilol9AvegR5jIpsXIYSIsCgvoVsBvXUPSM6IbF6EECLCojug288STUiuPZ0QQjQD0R3QK0vMe0KLyOZDCCGagOgO6LtWQnJLSOsQ6ZwIIUTERXdA37sR2vWFuOjeDSGEaAjRHQmrq6T+XAghLNEb0It3wcZvwFsd6ZwIIUSTEL0Bffbt5j3vh8jmQwghmojoDeheT6RzIIQQTUr0BnQhhBB+wgroSqlJSqk1SqlcpdStIdKco5RaqZRaoZT6X8Nm04XWjb4JIYSIJnXey0UpFQ88DZwI5AGLlFIztNYrHWn6AbcBR2ut9yqlDmHHcHl2qBBCQHgl9NFArtZ6vda6EngTOCMgzZXA01rrvQBa610Nm80AW5fAyg/M5/jERt2UEEJEi3ACeldgi2M6z5rn1B/or5T6Tim1QCk1yW1FSqmrlFI5Sqmc/Pz8A8sxwPqvfJ/jJKALIQQ0XKNoAtAPmABMAf6tlGodmEhr/YLWOltrnZ2ZmXngW4tLcP8shBDNWDgBfSvQ3THdzZrnlAfM0FpXaa03AGsxAb5xSDWLEEIECSegLwL6KaV6KaWSgPOAGQFpPsCUzlFKtcdUwaxvuGwG8CuVS28XIYSAMAK61toDTAVmA6uAt7TWK5RS9yulJlvJZgMFSqmVwFzgZq11QWNlmrh4ZwYbbTNCCBFNwqqA1lrPAmYFzLvb8VkDN1mvxicldCGECBKdI0WdAV1K6EIIAURtQHc2ikpAF0IIiNqA7qhDP/2pyOVDCCGakCgN6I4ql8N/G7l8CCFEExKdAV0IIUSQ6Azo2hvpHAghRJMT3QG9TVZEsyGEEE1JdAf0896IbD6EEKIJidKAbnVVjE+KbD6EEKIJidKAbpXQlTzcQgghbBLQhRAiRkRnQP/JemSpis7sCyFEY4jOiLjha/MuAV0IIWpEd0SUgC6EEDWiOyJKQBdCiBrRHREloAshRI3ojogS0IUQokZ0R0QJ6EIIUSO6I6IEdCGEqBHdEVEGFgkhRI3oDuhIQBdCCFt0B3SpchFCiBrRHREloAshRI3ojogS0IUQokZ0R0QJ6EIIUSO6I6IEdCGEqBHdEVECuhBC1AgrIiqlJiml1iilcpVSt9aS7iyllFZKZTdcFmvLmAR0IYSw1RkRlVLxwNPAKcBgYIpSarBLugzgBmBhQ2eylswdsk0JIURTF04RdzSQq7Ver7WuBN4EznBJ9wDwCFDegPmrnQR0IYSoEU5A7wpscUznWfNqKKVGAt211h/XtiKl1FVKqRylVE5+fn69MyuEECK0g66EVkrFAY8Df6orrdb6Ba11ttY6OzMz82A3LYQQwiGcgL4V6O6Y7mbNs2UAhwFfKaU2AmOAGYesYVQIIQQQXkBfBPRTSvVSSiUB5wEz7IVa60KtdXutdZbWOgtYAEzWWuc0So6FEEK4qjOga609wFRgNrAKeEtrvUIpdb9SanJjZ1AIIUR4EsJJpLWeBcwKmHd3iLQTDj5bYeg+5pBsRgghokX0jczR2rz3OS6y+RBCiCYmCgO61/ogfdCFEMIpegO6DPsXQgg/0RcV7SoXGSUqhBB+ojCg2yV0CehCCOEUfQEdu4QehVkXQohGFH1RUerQhRDCVfRFRenlIoQQrqIwoEuVixBCuIm+qCiNokII4Sr6Aro0igohhKvoi4pS5SKEEK6iLypKo6gQQriKwoAuI0WFEMJNFAZ06YcuhBBuoi8qSi8XIYRwFX0BXXq5CCGEq+iLitIoKoQQrqIwoEsJXQgh3ERfVJRGUSGEcBV9UVEaRYUQwlXUBfSNu4sB0FKHLoQQfqIuoM9ftxuAiuoIZ0QIIZqYqAvoGckmy2Uebx0phRCieYm6gJ6eZKpayjwRzogQQjQxURfQMxJMXUtpdXyEcyKEEE1L1AX09HgT0EuqEyOcEyGEaFrCCuhKqUlKqTVKqVyl1K0uy29SSq1USi1TSn2hlOrZ8Fk10uNNXUuRlNCFEMJPnQFdKRUPPA2cAgwGpiilBgckWwpka60PB94BHm3ojNpSrRJ6kUcCuhBCOIVTQh8N5Gqt12utK4E3gTOcCbTWc7XWpdbkAqBbw2bTJzXOKqF7EhprE0IIEZXCCehdgS2O6TxrXiiXA5+4LVBKXaWUylFK5eTn54efS4ckXQnA/qqoq/4XQohG1aBRUSl1IZANPOa2XGv9gtY6W2udnZmZeWDbqK4AYJ8EdCGE8BNOvcVWoLtjups1z49SaiJwBzBea13RMNlz4SkHYE+FBHQhhHAKJyouAvoppXoppZKA84AZzgRKqRHA88BkrfWuhs+mg8ecKzburWrUzQghRLSpM6BrrT3AVGA2sAp4S2u9Qil1v1JqspXsMSAdeFsp9aNSakaI1R28xBT2JnZi7R4J6EII4RRWVxGt9SxgVsC8ux2fJzZwvkLLvoyX9xxNwdxcvF5NXJzcdVEIISAKR4oCpCYnoDWUe+SWi0IIYYvKgJ6WZAYVlVZKQBdCCFtUBvTUJFNTVCo3RRdCiBpRGtBNCb2kUu6hK4QQtqgM6MmJJttPzvklwjkRQoimIyoDer8OGQB8umJHhHMihBBNR1QG9O5tU7l2Qh/i4xTVXh3p7AghRJMQlQEdoHPrFKq9mi9XN+7AVCGEiBZRG9DH9m4LQM6mPRHOiRBCNA1RG9D7dsigU8sW7CmujHRWhBCiSYjagA7QPiOJ3cWNd2NHIYSIJlEd0Hu0TWXRxr2UyYhRIYSI7oB+8pBOFFd4uPW9ZcyVxlEhRDMX1QG9R9tUAD78cRuXvrIowrkRQojIiuqA3q1NatC81Tv28/qCTRHIjRBCRFZY90NvqtqnJwXNO+2pb/F4Necd0Z2E+Kg+XwkhRL1EdcRTyv/hFuVV1XiskaO7pTujEKKZieqADvC/K47ktMM7A1BU7rv74s795ZHKkhBCRETUB/Sj+rbnhEEdAJjtuFnX9sKySGVJCCEiIuoDOkCrlEQA7vxgec28q19fwrr8YvKLKiipaNz7pm/bJycPIUTkxURAP6pPe9f5y/L2ccRf53D5q74ujV6vZvGmvQ227cWb9nDUw1/y/tK8BlunEEIciJgI6C0S4zmmX3BQv3H6TwAsWO+7gdd/F27irGe/564PltP/zk+Y+r8lNcuG3jObuz9czvbCMryO2/KWV1VTVe113fbancUALFwvNwkTQkRWTAR0gH9NGcGwbq1CLn/k09Ws3rGfuz5cAcBrCzZR6fEyc9l2rn9jKSf9cx5FFR6mzd/E2L99yb0fraCguIJNBSUMvOtTLn81h6pqL9Vezd4SXw+aOKujTUPel31TQYlfX/p7PlzOE3PW8vTcXDwhTixCCBEzAb11ahJPTRkBwFNTRtAuzb+P+rNfrWPSE9+4fnfGT9tqStq2afM3cfw/5jH+sa8A+HptPv3u+ITz/72AEQ98TnlVNT9t2cdPeYUAVGsT0FdsK+Sjn7axdLOvWsfr1Tw4cyWrtu/nu9zdaO0e/G+a/iPvLs7jghcXcucHyym26v5fnb+JJ+b8wmOz1zDjp231/GWgwlNdc8Wxq6icuz9cTnlV6PvfLN60h8KyKtdlmwpKyLr1Y75em89lryzi+9zd9c6PEKJxqFDBpbFlZ2frnJycBl9vpcdLUkIcu/aX89WafG55d1lQmsyMZM4f3YMnvzjwZ5LecEI/v++3Skmkb4d0v/r5D/9wNIM6t2Tn/nKOeXRuzfxnLhhJUXkV368rYPGmvZwwsAPz1uazsaDUbxtf33wc3dum0Ou2WTXzrjymF51apZDdsw2dW7Vgxfb99GmfTmpyPO8v2coVx/SiqMLDu4vzyGiRyKINe5ies4ULx/TgntOHcMObS5n18w76d0znsxvHB+3Xxt0lTPj7V2T3bMM71xxVM99T7aXvHZ+Q1S6VjQWljO3djvnrCwAYndWWlimJnD6sM0/PzeXJ80YwqHNLSio8aMxVTGpS6DFs6/OLSYiLo0e7VBasL2BQp5a0Sk2sWb6npJINu0sY1bNNGEfGp9qrKa+qJi25ccfPfbFqJ307pNOzXRoAWuugMRIHKm9vKd/l7ubcI3o0yPpE9FNKLdZaZ7sui7WAHujvs9fwf3Nz6dshnTG927J40z7euXosackJ3P/RSl76bgP9OqQzYUAm7y/d1qRux3tY15acObwrD368KuzvXDOhD89+tc512RFZbVi00XfCefK84aQlJXDFtBzevWYs+UUVrMsv4bHZa4hT8MMdE1m0YQ+V1V7u/GC5Xz//AR0zWLOzyHU74/tn8rujsvzurzNldHdm/LiNH+85iX2lVVR7NfPW7mLysK4MuvtTAC4a05PXFmxi0pBO/PPc4WzdV0ZGiwQmPj6PonIPH00dR0pSHMu37qdnu1R6t0+noKSC+DhFu/Rk0pMT+GLVTi5/NYdZ1x/DtPkbeXPRFh448zCO6duerPZpdf5+3+fuZkjXVrRKScTr1azfXUynVimkO04KhWVV7C6uIC0pgTU7i7jkpR9ISohjzo3j+fjn7Tzy6WqePG84k4d1qQnsgUHeU+1l+bb9LFxfwBXH9MarNYkuI5unvLCA+esLmHfzBHq2S6O00kNKYnzIE8au/eWs311Cvw7ptEtPrnN/AxUUV9AmNYm4uIM7IW3bV0ZCnKJDyxauyz/8cSvxcYrTDu8Sch3T5m9kfP/MmhNlU+P1aj5ZvoMTB3ckKeHQVXY064C+p6SSh2at4q7TBtd0b7R9sHQrf5z+Iy9cNIqThnQC4Lo3lvKRo1qjQ0Yy7dKTWbV9f9C6B3Vu6TpfhM8u8QeKU1DfZole7dPYsLsk5PLLju7F6F5t6NQqhYwWCSxcv4eHZq3i9GGdydtbxsVjs7hyWg7DurfmymN6sSG/hH98vrbmu51aJROnVNgn2BlTj2bbvnLKq6r54/QfAfj4+nEM7tySI/46p2Y08+istvywcQ/TrxrD4d1aA/D5qp2cclgnLvj3Qn7YuIdTh3YiOSGe95du5U8n9mfy8C70bJdGeVU1D81axe7iCtKTE3grx/S2GtGjNa9ffiRPffkL1x3fj8R4RXJCPAAPzlxJUkIcN57Yn2V5+wAY1bMthWVVDLvvMy49Oot7Th8CmBPRxz9vZ+KgjizfWsj36wo4a1Q3du0vx+PV/JxXyMDOGUE9zbJu/RiAeTdP4MGPV9E+PYnE+Dju/NVgdhWVM+4Rc8X66FmHc8u7y3jo10M5/0jfVciekkpGPvA5ADed2J/rT+gHQFF5FTv3l9PXelD8z3mFFFd4+GDpVh448zCSEuIor6pm+qIttElL4tTDOqGB0spqnp+3jgkDOjC6V1sqPNU1v8ePW/YxpEtLpv5vCevyS7jzV4PYvKeUi8b0dD1xVlV7SYhTvPTdRh6YuZJHzhpKpcfLjv3lXDGuN3FK0TIlIei7v38thxMGduScI7qH9fcTSrMO6LXRWrNi234O6+prTC2p8PDF6l1c/8ZSAN65eizZWW3Zuq+MllYQ6NY2hdYpSbRMSeChWat4fcFmjh/YgUuPzmJfaRXHD+zAkHtmA9AiMY7yKl9D5mmHd2bmsu0A3Dd5CCcP6URJpYdKj5dTnvyGK8b1oqSymq/W7GJ7oW+067vXHEWb1ESO/8c8AFIS4ymrpR68Nm1SE9lb6l5HLhpfn8w01uWHPvHUR2pSPKUhngfQOzON9QHbmTK6B2/8sDko7a+Gdubjn7cHzb96fB+em+d+xec0vn8mvzq8M7m7itmwu4TPV+50zcOQLi3pk5nu2hZ0wwn96NkulUUb97BqexE/btlXs+y8I7rTvW0qj81eA8AHfziaafM38t6SrTVpLjiyB4s37WX1Dv8rx86tWtCpVQuWbjbrOye7Gx/9tJ1nLhhJy5REznr2e0b2aM2Szfv8vjdhQCbXTujLmz9spndmGp+v3MnK7fupqtZMGtIJpeCT5TtwM2V0dy4ak8XiTXvYX+7hP99uYI/VmeLBMw/jwjE96/xNQznogK6UmgQ8CcQDL2qtHw5YngxMA0YBBcC5WuuNta2zKQT02jw9N5fM9OQ6z6Zer6ba5XJ5+dZCerVPIy05geIKD9MXbeHsUd1olZLI8/PW8bdPVvPZjcfSv2OG63r3lVYy/H5TQrl2Qh9umTQQgBe/WU9GiwTOPaIH6/KL6do6hUc/XcOATumM6tmGvaVVrM8v5jcju1FWVc2sZdv5Nnc3X67eRWllNR9NHcfAzhnsLa0kv6iC+z5aydCurdixv5yPrRPNG1eO4f2leTWlPTC3WNhYUMrt7/9cM2/K6B78+aT+VHi8bCwo4dXvN3LJUVnc+f5y2qUn1VTvtEtLosDRM6hTyxbscNya4fiBHfhy9S5OGtyRr3/J9zsB2s47ojtvLtpS8/mEQR35y7vLav5JQklLiqekkR+AMqxbq5rG8abiYE74ovHl3DmR9gdQJQYHGdCVUvHAWuBEIA9YBEzRWq90pLkWOFxrfbVS6jzg11rrc2tbb1MP6I3JU+1le2E53dsG3/7XyevVB12XadtVVM7C9Xs4fVjoOsvAK5b/+/IXxvXLZHj31jXL568rYGDnllRVe2mTmlRr3aHWmrU7i+nSugXvLM5j8rAutEtPRmtNhcfL6f/6li17S1lw2wm8t2QrZ2d3IzEuDq/WeK2/y7Ofnc+fTx7AiYM7sq+0klXbixjbp13NNgqKK6iq1jz8ySr2llZx+rAunD2qG1XVXnZYv7HXqykq9xAXBxktEqmq9vKXd5bx+cqdPPSbobRNS+KSl37g/jMOY8PuYvpkprNzfwX/nLOWVy8bzdzVu+jfMYPDu7WiwuOlR9tUCsuq2LW/nCN7tyM+TrG7uIJ/fLaW847oTt8O6WzZW8rfZq1m3tp8WqUkMrRrKxZt3MPQrq3ICRjYNvO6cQzq3JIKTzUpifEUlFTy9NxcXv5uIz3apvLMBSP5+2dr+GpNPsO6tya7ZxsKy6pIjI+rKW13btWC5IQ4jh/YkZIKDxeN7cnanUX8vLWQl7/b6Hp87HaQo/q0I29vGZv3lJKRnMB9ZwzhprfMGI42qYlUezX7y4NHW7dJTeSxs4dxxbTg/+PAK4G//vow7njfN5L7inG9SEqI4xmrvWfioI7MWbUz5N9SqCuL2rRPT3ZtE+uTmcbATi1rrkhuOrE/+UUVvOboKty5VQu/K2RbRosEv7YkN8kJcZw8pJPfVYjzyhzg9lMHctWxfeq1P7baAjpa61pfwFhgtmP6NuC2gDSzgbHW5wRgN9bJItRr1KhRWjRvxeVVuspTHelsaK21rq72+k17qr36y9U7tdfrDfGN8KzbVaQLyyqD5u/cX6a37SvVlSH23+v16nW7ivzys3bHfr80ZZUe/cvOosCvBiksq9R7Syp0flG5XrShIGS6iqpqXVrh0VprXVRepXcWlunqaq/2VHt1UXmVzi8q19MXbdZPzVmrf87bV5O2rNKj95ZUaK21Xrtjv561bFvNPnywNM9vmwXFFfq/CzZpr9eryyo9evqizbqs0hOUl9xdRbq4vMpvXnW1V+8sLNNrduzXn63Y4XdsPNVe/er3G/S+ksqadPb8ZVv2hXUc9xRX6Iqqal1YVqnLKj3667W79HtLtuh3crbofaWVfn+r9t/Lt7/k6+9zd2uv16vLqzx6fX6xzi8q116vV3+2Yoden1+s7/lwuS6v8u3jnJU7an67AwHk6BBxNZwS+tnAJK31Fdb0RcCRWuupjjTLrTR51vQ6K83ugHVdBVwF0KNHj1GbNsmDKIQQoj5qK6Ef0oFFWusXtNbZWuvszMzMQ7lpIYSIeeEE9K2As2WwmzXPNY1SKgFohWkcFUIIcYiEE9AXAf2UUr2UUknAecCMgDQzgEusz2cDX+q66nKEEEI0qDrHRGutPUqpqZiGz3jgJa31CqXU/ZjK+RnAf4DXlFK5wB5M0BdCCHEIhXWTC631LGBWwLy7HZ/Lgd82bNaEEELUR8zcbVEIIZo7CehCCBEjJKALIUSMkIAuhBAxQgK6EELECAnoQggRIySgCyFEjJCALoQQMUICuhBCxAgJ6EIIESMkoAshRIyQgC6EEDEirIdEN8qGlcoHDvSRRe0xj7lrTmSfmwfZ5+bhYPa5p9ba9QlBEQvoB0MplRPqEUyxSva5eZB9bh4aa5+lykUIIWKEBHQhhIgR0RrQX4h0BiJA9rl5kH1uHhpln6OyDl0IIUSwaC2hCyGECCABXQghYkTUBXSl1CSl1BqlVK5S6tZI56ehKKW6K6XmKqVWKqVWKKVusOa3VUp9rpT6xXpvY81XSqmnrN9hmVJqZGT34MAopeKVUkuVUjOt6V5KqYXWfk1XSiVZ85Ot6VxreVZEM36AlFKtlVLvKKVWK6VWKaXGNoNjfKP1N71cKfWGUqpFLB5npdRLSqldSqnljnn1PrZKqUus9L8opS6pTx6iKqArpeKBp4FTgMHAFKXU4MjmqsF4gD9prQcDY4A/WPt2K/CF1rof8IU1DeY36Ge9rgKePfRZbhA3AKsc048A/9Ra9wX2Apdb8y8H9lrz/2mli0ZPAp9qrQcCwzD7HrPHWCnVFbgeyNZaHwbEA+cRm8f5FWBSwLx6HVulVFvgHuBIYDRwj30SCIvWOmpewFhgtmP6NuC2SOerkfb1Q+BEYA3Q2ZrXGVhjfX4emOJIX5MuWl5AN+uP/HhgJqAwo+cSAo83MBsYa31OsNKpSO9DPfe3FbAhMN8xfoy7AluAttZxmwmcHKvHGcgClh/osQWmAM875vulq+sVVSV0fH8ctjxrXkyxLjNHAAuBjlrr7daiHUBH63Ms/BZPALcAXmu6HbBPa+2xpp37VLO/1vJCK3006QXkAy9b1UwvKqXSiOFjrLXeCvwd2Axsxxy3xcT2cXaq77E9qGMebQE95iml0oF3gT9qrfc7l2lzyo6JfqZKqdOAXVrrxZHOyyGUAIwEntVajwBK8F2CA7F1jAGs6oIzMCezLkAawdUSzcKhOLbRFtC3At0d092seTFBKZWICeb/1Vq/Z83eqZTqbC3vDOyy5kf7b3E0MFkptRF4E1Pt8iTQWimVYKVx7lPN/lrLWwEFhzLDDSAPyNNaL7Sm38EE+Fg9xgATgQ1a63ytdRXwHubYx/JxdqrvsT2oYx5tAX0R0M9qIU/CNK7MiHCeGoRSSgH/AVZprR93LJoB2C3dl2Dq1u35F1ut5WOAQselXZOntb5Na91Na52FOY5faq0vAOYCZ1vJAvfX/h3OttJHVUlWa70D2KKUGmDNOgFYSYweY8tmYIxSKtX6G7f3OWaPc4D6HtvZwElKqTbW1c1J1rzwRLoR4QAaHU4F1gLrgDsinZ8G3K9xmMuxZcCP1utUTP3hF8AvwBygrZVeYXr8rAN+xvQiiPh+HOC+TwBmWp97Az8AucDbQLI1v4U1nWst7x3pfB/gvg4Hcqzj/AHQJtaPMXAfsBpYDrwGJMficQbewLQTVGGuxi4/kGMLXGbtfy5waX3yIEP/hRAiRkRblYsQQogQJKALIUSMkIAuhBAxQgK6EELECAnoQggRIySgCyFEjJCALoQQMeL/ATab9mVHeYoOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# numpy training\n",
    "def layer_init(m, h):\n",
    "  # gaussian is strong\n",
    "  #ret = np.random.randn(m,h)/np.sqrt(m*h)\n",
    "  # uniform is stronger\n",
    "  ret = np.random.uniform(-1., 1., size=(m,h))/np.sqrt(m*h)\n",
    "  return ret.astype(np.float32)\n",
    "\n",
    "# reinit\n",
    "np.random.seed(1337)\n",
    "l1 = layer_init(784, 128)\n",
    "l2 = layer_init(128, 10)\n",
    "\n",
    "lr = 0.001\n",
    "BS = 128\n",
    "losses, accuracies = [], []\n",
    "for i in (t := trange(1000)):\n",
    "  samp = np.random.randint(0, X_train.shape[0], size=(BS))\n",
    "  X = X_train[samp].reshape((-1, 28*28))\n",
    "  Y = Y_train[samp]\n",
    "  x_loss, x_l2, d_l1, d_l2 = forward_backward(X, Y)\n",
    "  \n",
    "  cat = np.argmax(x_l2, axis=1)\n",
    "  accuracy = (cat == Y).mean()\n",
    "  \n",
    "  # SGD\n",
    "  l1 = l1 - lr*d_l1\n",
    "  l2 = l2 - lr*d_l2\n",
    "  \n",
    "  loss = x_loss.mean()\n",
    "  losses.append(loss)\n",
    "  accuracies.append(accuracy)\n",
    "  t.set_description(\"loss %.2f accuracy %.2f\" % (loss, accuracy))\n",
    "  \n",
    "plt.ylim(-0.1, 1.1)\n",
    "plot(losses)\n",
    "plot(accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9635"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate on test set\n",
    "# over 96% with numpy, wow numpy is better than pytorch!\n",
    "# note: add 0.01% more accuracy with float16\n",
    "numpy_eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x131696f40>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAKoElEQVR4nO3dT4ic933H8fentiwTJQWraYXsmCYNpmAKVcqiFmJKipvY8UXOJUSHoIJBOcSQQA416aE+mtIk9FACSi2iltShkBjrYOqoImACxXhtVFu228o1CpEiSw0+xClUlp1vD/s4bOxd7Wr+PZN+3y8YZuZ5Zvf5avDbM/PswC9VhaT//35t7AEkLYaxS00Yu9SEsUtNGLvUxPWLPNgN2Vk3smuRh5Ra+V/+hzfqcjbaN1XsSe4G/ga4Dvi7qnroao+/kV38Ye6c5pCSruKpOrnpvonfxie5Dvhb4JPA7cDBJLdP+vskzdc0n9n3Ay9X1StV9QbwbeDAbMaSNGvTxH4L8KN1988N235JksNJVpOsXuHyFIeTNI25n42vqiNVtVJVKzvYOe/DSdrENLGfB25dd/8DwzZJS2ia2J8GbkvyoSQ3AJ8Bjs9mLEmzNvGf3qrqzST3A0+w9qe3o1X1wswmkzRTU/2dvaoeBx6f0SyS5sivy0pNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTRi71ISxS00Yu9SEsUtNGLvUhLFLTSx0yeaunvjxqdGOfdfN+0Y7NvT+ty8bX9mlJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdamKqL9UkOQu8DrwFvFlVK7MYStLszeIbdH9SVT+Zwe+RNEe+jZeamDb2Ar6X5Jkkhzd6QJLDSVaTrF7h8pSHkzSpad/G31FV55P8FnAiyb9X1ZPrH1BVR4AjAL+e3TXl8SRNaKpX9qo6P1xfAh4F9s9iKEmzN3HsSXYled/bt4FPAKdnNZik2Zrmbfwe4NEkb/+ef6yqf57JVJJmbuLYq+oV4PdnOIukOfJPb1ITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjWxZexJjia5lOT0um27k5xIcma4vmm+Y0qa1nZe2b8J3P2ObQ8AJ6vqNuDkcF/SEtsy9qp6EnjtHZsPAMeG28eAe2c7lqRZu37Cn9tTVReG268CezZ7YJLDwGGAG3nPhIeTNK2pT9BVVQF1lf1HqmqlqlZ2sHPaw0ma0KSxX0yyF2C4vjS7kSTNw6SxHwcODbcPAY/NZhxJ87KdP709Avwr8LtJziW5D3gI+HiSM8CfDvclLbEtT9BV1cFNdt0541kkzZHfoJOaMHapCWOXmjB2qQljl5qY9Ouy+hXxxI9PjT3CaMb8t991877Rjr0ZX9mlJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qYjvrsx9NcinJ6XXbHkxyPsmp4XLPfMeUNK3tvLJ/E7h7g+1fq6p9w+Xx2Y4lada2jL2qngReW8AskuZoms/s9yd5bnibf9NmD0pyOMlqktUrXJ7icJKmMWnsXwc+DOwDLgBf2eyBVXWkqlaqamUHOyc8nKRpTRR7VV2sqreq6ufAN4D9sx1L0qxNFHuSvevufgo4vdljJS2HLddnT/II8DHg/UnOAX8JfCzJPqCAs8Dn5jfir75lXKt7UVwjfXlsGXtVHdxg88NzmEXSHPkNOqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5owdqkJY5eaMHapCWOXmjB2qQljl5rYMvYktyb5fpIXk7yQ5AvD9t1JTiQ5M1zfNP9xJU1qO6/sbwJfqqrbgT8CPp/kduAB4GRV3QacHO5LWlJbxl5VF6rq2eH268BLwC3AAeDY8LBjwL1zmlHSDFx/LQ9O8kHgI8BTwJ6qujDsehXYs8nPHAYOA9zIeyYeVNJ0tn2CLsl7ge8AX6yqn67fV1UF1EY/V1VHqmqlqlZ2sHOqYSVNbluxJ9nBWujfqqrvDpsvJtk77N8LXJrPiJJmYTtn4wM8DLxUVV9dt+s4cGi4fQh4bPbjSZqV7Xxm/yjwWeD5JKeGbV8GHgL+Kcl9wA+BT89lQkkzsWXsVfUDIJvsvnO240iaF79BJzVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41YexSE8YuNWHsUhPGLjVh7FITxi41cU3LP0nX6q6b9409gga+sktNGLvUhLFLTRi71ISxS00Yu9SEsUtNbGd99luTfD/Ji0leSPKFYfuDSc4nOTVc7pn/uJImtZ0v1bwJfKmqnk3yPuCZJCeGfV+rqr+e33iSZmU767NfAC4Mt19P8hJwy7wHkzRb1/SZPckHgY8ATw2b7k/yXJKjSW7a5GcOJ1lNsnqFy9NNK2li2449yXuB7wBfrKqfAl8HPgzsY+2V/ysb/VxVHamqlapa2cHO6SeWNJFtxZ5kB2uhf6uqvgtQVRer6q2q+jnwDWD//MaUNK3tnI0P8DDwUlV9dd32vese9ing9OzHkzQr2zkb/1Hgs8DzSU4N274MHEyyDyjgLPC5OcwnaUa2czb+B0A22PX47MeRNC9+g05qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJoxdasLYpSaMXWrC2KUmjF1qwtilJlJViztY8t/AD9dtej/wk4UNcG2WdbZlnQucbVKznO23q+o3N9qx0NjfdfBktapWRhvgKpZ1tmWdC5xtUouazbfxUhPGLjUxduxHRj7+1SzrbMs6FzjbpBYy26if2SUtztiv7JIWxNilJkaJPcndSf4jyctJHhhjhs0kOZvk+WEZ6tWRZzma5FKS0+u27U5yIsmZ4XrDNfZGmm0plvG+yjLjoz53Yy9/vvDP7EmuA/4T+DhwDngaOFhVLy50kE0kOQusVNXoX8BI8sfAz4C/r6rfG7b9FfBaVT00/I/ypqr68yWZ7UHgZ2Mv4z2sVrR3/TLjwL3AnzHic3eVuT7NAp63MV7Z9wMvV9UrVfUG8G3gwAhzLL2qehJ47R2bDwDHhtvHWPuPZeE2mW0pVNWFqnp2uP068PYy46M+d1eZayHGiP0W4Efr7p9judZ7L+B7SZ5JcnjsYTawp6ouDLdfBfaMOcwGtlzGe5Hescz40jx3kyx/Pi1P0L3bHVX1B8Angc8Pb1eXUq19Blumv51uaxnvRdlgmfFfGPO5m3T582mNEft54NZ19z8wbFsKVXV+uL4EPMryLUV98e0VdIfrSyPP8wvLtIz3RsuMswTP3ZjLn48R+9PAbUk+lOQG4DPA8RHmeJcku4YTJyTZBXyC5VuK+jhwaLh9CHhsxFl+ybIs473ZMuOM/NyNvvx5VS38AtzD2hn5/wL+YowZNpnrd4B/Gy4vjD0b8Ahrb+uusHZu4z7gN4CTwBngX4DdSzTbPwDPA8+xFtbekWa7g7W36M8Bp4bLPWM/d1eZayHPm1+XlZrwBJ3UhLFLTRi71ISxS00Yu9SEsUtNGLvUxP8BWD9LxPoNV6cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# for fun, can it recognize 4?\n",
    "m = [[0,0,0,0,0,0,0],\n",
    "     [0,0,1,0,1,0,0],\n",
    "     [0,0,1,0,1,0,0],\n",
    "     [0,0,1,1,1,1,0],\n",
    "     [0,0,0,0,1,0,0],\n",
    "     [0,0,0,0,1,0,0],\n",
    "     [0,0,0,0,0,0,0]]\n",
    "# upscale to 28x28\n",
    "m = np.concatenate([np.concatenate([[x]*4 for x in y]*4) for y in m])\n",
    "imshow(m.reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# what is it?\n",
    "x = m.reshape(1, -1).dot(l1)\n",
    "x = np.maximum(x, 0)\n",
    "x = x.dot(l2)\n",
    "np.argmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}