4.prioritized dqn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd \n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Use Cuda</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Prioritized Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Prioritized Experience Replay: https://arxiv.org/abs/1511.05952</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaivePrioritizedBuffer(object):\n",
    "    def __init__(self, capacity, prob_alpha=0.6):\n",
    "        self.prob_alpha = prob_alpha\n",
    "        self.capacity   = capacity\n",
    "        self.buffer     = []\n",
    "        self.pos        = 0\n",
    "        self.priorities = np.zeros((capacity,), dtype=np.float32)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        assert state.ndim == next_state.ndim\n",
    "        state      = np.expand_dims(state, 0)\n",
    "        next_state = np.expand_dims(next_state, 0)\n",
    "        \n",
    "        max_prio = self.priorities.max() if self.buffer else 1.0\n",
    "        \n",
    "        if len(self.buffer) < self.capacity:\n",
    "            self.buffer.append((state, action, reward, next_state, done))\n",
    "        else:\n",
    "            self.buffer[self.pos] = (state, action, reward, next_state, done)\n",
    "        \n",
    "        self.priorities[self.pos] = max_prio\n",
    "        self.pos = (self.pos + 1) % self.capacity\n",
    "    \n",
    "    def sample(self, batch_size, beta=0.4):\n",
    "        if len(self.buffer) == self.capacity:\n",
    "            prios = self.priorities\n",
    "        else:\n",
    "            prios = self.priorities[:self.pos]\n",
    "        \n",
    "        probs  = prios ** self.prob_alpha\n",
    "        probs /= probs.sum()\n",
    "        \n",
    "        indices = np.random.choice(len(self.buffer), batch_size, p=probs)\n",
    "        samples = [self.buffer[idx] for idx in indices]\n",
    "        \n",
    "        total    = len(self.buffer)\n",
    "        weights  = (total * probs[indices]) ** (-beta)\n",
    "        weights /= weights.max()\n",
    "        weights  = np.array(weights, dtype=np.float32)\n",
    "        \n",
    "        batch       = zip(*samples)\n",
    "        states      = np.concatenate(batch[0])\n",
    "        actions     = batch[1]\n",
    "        rewards     = batch[2]\n",
    "        next_states = np.concatenate(batch[3])\n",
    "        dones       = batch[4]\n",
    "        \n",
    "        return states, actions, rewards, next_states, dones, indices, weights\n",
    "    \n",
    "    def update_priorities(self, batch_indices, batch_priorities):\n",
    "        for idx, prio in zip(batch_indices, batch_priorities):\n",
    "            self.priorities[idx] = prio\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "beta_start = 0.4\n",
    "beta_frames = 1000 \n",
    "beta_by_frame = lambda frame_idx: min(1.0, beta_start + frame_idx * (1.0 - beta_start) / beta_frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fdb0bda88d0>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFhdJREFUeJzt3X+Q3HV9x/HnK3e5JBcCScgF4yUhCY3W1KmIJ0XrKNWq\ngbFl7PhHcByU0cnQimPbaSvUTjud/lNr27EO1DRjU+wvsFWKqROLrbbacRQJFZEA0dsN5Afi7SWQ\nsJsfl7t794/9Xlg3ye0mt5fN9/N9PWZu2P3u927fnwCvfO7z/ez7q4jAzMzSMqfbBZiZWec53M3M\nEuRwNzNLkMPdzCxBDnczswQ53M3MEuRwNzNLkMPdzCxBDnczswT1duuNly1bFmvWrOnW25uZ5dIj\njzwyGhEDrc7rWrivWbOGnTt3duvtzcxySdIz7ZznZRkzswQ53M3MEuRwNzNLkMPdzCxBDnczswS1\nDHdJ2ySNSHr8LK9L0qclDUt6TNI1nS/TzMzORTsz93uAjdO8fgOwPvvaDHxm5mWZmdlMtNznHhHf\nlLRmmlNuAv4+6vfr+46kxZJWRMSPO1TjReNbw6M8VD7Y7TLMLOeG1izlza9o+TmkGenEh5gGgX0N\nz/dnx04Ld0mbqc/uWb16dQfe+sL6gwceZ89oDanblZhZnt32lqtyEe5ti4itwFaAoaGhXN2Ze2x8\nkr2HjnL7L/0Mv/POV3a7HDOzaXVit8wBYFXD85XZsaTsPVRjYjK4avnCbpdiZtZSJ8J9O3BLtmvm\nOuBwiuvtwyM1ANYtu6TLlZiZtdZyWUbSvcD1wDJJ+4E/AuYCRMQWYAdwIzAMHAVuna1iu6lUqQKw\nbsAzdzO7+LWzW+bmFq8H8OGOVXSRKldqXHHpPBbNn9vtUszMWvInVNtUqlS9JGNmueFwb0NEUK5U\nfTHVzHLD4d6G0eoYR46Pe+ZuZrnhcG9DObuYetVyh7uZ5YPDvQ2lytQ2SC/LmFk+ONzbUK5Umdc7\nh8HFC7pdiplZWxzubShVqqxdtpA5c9xUxszyweHehvJozevtZpYrDvcWToxPsO/QUa7yeruZ5YjD\nvYVnDh5lMrxTxszyxeHeQmkk2wY54HA3s/xwuLcw1TBsrZdlzCxHHO4tlCs1Vlw2n4XzLuh9TczM\nZsTh3kKpUnWbXzPLHYf7NOoNw2pebzez3HG4T6Py4glePDHutgNmljsO92lM9ZTxNkgzyxuH+zRe\nurWew93M8sXhPo1ypcaCuT2suHR+t0sxMzsnDvdpuGGYmeWVw30a5dGq19vNLJcc7mdx/OQE+58/\n5p0yZpZLDvezePpgjXDDMDPLKYf7WZRGsm2Q/nSqmeVQW+EuaaOk3ZKGJd1xhteXSPo3SY9J+q6k\nV3e+1AvLDcPMLM9ahrukHuBu4AZgA3CzpA1Np/0+8GhE/DxwC/BXnS70QitXqgwuXkB/nxuGmVn+\ntDNzvxYYjohyRIwB9wE3NZ2zAfg6QEQ8BayRdEVHK73ASpWaG4aZWW61E+6DwL6G5/uzY42+D/wa\ngKRrgSuBlc0/SNJmSTsl7axUKudX8QVQbxhWdcMwM8utTl1Q/VNgsaRHgY8A3wMmmk+KiK0RMRQR\nQwMDAx166877yZET1MYmPHM3s9xqZ0H5ALCq4fnK7NgpEXEEuBVAkoA9QLlDNV5w5YpvrWdm+dbO\nzP1hYL2ktZL6gE3A9sYTJC3OXgP4EPDNLPBz6aWGYZ65m1k+tZy5R8S4pNuBB4EeYFtE7JJ0W/b6\nFuBVwOckBbAL+OAs1jzrSpUa/X09vMwNw8wsp9ra5xcRO4AdTce2NDz+NvCKzpbWPVO31quvMJmZ\n5Y8/oXoGvrWemeWdw73JsbEJDrxwzOFuZrnmcG+yZ7TeU8YXU80szxzuTUreBmlmCXC4NylVqkhu\nGGZm+eZwb1Ku1BhcvID5c3u6XYqZ2XlzuDepb4P0koyZ5ZvDvcHkZGTbIL0kY2b55nBv8NyR4xw7\nOeGZu5nlnsO9QbniW+uZWRoc7g28DdLMUuFwb1CuVLlkXi/LF83rdilmZjPicG9Qyi6mumGYmeWd\nw71B2dsgzSwRDvfM0bFxnj183BdTzSwJDvfM1E4Zz9zNLAUO94x3yphZShzumVKlhgRXXt7f7VLM\nzGbM4Z4pV6qsWtLvhmFmlgSHe6ZUqfkGHWaWDIc79YZhe0arXm83s2Q43IFnDx/j+MlJz9zNLBkO\ndxobhnnmbmZpaCvcJW2UtFvSsKQ7zvD6ZZL+XdL3Je2SdGvnS5093gZpZqlpGe6SeoC7gRuADcDN\nkjY0nfZh4ImIeA1wPfAXkvo6XOusKVdqLJrfy7JLclOymdm02pm5XwsMR0Q5IsaA+4Cbms4JYJHq\nHbcuAQ4B4x2tdBaVKvWLqW4YZmapaCfcB4F9Dc/3Z8ca3QW8CngW+AHw0YiY7EiFF0DZ2yDNLDGd\nuqD6TuBR4OXA1cBdki5tPknSZkk7Je2sVCodeuuZqZ4Y57kjx73ebmZJaSfcDwCrGp6vzI41uhW4\nP+qGgT3Azzb/oIjYGhFDETE0MDBwvjV31B7fWs/MEtROuD8MrJe0NrtIugnY3nTOXuBtAJKuAF4J\nlDtZ6GzxThkzS1FvqxMiYlzS7cCDQA+wLSJ2Sbote30L8CfAPZJ+AAj4WESMzmLdHVOqVJkjWO2G\nYWaWkJbhDhARO4AdTce2NDx+FnhHZ0u7MMqVGquX9jOv1w3DzCwdhf+Easm31jOzBBU63Ccmgz2j\nNV9MNbPkFDrcn33hGCfGJz1zN7PkFDrcvVPGzFJV8HD3HnczS1Ohw71cqXLZgrksXeiGYWaWlkKH\ne71h2EI3DDOz5BQ63OsNw7zebmbpKWy4v3j8JCMvnvDFVDNLUmHDferWem71a2YpKmy4exukmaWs\n0OHeM0esXuqGYWaWnsKGe7lS48ql/fT1FvaPwMwSVthkqzcM83q7maWpkOE+MRk8PXrU6+1mlqxC\nhvv+548yNjHpcDezZBUy3L0N0sxSV8hw9zZIM0tdQcO9xpL+uSxxwzAzS1RBw73qWbuZJa2Q4V5v\nGOb1djNLV+HC/fCxk4xW3TDMzNJWuHAvZxdT3erXzFJWuHD3rfXMrAjaCndJGyXtljQs6Y4zvP67\nkh7Nvh6XNCFpaefLnblSpUrvHLHKDcPMLGEtw11SD3A3cAOwAbhZ0obGcyLikxFxdURcDdwJfCMi\nDs1GwTNVrlS58vJ+5vYU7pcWMyuQdhLuWmA4IsoRMQbcB9w0zfk3A/d2orjZUKrUfDHVzJLXTrgP\nAvsanu/Pjp1GUj+wEfjizEvrvPGJSZ456Pummln6Or028SvAt862JCNps6SdknZWKpUOv3Vr+54/\nxsmJ8MVUM0teO+F+AFjV8HxlduxMNjHNkkxEbI2IoYgYGhgYaL/KDvE2SDMrinbC/WFgvaS1kvqo\nB/j25pMkXQa8BfhSZ0vsnJcahnnmbmZp6211QkSMS7odeBDoAbZFxC5Jt2Wvb8lOfTfw1YiozVq1\nM1Su1Lh8YR+L+90wzMzS1jLcASJiB7Cj6diWpuf3APd0qrDZ4IZhZlYUhdrs7YZhZlYUhQn3F46O\ncbA25pm7mRVCYcK95FvrmVmBFCjcfWs9MyuOQoV7X88cVi5Z0O1SzMxmXWHCvVypceXl/fS6YZiZ\nFUBhks7bIM2sSAoR7icnJtl78KgvpppZYRQi3PceOsr4ZHjmbmaFUYhwL3sbpJkVTCHCveRukGZW\nMIUI93KlyrJL5nHZgrndLsXM7IIoRLjXb63nJRkzK45ChHu5UvWSjJkVSvLhfqg2xvNHT3rmbmaF\nkny4T91a76rlnrmbWXEkH+6nGoYtc7ibWXEUINxr9PXOYdANw8ysQJIP93KlytrLF9IzR90uxczs\ngkk+3EuVGlct98VUMyuWpMN9bHySvYeOss7r7WZWMEmH+95DNSYmwzN3MyucpMP91H1TPXM3s4JJ\nPNynGoZ55m5mxdJWuEvaKGm3pGFJd5zlnOslPSppl6RvdLbM81Ou1Fi+aB6L5rthmJkVS2+rEyT1\nAHcDbwf2Aw9L2h4RTzScsxj4a2BjROyVtHy2Cj4XvrWemRVVOzP3a4HhiChHxBhwH3BT0znvBe6P\niL0AETHS2TLPXURQrtS8JGNmhdROuA8C+xqe78+ONXoFsETS/0h6RNItnSrwfB2sjXH42EnP3M2s\nkFouy5zDz3kd8DZgAfBtSd+JiB82niRpM7AZYPXq1R166zOburWeG4aZWRG1M3M/AKxqeL4yO9Zo\nP/BgRNQiYhT4JvCa5h8UEVsjYigihgYGBs635rac2imzzMsyZlY87YT7w8B6SWsl9QGbgO1N53wJ\neJOkXkn9wC8AT3a21HNTGqkyr3cOg4vdMMzMiqflskxEjEu6HXgQ6AG2RcQuSbdlr2+JiCcl/Qfw\nGDAJfDYiHp/Nwlspj9ZYu2whc9wwzMwKqK0194jYAexoOral6fkngU92rrSZKVWqvHrwsm6XYWbW\nFUl+QvXE+AT7Dh3lKq+3m1lBJRnuzxw8ymR4p4yZFVeS4V4+tVPG4W5mxZRkuJ/qBulPp5pZQSUa\n7lVedul8Fs7r1Ge0zMzyJdFw9631zKzYkgv3esMwd4M0s2JLLtwr1RO8eHzcbQfMrNCSC3c3DDMz\nSzDcX7q1nsPdzIorvXAfqbFgbg8rLp3f7VLMzLomuXAvj1bdMMzMCi+5cC9Vql5vN7PCSyrcj5+c\nYP/zx7xTxswKL6lwf/pgjXDDMDOztMJ9ahukZ+5mVnRJhXtpZGobpMPdzIotqXAvj9YYXLyA/j43\nDDOzYksq3EuVqmftZmYkFO71hmE1NwwzMyOhcB958QTVE+OeuZuZkVC4T/WU8czdzCypcPet9czM\npqQT7iNV+vt6eJkbhpmZtRfukjZK2i1pWNIdZ3j9ekmHJT2aff1h50udXnm0xrqBhUhuGGZm1nJD\nuKQe4G7g7cB+4GFJ2yPiiaZT/zci3jULNbalNFJlaM2Sbr29mdlFpZ2Z+7XAcESUI2IMuA+4aXbL\nOjfHxiZ49vAx1i3zxVQzM2gv3AeBfQ3P92fHmr1R0mOSviLp5zpSXZv2jE41DPPFVDMzaGNZpk3/\nB6yOiKqkG4EHgPXNJ0naDGwGWL16dYfeun6DDvA2SDOzKe3M3A8Aqxqer8yOnRIRRyKimj3eAcyV\ntKz5B0XE1ogYioihgYGBGZT900ojNSRY626QZmZAe+H+MLBe0lpJfcAmYHvjCZJepmybiqRrs597\nsNPFnk15tMrg4gXMn9tzod7SzOyi1nJZJiLGJd0OPAj0ANsiYpek27LXtwDvAX5d0jhwDNgUETGL\ndf+UesMwL8mYmU1pa809W2rZ0XRsS8Pju4C7Oltae6Yahr1+zdJuvL2Z2UUp959Qfe7IcY6OTXjm\nbmbWIPfhPnVrvavcU8bM7JTch7u7QZqZnS7/4T5S5ZJ5vSxfNK/bpZiZXTRyH+5uGGZmdrrch3tp\npOolGTOzJrkO96Nj4zx7+Djr/MlUM7OfkutwP7VTZrln7mZmjfId7qNT2yAd7mZmjXId7qWRKhJc\neXl/t0sxM7uo5Drcy6M1Vi3pd8MwM7MmuQ730kiVdf5kqpnZaXIb7pOTwZ7RmtfbzczOILfh/uMj\nxzl2csIzdzOzM8htuJfdU8bM7KxyG+6lkXq4e+ZuZna6/IZ7pcai+b0MXOKGYWZmzXIb7uXR+q31\n3DDMzOx0uQ330kjNN+gwMzuLXIZ79cQ4zx057oupZmZnkctw3+Nb65mZTSuX4V4e9TZIM7Pp5DLc\nSyNV5ghWu2GYmdkZ5TPcR2usXtrPvF43DDMzO5O2wl3SRkm7JQ1LumOa814vaVzSezpX4unqDcO8\nJGNmdjYtw11SD3A3cAOwAbhZ0oaznPcJ4KudLrLRSw3DfDHVzOxs2pm5XwsMR0Q5IsaA+4CbznDe\nR4AvAiMdrO80B144xonxSc/czcym0U64DwL7Gp7vz46dImkQeDfwmc6Vdma+tZ6ZWWuduqD6KeBj\nETE53UmSNkvaKWlnpVI5rzda2NfD2zdc4WUZM7Np9LZxzgFgVcPzldmxRkPAfVmfl2XAjZLGI+KB\nxpMiYiuwFWBoaCjOp+ChNUsZWrP0fL7VzKww2gn3h4H1ktZSD/VNwHsbT4iItVOPJd0DfLk52M3M\n7MJpGe4RMS7pduBBoAfYFhG7JN2Wvb5llms0M7Nz1M7MnYjYAexoOnbGUI+ID8y8LDMzm4lcfkLV\nzMym53A3M0uQw93MLEEOdzOzBDnczcwSpIjz+izRzN9YqgDPnOe3LwNGO1hOHnjMxeAxF8NMxnxl\nRAy0Oqlr4T4TknZGxFC367iQPOZi8JiL4UKM2csyZmYJcribmSUor+G+tdsFdIHHXAweczHM+phz\nueZuZmbTy+vM3czMppG7cG/3Zt0XO0mrJP23pCck7ZL00ez4Ukn/KelH2T+XNHzPndm4d0t6Z8Px\n10n6Qfbap5U11r9YSeqR9D1JX86eJz1mSYslfUHSU5KelPSGAoz5t7L/rh+XdK+k+amNWdI2SSOS\nHm841rExSpon6fPZ8YckrTmnAiMiN1/UWw6XgHVAH/B9YEO36zrPsawArskeLwJ+SP0G5H8G3JEd\nvwP4RPZ4QzbeecDa7M+hJ3vtu8B1gICvADd0e3wtxv7bwD9T7/tP6mMGPgd8KHvcByxOeczUb8O5\nB1iQPf8X4AOpjRl4M3AN8HjDsY6NEfgNYEv2eBPw+XOqr9t/QOf4h/kG4MGG53cCd3a7rg6N7UvA\n24HdwIrs2Apg95nGSr2//huyc55qOH4z8DfdHs8041wJfA14a0O4Jztm4LIs6NR0POUxT913eSn1\ntuJfBt6R4piBNU3h3rExTp2TPe6l/qEntVtb3pZlWt6sO4+yX7deCzwEXBERP85eeg64Int8trEP\nZo+bj1+sPgX8HtB4v92Ux7wWqAB/ly1FfVbSQhIec0QcAP4c2Av8GDgcEV8l4TE36OQYT31PRIwD\nh4HL2y0kb+GeHEmXAF8EfjMijjS+FvW/spPZziTpXcBIRDxytnNSGzP1Gdc1wGci4rVAjfqv66ek\nNuZsnfkm6n+xvRxYKOl9jeekNuYz6fYY8xbu7dysOzckzaUe7P8UEfdnh38iaUX2+gpgJDt+trEf\nyB43H78Y/SLwq5KeBu4D3irpH0l7zPuB/RHxUPb8C9TDPuUx/zKwJyIqEXESuB94I2mPeUonx3jq\neyT1Ul/iO9huIXkL91M365bUR/0iw/Yu13Resivifws8GRF/2fDSduD92eP3U1+Lnzq+KbuCvhZY\nD3w3+xXwiKTrsp95S8P3XFQi4s6IWBkRa6j/u/t6RLyPtMf8HLBP0iuzQ28DniDhMVNfjrlOUn9W\n69uAJ0l7zFM6OcbGn/Ue6v+/tP+bQLcvSJzHBYwbqe8sKQEf73Y9MxjHm6j/yvYY8Gj2dSP1NbWv\nAT8C/gtY2vA9H8/GvZuGXQPAEPB49tpdnMNFly6O/3peuqCa9JiBq4Gd2b/rB4AlBRjzHwNPZfX+\nA/VdIkmNGbiX+jWFk9R/Q/tgJ8cIzAf+FRimvqNm3bnU50+ompklKG/LMmZm1gaHu5lZghzuZmYJ\ncribmSXI4W5mliCHu5lZghzuZmYJcribmSXo/wGglyzEj5hydgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb159b4790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([beta_by_frame(i) for i in range(10000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Cart Pole Environment</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <type 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env_id = \"CartPole-v0\"\n",
    "env = gym.make(env_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 500\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fdb0bcc4090>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGZ9JREFUeJzt3Xt0XeV55/Hvc87R0f0uIVuyfOWSOCYGIxyTpAkJk8Sw\nhrjpZFaBpiRMM5QZaDtt15qQ1TUzTbramSSTWSkNwWEYwjTthJKUJE5iYiYXYBIuQYABG9sgG4xk\nW1i+ybZkWbdn/jhb5liWrGP5SFtn799nLS3ts/d7dJ7XJD+9eve79zZ3R0REoiURdgEiIpJ/CncR\nkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQamwPrihocEXL14c1seLiBSk\n55577oC7N07VLrRwX7x4Me3t7WF9vIhIQTKz3bm007SMiEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hE\n0JThbmb3m9l+M9syyXEzs7vMrMPMXjKzVfkvU0REzkUuI/cHgLVnOX4tcFHwdStwz/mXJSIi52PK\ncHf3J4BDZ2myDvh7z3gaqDGz+fkqcLwd3cf4yqbtHO4bnKmPEBEpePmYc28BOrNedwX7zmBmt5pZ\nu5m19/T0TOvDXj/Qx92/3MmeIyem9X4RkTiY1ROq7n6vu7e5e1tj45RXz06oviINwCGN3EVEJpWP\ncN8DtGa9XhDsmxF15Qp3EZGp5CPcNwA3B6tm1gC97r4vDz93QvVBuB9UuIuITGrKG4eZ2XeAq4EG\nM+sC/gtQBODu64GNwHVAB9AP3DJTxQJUlRSRTBiH+k7O5MeIiBS0KcPd3W+c4rgDt+etoikkEkZt\nWVrTMiIiZ1GQV6jWl6c5eFzhLiIymYIM99ryIo3cRUTOoiDDvb68WOEuInIWBRnudeVprZYRETmL\ngg333hNDDI2Mhl2KiMicVJDhPnaV6pH+oZArERGZmwoy3HWVqojI2RV0uB/UhUwiIhMqyHCvLy8G\nNHIXEZlMQYa7pmVERM6uIMO9tqwIQFepiohMoiDDPZVMUFOmq1RFRCZTkOEOmakZhbuIyMQKNtzr\ny9NaLSMiMomCDXeN3EVEJlfA4a6bh4mITKZgw72+PM3h/iFGRz3sUkRE5pzCDfeKNCOjzpETur+M\niMh4BRvuDRWZq1QPHNdJVRGR8Qo23Bsrg3A/pnAXERmv4MO9RyN3EZEzFGy4j03L9GjkLiJyhoIN\n96qSFOlUQuEuIjKBgg13M6OxoljTMiIiEyjYcIfMvLtG7iIiZyrocG+oULiLiEykoMO9sbJY69xF\nRCZQ8OF+qG+QEd2CQETkNIUd7hVpRl0PyhYRGa+ww71Sa91FRCYSiXA/oGepioicJqdwN7O1ZrbD\nzDrM7M4Jjleb2Y/M7EUz22pmt+S/1DM1VpQAGrmLiIw3ZbibWRK4G7gWWA7caGbLxzW7HXjF3VcC\nVwNfNbN0nms9Q0Nl5iMU7iIip8tl5L4a6HD3Xe4+CDwIrBvXxoFKMzOgAjgEDOe10gmUpVOUp5MK\ndxGRcXIJ9xagM+t1V7Av29eBdwJ7gZeBP3H30fE/yMxuNbN2M2vv6emZZsmn01p3EZEz5euE6seA\nzUAzcBnwdTOrGt/I3e919zZ3b2tsbMzLB+sqVRGRM+US7nuA1qzXC4J92W4BHvaMDuB14B35KfHs\nGit18zARkfFyCfdngYvMbElwkvQGYMO4Nm8C1wCYWRNwCbArn4VOprGymP1HB2bjo0RECkZqqgbu\nPmxmdwCbgCRwv7tvNbPbguPrgb8CHjCzlwEDPufuB2aw7lOaqko4OjDMwNAIJUXJ2fhIEZE5b8pw\nB3D3jcDGcfvWZ23vBT6a39Jy01SVWev+1tEBFtWXh1GCiMicU9BXqAI0VWWuUu3u1dSMiMiYgg/3\necHIvVvz7iIipxR8uDdVvz0tIyIiGQUf7pXFKcrSSbp7tRxSRGRMwYe7mTGvqkQjdxGRLAUf7pBZ\nMaM5dxGRt0Ui3OdVa+QuIpItEuHeVFXC/qMncdezVEVEICLhPq+qmMGRUQ716YlMIiIQkXBv0lp3\nEZHTRCPctdZdROQ0kQj3U1epaq27iAgQkXBvrCzGTNMyIiJjIhHuRckEDRW6r7uIyJhIhDtkpmY0\nchcRyYhMuDdVlei2vyIigciEe0tNCXuPnAi7DBGROSEy4d5cU8rRgWGODQyFXYqISOgiFe4Ae49o\nakZEJDLh3lI7Fu6amhERiU64ByP3LoW7iEh0wr2xopiipGnkLiJChMI9kTDmV5ey57DCXUQkMuEO\n0KzlkCIiQMTCvaWmTOEuIkLkwj1zC4KhkdGwSxERCVWkwr25ppRR133dRUQiFe5ja911UlVE4i5S\n4X7qKtVehbuIxFu0wr1aI3cREcgx3M1srZntMLMOM7tzkjZXm9lmM9tqZo/nt8zclKaT1Jen2aP7\ny4hIzKWmamBmSeBu4CNAF/CsmW1w91ey2tQA3wDWuvubZnbBTBU8leaaUroO94f18SIic0IuI/fV\nQIe773L3QeBBYN24NjcBD7v7mwDuvj+/ZeZuYV0ZXZqWEZGYyyXcW4DOrNddwb5sFwO1ZvaYmT1n\nZjfnq8Bz1VpXRtfhfkZGPawSRERCN+W0zDn8nCuAa4BS4Ckze9rdX81uZGa3ArcCLFy4ME8ffbpF\n9WUMjTj7ek+woLZsRj5DRGSuy2XkvgdozXq9INiXrQvY5O597n4AeAJYOf4Hufu97t7m7m2NjY3T\nrfmsFtZlAv3NQ5p3F5H4yiXcnwUuMrMlZpYGbgA2jGvzQ+D9ZpYyszLgPcC2/Jaam1PhflDhLiLx\nNeW0jLsPm9kdwCYgCdzv7lvN7Lbg+Hp332ZmPwVeAkaB+9x9y0wWPpn51SWkEqaRu4jEWk5z7u6+\nEdg4bt/6ca+/Anwlf6VNTyqZYEFtKbsV7iISY5G6QnVMa10ZnQp3EYmxSIb7ovoyTcuISKxFMtwX\n1pVxpH+I3hNDYZciIhKKyIY7oKkZEYmtiIZ7OQC7tRxSRGIqmuFerwuZRCTeIhnuFcUp6svT7D7Y\nF3YpIiKhiGS4AyxpKGfXAYW7iMRTZMN9aWM5u3oU7iIST5EN9yUNFRw4fpKjA1oOKSLxE9lwX9qY\nWTGj0buIxFFkw33ZqXA/HnIlIiKzL7LhvrCunGTCeF0nVUUkhiIb7ulUgtbaUk3LiEgsRTbcAZY2\nVrBT0zIiEkPRDveGct442MeoHpYtIjET7XBvrGBgaJS9vSfCLkVEZFZFPNy1HFJE4ikm4a55dxGJ\nl0iHe2NFMVUlKV7br3AXkXiJdLibGZfMq+TVt46FXYqIyKyKdLgDXNxUyY7uY7hrxYyIxEfkw/2S\neZUcHRjmraMnwy5FRGTWRD/cmyoB2KGpGRGJkciH+8Vj4d59NORKRERmT+TDvbY8zQWVxezo1ooZ\nEYmPyIc7oBUzIhI7sQj3i5sqeW3/MUZ0jxkRiYlYhPslTZUMDI3Seag/7FJERGZFLML94nmZk6rb\nuzU1IyLxEItwv6SpkoTBtn1aMSMi8ZBTuJvZWjPbYWYdZnbnWdpdaWbDZvbJ/JV4/krTSZY1VrB1\nb2/YpYiIzIopw93MksDdwLXAcuBGM1s+SbsvAY/mu8h8WNFSzZY9GrmLSDzkMnJfDXS4+y53HwQe\nBNZN0O6PgH8G9uexvrx5V3MV3UcH6Dmm2xCISPTlEu4tQGfW665g3ylm1gJ8Argnf6Xl14qWagBN\nzYhILOTrhOrXgM+5++jZGpnZrWbWbmbtPT09efro3CxvrgJg615NzYhI9KVyaLMHaM16vSDYl60N\neNDMABqA68xs2N1/kN3I3e8F7gVoa2ub1SuKqkqKWFRfxpY9GrmLSPTlEu7PAheZ2RIyoX4DcFN2\nA3dfMrZtZg8APx4f7HPBiuZqXtpzJOwyRERm3JTTMu4+DNwBbAK2AQ+5+1Yzu83MbpvpAvPpXS1V\ndB46QW//UNiliIjMqFxG7rj7RmDjuH3rJ2n7mfMva2asaM6cVN2yt5f3XdgQcjUiIjMnFleojlm5\noAaAzZ2amhGRaItVuFeXFbGssZzndx8OuxQRkRkVq3AHWLWwlhc6j+iB2SISabEL98sX1nKob5Dd\nB3X7XxGJrtiF+6pFmXn359/U1IyIRFfswv2iCyqpKE7xwps6qSoi0RW7cE8mjJWt1Rq5i0ikxS7c\nAS5vrWV79zH6B4fDLkVEZEbEMtyvWFTLyKizWVMzIhJRsQz3tsW1JAye3nUw7FJERGZELMO9sqSI\nS1uqeUrhLiIRFctwB1izrJ7NnUc4MTgSdikiInkX33BfWs/QiGvVjIhEUmzDvW1RLcmEad5dRCIp\ntuFeWVLEipZqhbuIRFJswx1gzdI6Nnce0Xp3EYmcWIf7+y9sYGjENXoXkciJdbhfubiO0qIkj+/o\nCbsUEZG8inW4lxQluWpZPY+9qnAXkWiJdbgDXH1JI7sP9vP6gb6wSxERyZvYh/sHL24E4PEd+0Ou\nREQkf2If7ovqy1nSUK6pGRGJlNiHO2RG70/tPKglkSISGQp34KPLmzg5PMoTGr2LSEQo3IHVS+qo\nKSvip1u6wy5FRCQvFO5AKpngI+9s4ufb9nNyWHeJFJHCp3APXHvpPI6dHObJnbpaVUQKn8I98N5l\nDVQUp/jpy5qaEZHCp3APlBQl+dA7LuDRV7oZHB4NuxwRkfOicM/y25c1c7h/iMe1akZECpzCPcsH\nLm6kvjzN91/oCrsUEZHzklO4m9laM9thZh1mducEx3/PzF4ys5fN7EkzW5n/UmdeUTLB9Sub+dkr\n++ntHwq7HBGRaZsy3M0sCdwNXAssB240s+Xjmr0OfNDdLwX+Crg334XOlt9Z1cLgyCg/eXlf2KWI\niExbLiP31UCHu+9y90HgQWBddgN3f9Ldx540/TSwIL9lzp5LW6q58IIKvvdcZ9iliIhMWy7h3gJk\nJ11XsG8yfwA8cj5FhcnM+N22Vp5/8wjb9h0NuxwRkWnJ6wlVM/sQmXD/3CTHbzWzdjNr7+mZuytS\nPnnFAopTCf7h6d1hlyIiMi25hPseoDXr9YJg32nM7N3AfcA6d5/wMk93v9fd29y9rbGxcTr1zora\n8jTXr2zm+y/s4diATqyKSOHJJdyfBS4ysyVmlgZuADZkNzCzhcDDwO+7+6v5L3P2/f6aRfQPjvD9\nF874PSYiMudNGe7uPgzcAWwCtgEPuftWM7vNzG4Lmv1noB74hpltNrP2Gat4lqxsrWHlgmoe+PUb\njIx62OWIiJyTnObc3X2ju1/s7svc/a+DfevdfX2w/Vl3r3X3y4Kvtpkserb82w8sZdeBPh7dqvvN\niEhh0RWqZ3Htivksri/jG4/txF2jdxEpHAr3s0gmjD/84DJe3tPLrzoOhF2OiEjOFO5T+J1VLTRV\nFfN3v+jQ6F1ECobCfQrFqSS3f+hCfvP6IR7T3SJFpEAo3HNww5ULWVhXxpce2c6oVs6ISAFQuOcg\nnUrw5x+9mO3dx/jhi1r3LiJzn8I9R9e/u5kVLVV8+ac76Ds5HHY5IiJnpXDPUSJhfOHjK9jXO8DX\nfhaJi3BFJMIU7ufgikW13HBlK/f/+g3dMVJE5jSF+zn63Np3UFWS4s6HX2Z4RA/SFpG5SeF+jmrL\n03xh3Qpe7DzC3b/cGXY5IiITUrhPw8dXNrPusmbu+sVrbO48EnY5IiJnULhP0xfXraCpspg//s4L\nepi2iMw5Cvdpqi4t4u9uWsW+3hP80YMv6LbAIjKnKNzPwxWLavniuhU88WoPX960PexyREROSYVd\nQKG7cfVCtu7t5ZuP72J+VQmfed+SsEsSEVG458NfXv8u9h89yV/+6BVqytL89uUtYZckIjGnaZk8\nSCUT3HXj5Vy1tJ4//+6L/EDPXRWRkCnc86SkKMn//HQbqxfX8acPbebbT70RdkkiEmMK9zyqKE7x\nrVuu5Jp3NPGffriV//rINq2iEZFQKNzzrKQoyT2fWsXvvWch33x8F//mgWe1Dl5EZp3CfQYUJRP8\n9Scu5W8+cSlP7jzAtX/7BE/qGawiMosU7jPopvcs5Lu3vZeSoiQ33fcMX/jRVo4NaBQvIjNP4T7D\nLmut4Sd//FvcfNUivvXrN/jwVx/n4ee79Lg+EZlRCvdZUJpO8sV1K/jB7e+juaaUP3voRa7/+q/Y\ntLUbd4W8iOSfwn0WXdZaw/f/3Xv56r9eyfGTw/zht5/jurt+xXfbOxkYGgm7PBGJEAtr5NjW1ubt\n7e2hfPZcMDwyyoYX9/KNx3bSsf84VSUp/tUVC/jE5S1c2lKNmYVdoojMQWb2nLu3TdlO4R4ud+fp\nXYf4x2d2s2lrN0MjTmtdKdddOp+PvLOJla01FCX1B5aIZCjcC9DhvkEefaWbn7zczZMdBxgedSqK\nU6xZWsdVyxq4fGENy+dXUVKUDLtUEQmJwr3A9fYP8dSuA/y/1w7w644DvHGwH4BUwrhkXiXvXlDN\nhRdUsqyxnGWNFbTUlJJIaCpHJOpyDXfdFXKOqi4rYu2K+axdMR+A7t4BXuw6wktdR3ixs5dHtnRz\npL/zVPuSogSttWU015TSXFPC/OpS5leXMK+6hLryNHXlaWrL0hr1i8RETuFuZmuBvwWSwH3u/t/G\nHbfg+HVAP/AZd38+z7XG2rzqEuZVz+Nj75p3at/B4yfZ2dNHx/7jdOw/Ttfhfvb1DrB1by8Hjg9O\n+HNKi5LUlaepKSuisiRFeTpFeXGK8uIk5ekUZcUpKoqTlKVTFKcSpFMJ0snge7BdFHwvTiUoCl4n\nzUgkIGlGMmEkEvb29qnv6ESxyCyZMtzNLAncDXwE6AKeNbMN7v5KVrNrgYuCr/cA9wTfZQbVVxRT\nX1HM6iV1ZxwbGBqhu3eAt44OcLh/iMP9gxzqG+Rw3yCH+jPf+06O0H10gL6Tw/QNjtAffJ9JCeO0\nwE+aYUHoj+W+8fYvgcz22FZme+zXQ2Z7ov1v/wLJ/OzJ2825XzVzrKA5Vs6cGxxMt5rfvbKVz/7W\n0rzWMl4uI/fVQIe77wIwsweBdUB2uK8D/t4zE/hPm1mNmc139315r1hyUlKUZHFDOYsbys/pfaOj\nzomhEfoGhzk5NMrgyCiDw6MMBd8Hh0c5mbU9tn/UYcSd0VFnZNQZ9cz3t/dlHc9qN+KOO6cu5nJg\n7DSQExwLasvs91Pb49tN9H5Oe7+f9rPm2uVjc+2CtrlVDXOuID+PghoqivNYycRyCfcWoDPrdRdn\njsonatMCKNwLTCJhwTSNTseIFLJZXUBtZreaWbuZtff09MzmR4uIxEou4b4HaM16vSDYd65tcPd7\n3b3N3dsaGxvPtVYREclRLuH+LHCRmS0xszRwA7BhXJsNwM2WsQbo1Xy7iEh4ppxYdfdhM7sD2ERm\nKeT97r7VzG4Ljq8HNpJZBtlBZinkLTNXsoiITCWns2buvpFMgGfvW5+17cDt+S1NRESmS3ekEhGJ\nIIW7iEgEKdxFRCIotLtCmlkPsHuab28ADuSxnEKgPseD+hwP59PnRe4+5Vry0ML9fJhZey63vIwS\n9Tke1Od4mI0+a1pGRCSCFO4iIhFUqOF+b9gFhEB9jgf1OR5mvM8FOecuIiJnV6gjdxEROYuCC3cz\nW2tmO8ysw8zuDLue6TKzVjP7pZm9YmZbzexPgv11ZvZ/zey14Htt1ns+H/R7h5l9LGv/FWb2cnDs\nLptrj6sZx8ySZvaCmf04eB3pPgcPr/memW03s21mdlUM+vynwf+ut5jZd8ysJGp9NrP7zWy/mW3J\n2pe3PppZsZn9U7D/GTNbfE4FunvBfJG5cdlOYCmQBl4Elodd1zT7Mh9YFWxXAq8Cy4EvA3cG++8E\nvhRsLw/6WwwsCf4dksGx3wBryDz16xHg2rD7N0Xf/wz4P8CPg9eR7jPwv4HPBttpoCbKfSbzoJ7X\ngdLg9UPAZ6LWZ+ADwCpgS9a+vPUR+PfA+mD7BuCfzqm+sP+BzvEf8ypgU9brzwOfD7uuPPXth2Se\nU7sDmB/smw/smKivZO7SeVXQZnvW/huBb4bdn7P0cwHwc+DDWeEe2T4D1UHQ2bj9Ue7z2JPZ6sjc\nnPDHwEej2Gdg8bhwz1sfx9oE2ykyFz1ZrrUV2rTMZI/zK2jBn1uXA88ATf72vfC7gaZge7K+twTb\n4/fPVV8D/iMwmrUvyn1eAvQA3wqmou4zs3Ii3Gd33wP8d+BNMo/a7HX3R4lwn7Pks4+n3uPuw0Av\nUJ9rIYUW7pFjZhXAPwP/wd2PZh/zzK/syCxnMrN/Cex39+cmaxO1PpMZca0C7nH3y4E+Mn+unxK1\nPgfzzOvI/GJrBsrN7FPZbaLW54mE3cdCC/ecHudXKMysiEyw/6O7PxzsfsvM5gfH5wP7g/2T9X1P\nsD1+/1z0PuDjZvYG8CDwYTP7B6Ld5y6gy92fCV5/j0zYR7nP/wJ43d173H0IeBh4L9Hu85h89vHU\ne8wsRWaK72CuhRRauOfyyL+CEJwR/1/ANnf/H1mHNgCfDrY/TWYufmz/DcEZ9CXARcBvgj8Bj5rZ\nmuBn3pz1njnF3T/v7gvcfTGZ/3a/cPdPEe0+dwOdZnZJsOsa4BUi3Gcy0zFrzKwsqPUaYBvR7vOY\nfPYx+2d9ksz/X3L/SyDsExLTOIFxHZmVJTuBvwi7nvPox/vJ/Mn2ErA5+LqOzJzaz4HXgJ8BdVnv\n+Yug3zvIWjUAtAFbgmNf5xxOuoTY/6t5+4RqpPsMXAa0B/+tfwDUxqDPXwC2B/V+m8wqkUj1GfgO\nmXMKQ2T+QvuDfPYRKAG+S+bxpb8Blp5LfbpCVUQkggptWkZERHKgcBcRiSCFu4hIBCncRUQiSOEu\nIhJBCncRkQhSuIuIRJDCXUQkgv4/vspwDQYKcwUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb6028f5d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(10000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Deep Q Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions):\n",
    "        super(DQN, self).__init__()\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(env.observation_space.shape[0], 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, env.action_space.n)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            state   = Variable(torch.FloatTensor(state).unsqueeze(0), volatile=True)\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0]\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_model = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "target_model  = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    current_model = current_model.cuda()\n",
    "    target_model  = target_model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(current_model.parameters())\n",
    "\n",
    "replay_buffer = NaivePrioritizedBuffer(100000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Synchronize current policy net and target net</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_target(current_model, target_model):\n",
    "    target_model.load_state_dict(current_model.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "update_target(current_model, target_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Computing Temporal Difference Loss</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_td_loss(batch_size, beta):\n",
    "    state, action, reward, next_state, done, indices, weights = replay_buffer.sample(batch_size, beta) \n",
    "\n",
    "    state      = Variable(torch.FloatTensor(np.float32(state)))\n",
    "    next_state = Variable(torch.FloatTensor(np.float32(next_state)))\n",
    "    action     = Variable(torch.LongTensor(action))\n",
    "    reward     = Variable(torch.FloatTensor(reward))\n",
    "    done       = Variable(torch.FloatTensor(done))\n",
    "    weights    = Variable(torch.FloatTensor(weights))\n",
    "\n",
    "    q_values      = current_model(state)\n",
    "    next_q_values = target_model(next_state)\n",
    "\n",
    "    q_value          = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n",
    "    next_q_value     = next_q_values.max(1)[0]\n",
    "    expected_q_value = reward + gamma * next_q_value * (1 - done)\n",
    "    \n",
    "    loss  = (q_value - expected_q_value.detach()).pow(2) * weights\n",
    "    prios = loss + 1e-5\n",
    "    loss  = loss.mean()\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    replay_buffer.update_priorities(indices, prios.data.cpu().numpy())\n",
    "    optimizer.step()\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Training</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv4AAAE/CAYAAAA+Occ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XucXHV5P/DPMzO7m3sCJERMCAGJWiiKNaBtlaLYgpeK\n2lZBf3jDIq3609b+Kmop3mitilrkZhTkInJRFKJcAxISQgJJICTZhCSbkJBkk93NZrP33bmc5/fH\nOd9zvjN7Znd2LjuZM5/365XXzpw5M/OdnezMc57zfJ+vqCqIiIiIiCjaYtUeABERERERVR4DfyIi\nIiKiOsDAn4iIiIioDjDwJyIiIiKqAwz8iYiIiIjqAAN/IiIiIqI6wMA/AkTkdSKyQUR6ReT/Vns8\nVFki8kkRebra4yAiihoR2S0i76r2OIgqhYF/NPw7gCdVdbqqXlvtweQSkSUisk1EHBH5ZMjt/yIi\nB0WkR0RuEZEm67ZjReR3ItIvIntE5KM59z1PRF4SkQEReVJETrJuExH5HxHp9P79j4hIRV/sUUZE\nXisiD4hIh4gcFpFHReR1OfsU/fsPeb68j0VERETVxcA/Gk4C0JzvRhGJT+BYwrwI4J8BPJ97g4ic\nD+AKAOfBfR2nAPimtcv1AJIA5gL4GIAbReR0776zAfwWwJUAjgWwDsA91n0vA/ABAG8E8AYAfwvg\ns8W8ABFJFHO/UpXhvZsFYCmA18H9HT4H4AHr8Yv+/YeMdazHIiIioipi4F/jROSPAN4B4DoR6fMy\nvLeKyI0i8pCI9AN4h4i8V0Re8DKxe0XkG9ZjLBQRFZFPebd1icjlInKWiGwUkSMicl3O835aRLZ6\n+z5qZ9pzqer1qvoEgKGQmz8B4GZVbVbVLgDfAvBJ7zmmAvg7AFeqap+qPg03aL3Eu++HADSr6q9V\ndQjANwC8UURebz32Naq6T1X3A/iBeewCfq+fFJFVIvIjEen0Hjvv6xaRb4rIT7zLDV6G/Pve9cki\nMiQix3rXf+1lxbtFZIUdSOd5744TkaXee/ccgNcU8hoAQFWfU9WbVfWwqqYA/AjA60TkuDL8/nPl\nfSwioloiIk0i8mMRafX+/dicwRSR2SLyB++78bCIrBSRmHfbV0Rkv7ilt9tE5LzqvhKibAz8a5yq\nvhPASgCfV9Vpqrrdu+mjAK4GMB3A0wD6AXwcbgb4vQD+SUQ+kPNwbwGwCMBHAPwYwNcBvAvA6QA+\nLCJ/BQAiciGAr8ENvOd4z39XkS/hdLhnBIwXAcz1AtPXAkhbr8ncfnrYfVW1H0BLvttz7luItwDY\nBTfbffUYr/spAOd6l88CcBDAOd71PwewTVUPe9cfhvt7Ph7uWZA7c5439727Hu5B0wkAPu3983lf\nQFcU+JrOAXBQVTu966X8/nON9lhERLXk6wDeCuBMuGeNzwbwH95tXwawD+73wFy43wvqlVF+HsBZ\nqjodwPkAdk/ssIlGx8A/uh5Q1VWq6qjqkKouV9VN3vWNcAPWv8q5z7e9fR+De6Bwl6q2e9nylQDe\n5O13OYD/VtWtqpoG8F8Azhwt6z+KaQC6res93s/p3m09Ofv3eLeF3Xes23sATBtHnX+rqv5EVdOq\nOojRX/dqAIu8IPccADcDmCci0+D+np8yD6qqt6hqr6oOIzhLMdN6Xv+9A5CCm3X/T1XtV9XNAG6z\nB6mq71PV7471YkRkPtyDiH+1Npfy+8812mMREdWSjwH4lvcd2AG3bNGc7UzBTcScpKopVV2pqgog\nA6AJwGki0qCqu1V1Z1VGT5QHA//o2mtfEZG3iDv5tUNEuuEGsbNz7tNmXR4MuT7Nu3wSgP/1TnMe\nAXAYgACYV8Q4+wDMsK6bALg35DZze2+e+451+0wAfd4HdCH25lzP+7q9A4N1cIP8c+AG+s8A+EtY\ngb+IxEXkuyKyU0R6EGSD7PfCft45ABI52/YUOH6fiMwB8BiAG1TVPjtTyu8/12iPRURUS16N7M/a\nPd42APg+3LPLj4nILnPGVVVbAHwJbkKnXUTuFpFXg+gowsA/unKD21/BneR5oqrOBHAT3KC1GHsB\nfFZVZ1n/JqvqM0U8VjPc06jGGwG0eaUo2wEkRGRRzu3NYff1atJfk+/2nPsWIvd3ONbrfgrAO+Ge\nGVnrXT8f7iniFd4+HwVwIdwSqpkAFprh53neDgBpACda2xaM4zVARI6BG/QvVdWrc24u5fefa7TH\nIiKqJa1wkz3GAm8bvDO2X1bVUwC8H8C/mlp+Vf2Vqr7Nu68C+J+JHTbR6Bj414/pAA6r6pCInA03\nAC3WTQC+KkF3nZki8g/5dhaRRhGZBDe4bRCRSWYiFIDbAVwqIqd5AeqVAG4F/Jr93wL4lohMFZG3\nwf2QvcO77+8A/KmI/J33+FcBeFFVX7Ie+19FZJ6IzINbl3lrBV/3U3DnUWxR1SSA5QA+A+Bl71Qx\n4L4PwwA6AUyBWy6Ul6pm4P4OviEiU0TkNLiTaAsiIjMAPApglaqGzQMo5fdf8GMREdWYuwD8h4jM\nEbeD3H8C+CUAiMj7RORUr2y0G26JjyPumjrv9CYBD8E9U+5UafxEoRj4149/hhvA9cL9ALu32AdS\n1d/BzWLc7ZWrbAbw7lHu8hjcD8C/ALDEu3yO91iPAPgegCfhnkp9GW4Ab497MoB2uGct/klVm737\ndsCtf78aQBfczPpF1n1/CuD3ADZ5//7gbQMAiEiziHysjK/7GW+sJru/Be6H/wprn9u917nfu31N\nAU/9ebhlVgfhBtK/sG8UkYdF5Gt57vtBuJONPyVu1yfzb4H3mor+/YvIgnE+FhFRrfgO3PLNjXC/\nP573tgFuc4bH4ZY3roZbQvkk3Pr+7wI4BPfz+ngAX53YYRONTgovdyYiIiIiolrFjD8RERERUR1g\n4E9EREREVAcY+BMRERER1QEG/kREREREdYCBPxERERFRHUhUewAAMHv2bF24cGG1h0FEdFRav379\nIVWdU+1xVBO/J4iIwo3nO+KoCPwXLlyIdevWVXsYRERHJRHZU+0xVBu/J4iIwo3nO4KlPkRERERE\ndYCBPxERERFRHWDgT0RERERUBxj4ExERERHVAQb+RERERER1gIE/EREREVEdYOBPRERERFQHxgz8\nReREEXlSRLaISLOIfNHbfqyILBORHd7PY6z7fFVEWkRkm4icX8kXQEREREREYysk458G8GVVPQ3A\nWwF8TkROA3AFgCdUdRGAJ7zr8G67CMDpAC4AcIOIxCsxeCIiIiIiKsyYK/eq6gEAB7zLvSKyFcA8\nABcCONfb7TYAywF8xdt+t6oOA3hZRFoAnA1gdbkHT0RUiMFkBpv2d+Psk48FAKgqVuw4hLefOhtr\ndnXirJOPRUM8yIOs3X0YLe19AIC3nTobaUcRE0AgWLXzUFFjOGvhsTj1+Gmlvxgal9U7O9HeO4QL\nz5xX7aEQEVXdmIG/TUQWAngTgGcBzPUOCgDgIIC53uV5ANZYd9vnbct9rMsAXAYACxYsGM8wiIjG\n5RtLm3HPur148t/Oxcmzp+L21Xtw1dJmXHbOKViyYhf+8e0n4+vvPc3f//I71qOzPwkA+NCb5uG3\nL+wHAJz3+uPxxEvtRY3hux86g4F/Ffxm/T6s2dXJwJ+ICOMI/EVkGoD7AHxJVXtExL9NVVVEdDxP\nrKpLACwBgMWLF4/rvkRE4/HK4QEAQOuRQZw8eyoe39oGAOgZTAEAtrX1Ze0/lMrg4rNPxJpdhzGU\nzvjbt7X14vzT5+Kb7//TcY9hxuRx5VmIiIjKrqBvIhFpgBv036mqv/U2t4nICap6QEROAGDSYPsB\nnGjdfb63jYioKmZNaQAAdA24Wfzm1h4AwIzJ7vaM42Ttn1HF9EkNaErEkM4EeYl9XYP48OIT8aqZ\nkyZi2ERERGVVSFcfAXAzgK2q+kPrpqUAPuFd/gSAB6ztF4lIk4icDGARgOfKN2QiovGZNaURANA1\n4Gb4D3tlPDHvzKUd3AOA47i3xWMCR7NvW8RynZpinZwmIqp7hWT8/xLAJQA2icgGb9vXAHwXwL0i\ncimAPQA+DACq2iwi9wLYArcj0OdUNTPyYYmIJsYxXsb/SH8S7T1D/vaYFxRmnOzgPqOKeAxIxARp\nR3HMlAb/oIF1+rVHldWkRERAYV19ngaQL2dyXp77XA3g6hLGRURUNqakp2sghe1WPb+f8c8N/B1F\n3Mv4ZxzFsVMb0TWQQiImOOm4qRM3cCoZE/5ERAGu3EtEkWeCvyMDSQymghOQMS/lb2f8TXZYrMDf\n3HzScVPQmODHJhER1SZ+gxFR5JnAvWsgOWIiL5Cd8TcHAfGYG/inHUXGUSw8bgq+84EzJmS8VF4s\n9CEicjHwJ6LIMxN0uwZSSFkTeR0vyHfswF+DwD8RiyHjBf6LFx6LP3/NcRM4aioHTu4lIgow8Cei\nyDPlO92DKaStjL8J8lPWNnMxJoKYl/FPOw7ijCALIiK3iEi7iGzOc7uIyLUi0iIiG0Xkzyo9Js7t\nJSJyMfAnosizS33SIRn/TGjG3+3q43gZ/3icgX+BbgVwwSi3vxtum+dFcFdvv7GSgxFO7yUi8jHw\nJ6LIc6yMfzITZPdNbb99MGAOAkwff1Pjn4gxgCyEqq4AcHiUXS4EcLu61gCY5S0CSUREFcbAn4gi\nzyT0VYHOvqS/PROS8XeswD8RE2QcB2lH/dafVLJ5APZa1/d52ypGOb2XiAgAA38iqgP2Ak5m1V4g\nOBOQsW53rMm9Ma+dJzP+1SEil4nIOhFZ19HRUeRjlHlQREQ1jIE/EUWeYwX2g8mgj39Q6jNywm8s\nZjL+ijRr/MtpP4ATrevzvW0jqOoSVV2sqovnzJlT9BNyci8RkYuBPxFFnr0wr72AlynrSWeV+rg/\n41aNv8OMfzktBfBxr7vPWwF0q+qBSj0ZM/5ERIFEtQdARFRpWRl/K/APq/G3u/rExcr4M4IsiIjc\nBeBcALNFZB+AqwA0AICq3gTgIQDvAdACYADAp6ozUiKi+sPAn4gizy71GLIDfw3L+FuTe+OClFcG\nFI/xBGkhVPXiMW5XAJ+boOG4zzmRT0ZEdBTjNxkRRZ69Mq9d4x/ax98JJvfGY4LhlBv4J1jjX6P4\nvhERGQz8iSjy7Br/gZDJvWGlPjERxEUw7Gf8GUDWKk7uJSJyMfAnosiza/ztUh8nJCJUq6tPPBZD\nMu0F/qzxr0l824iIAgz8iSjydIzJvbaM1dXHLu9hxp+IiGodA38iijw7vB878Le6+ljBPmv8axlr\nfYiIAAb+RFQHchfwMj35wwJ/J6fG34ixZqQm8V0jIgow8CeiyLPj++G0g4a4+9GXCUkE53b1MbiA\nFxER1ToG/kQUeZozibcx4X70OWGlPlbG3w72WeNfu9jVh4jIxcCfiCLPcYAGq0bfZPzTjjNiX7ur\nT4w1/jWPFVpERAEG/kQUeY4qGuPBx12jF8SHxP3ZXX1irPGPAib8iYhcDPyJKPIcDcp7AKAhYWr8\n83f1ieV29Ynx47IWCaf3EhH5xvwmE5FbRKRdRDZb2+4RkQ3ev90issHbvlBEBq3bbqrk4ImICqGq\naErE/euNfqlP/q4+ccme3MsafyIiqnWJAva5FcB1AG43G1T1I+ayiFwDoNvaf6eqnlmuARIRlcpR\nzc74x0dO7s04inhMsrr6cHJvNORO7iYiqldjZvxVdQWAw2G3iYgA+DCAu8o8LiKisnE0O5D3S32s\nwD/lFfeb8h8RQdwq72E7z9rEqRlERIFSi1bfDqBNVXdY2072ynyeEpG3l/j4REQlc1QhEnTmaTIZ\nfx0Z+DvM+EcO8/1ERK5SA/+LkZ3tPwBggVfq868AfiUiM8LuKCKXicg6EVnX0dFR4jCIiPJTdbvy\nmBKfxpCM/7t++BQe2XzAX+wrLjntPBn41yS+a0REgaIDfxFJAPgQgHvMNlUdVtVO7/J6ADsBvDbs\n/qq6RFUXq+riOXPmFDsMIqIxOaqISVDbb3r624F/W88wLv/l81ldfbLaeTLwJyKiGldKxv9dAF5S\n1X1mg4jMEZG4d/kUAIsA7CptiEREpXEDf/EDfnMAMFo7z3hMctp5MvCvVZzbS0TkKqSd510AVgN4\nnYjsE5FLvZsuwshJvecA2Oi19/wNgMtVNXRiMBHRRHHUnaxrevGHTe41mlvdJmVs5xkNwtm9RES+\nMdt5qurFebZ/MmTbfQDuK31YRETlo6oQBLX9TSHtPI1VOzsBuKU9DPyjge08iYhcXIqSiCLP0eya\n/dFKfboHkgDcycDs6kNERFHCwJ+IIk/9Gn/3Iy8eF4iEl/oMp922niO7+vDjkoiIalshK/cSEdU0\nU+PfEHevN8QEMZERgX9jPIakF/jndvVhxr92sdCHiMjFwJ+IIs+084x5WftEPAbByIx/Ii5+4J/b\n1YeBf23i3F4iogDPXRNR5AULeLlRYCJPxj8ek6xSH7u8h+08axhT/kREABj4E1EdyF3AK2Fq/HMm\n9zbEY0hmTKmPIG59QjLjX5uEa/cSEfkY+BNR5DmqXo2/F/jHYhABHCfYpykRy16pVwRxK+PPwJ+I\niGodA38iijxHgZjY7Ty9Uh8r4z+1KeEfGACm1Ic1/lHASh8iIhcn9xJR5KkqYrGYv2JvPDZycu+0\npuyPw1jMzfobrPGvTZzcS0QUYMafiCLPMZN7czL+tllTGpCIZ2f4c69TbeLKvURELgb+RBR5bo2/\nNbk3JlmZ4A8vno/rP/pnaLBq+t0afwb+tY7vGhFRgKU+RBR5JuOf8FfujUGsyP+b7/9TTG6Mjwj0\n48LAn4iIooMZfyKKPPXaeTZ6pTvuyr3B7Sa+b7BLe3Iy/nZPf6otLPQhInLxm4yIIs/t4x9k/BPx\nWFaNv7mYsLr6iCCrxp8J/9rEyb1ERAEG/kQUeY6DrD7+DfHsGn9zEGA698TE3d9k/OMxySoNIiIi\nqkUM/Iko8oKVe8MDeXMpYd0OwK/xZ31/bWNTHyIiFwN/Ioo8Va90J2at3GvdHmT8Y6HX2cO/dvFM\nDRFRgIE/EUWewq3xb0iE9/HPndzrZ/zj2Zl/qk3K6b1ERAAY+BNRHQgW8LIn9wa3S06G3y/xkewD\nACqMiFwgIttEpEVErgi5faaI/F5EXhSRZhH5VMXGUqkHJiKqQQz8iSjyggW8gkm8Jti3DwBMgG8S\n/Cbzz1KfwolIHMD1AN4N4DQAF4vIaTm7fQ7AFlV9I4BzAVwjIo0TOlAiojrEwJ+IIk9zFvCyV+61\nS34aYtmlPomc61SQswG0qOouVU0CuBvAhTn7KIDp4h59TQNwGEC6UgPi5F4iIhcDfyKKPMdfwGtk\nH3+7fN9f2de09Yyxxr8I8wDsta7v87bZrgPwJwBaAWwC8EVVdSoyGr51REQ+Bv5EFHnBAl52qY97\nm931Jejjn72NNf5ldz6ADQBeDeBMANeJyIzcnUTkMhFZJyLrOjo6in4yJvyJiFxjBv4icouItIvI\nZmvbN0Rkv4hs8P69x7rtq96Erm0icn6lBk5EVKjcBbwSVlcfO6TP7eNvLptJv1SQ/QBOtK7P97bZ\nPgXgt+pqAfAygNfnPpCqLlHVxaq6eM6cOUUNRpjyJyLyFfJtdiuAC0K2/0hVz/T+PQQA3gSuiwCc\n7t3nBm+iFxFR1ahX6vOG+TPx56cch5OOm+qHg9nZ/VjINgFL/MdlLYBFInKyN2H3IgBLc/Z5BcB5\nACAicwG8DsCuCR0lEVEdGjPwV9UVcCdeFeJCAHer6rCqvgygBe5ELyKiqjHtPE86biruuuytmNaU\nsCb3BvuZrj92gj/GjP+4qGoawOcBPApgK4B7VbVZRC4Xkcu93b4N4C9EZBOAJwB8RVUPVW5QFXtk\nIqKakijhvl8QkY8DWAfgy6raBXcC1xprn7BJXUREE8pRRW7sHkzutct6svv4A16NP1P+4+KdBX4o\nZ9tN1uVWAH8zEWPhvGwiokCxaawbAZwCd1LWAQDXjPcByjVpi4hoLI5mB/gArMm9wbYg4599MJDg\n5N6axpV7iYhcRQX+qtqmqhmv/drPEJTzFDKpyzxGyZO2iIgKYWr8bTEJ6+AzMuMfj2XvQ7WF7xwR\nUaCowF9ETrCufhCA6fizFMBFItIkIicDWATgudKGSERUGtPO0yahffxHdvVJxGJcuZeIiCJhzBp/\nEbkL7pLqs0VkH4CrAJwrImfCnTK1G8BnAcCbwHUvgC1wV2H8nKpmKjN0IqLCmMm9tvCuPiPPAsRZ\n41/zuHIvEZFrzMBfVS8O2XzzKPtfDeDqUgZFRFROjuqISZ5msm92H//slXvNZQb+tYtVWkREAfao\nI6LIUx25kJO5bk/69Sf3Wrsy8K99TPgTEbkY+BNR5IVP7s3+CQSZ/lgsu/yHNf61iyv3EhEFGPgT\nUeQ5mh3MA+GTextCuvrMnTEJc2dMqvwgiYiIKqyUBbyIiGpCWI1/sHKvld0P6eP/00vezDrxGqec\n3UtEBICBPxHVAQ3p6uOv3Gtt8yf3WvtOaohXfHxUOTxoIyIKsNSHiCLPGaXGX0LaeXIyb7Qw309E\n5GLgT0SRF7qAF0xZT7DNBP7MEkcH30oiogADfyKKPEezM/tAENzbXV8aQvr4ExERRQUDfyKKNDOx\nMzeWDyb3BttMwB9nyj9SOLeXiMjFwJ+IIs3xgr58k3vH6upDNY4HcUREPgb+RBRpTp6Mfyyo9fE1\nhHT1ISIiigoG/kQUaSbwz1fjH2NXHyIiqhMM/Iko0jRPqY+E9fH3Wvyw1Cc6+E4SEQUY+BNRpOUv\n9TE/Q2r8GS1GDlfvJSJi4E9EEZdvcq+5Jlk1/uzqEzV8K4mIAgz8iSjSghr/7O3mQCB75V6W+hAR\nUXQx8CeiSFPH/Zl/cm+wjX38o4uVPkREDPyJKOLy1fhLSB9/086TGf/oEE7vJSLyMfAnokgzid6R\nC3i5P+3NZnJvnJ+MkcOEPxERA38iiri8GX+E1fiPPAtAtY1vJRFRgIE/EUVavgW8vHm82X38TakP\no0UiIoogBv5EFGljLeBlnwngyr3RxT7+REQM/Iko4vKX+sDbPnJyLwP/6OA7SUQUGDPwF5FbRKRd\nRDZb274vIi+JyEYR+Z2IzPK2LxSRQRHZ4P27qZKDJyIaS74FvII+/va28H2p9jHfT0RUWMb/VgAX\n5GxbBuBPVfUNALYD+Kp1205VPdP7d3l5hklEVBzHCV/AS/yuPmJtE/zJCTNwypypEzU8qjAewxER\nBRJj7aCqK0RkYc62x6yrawD8fXmHRURUHvlq/GMhNf4A8PAX3z4RwyIiIppw5ajx/zSAh63rJ3tl\nPk+JCL9Biaiq/Br/nE87P+PPKvC6wLm9REQFZPxHIyJfB5AGcKe36QCABaraKSJvBnC/iJyuqj0h\n970MwGUAsGDBglKGQUSUVzC5N6erjxfw5x4QULTktnElIqpnRX/licgnAbwPwMfU65OmqsOq2uld\nXg9gJ4DXht1fVZeo6mJVXTxnzpxih0FENCozuXdEH39m/OuKcnovEVFxgb+IXADg3wG8X1UHrO1z\nRCTuXT4FwCIAu8oxUCKiYmiedp5hXX2IiIiibMxSHxG5C8C5AGaLyD4AV8Ht4tMEYJmXRVvjdfA5\nB8C3RCQFwAFwuaoertDYiYjGlK+dp7B1JxER1ZlCuvpcHLL55jz73gfgvlIHRURULnkX8GLGv65w\nci8REVfuJaKIM4F/bo0/M/71gW8vEVGAgT8RRZrJ9ObGf8HkXio3EblARLaJSIuIXJFnn3O91s/N\nIvLURI+RiKgeldTOk4joaDfWAl5s91heXoOH6wH8NYB9ANaKyFJV3WLtMwvADQAuUNVXROT4io2H\nh3ZERD5m/Iko0vIu4OX9zK39p5KdDaBFVXepahLA3QAuzNnnowB+q6qvAICqtk/wGImI6hIDfyKK\ntPw1/pzcWyHzAOy1ru/zttleC+AYEVkuIutF5OOVHhQn9xIRsdSHiCIuXztPc52Te6siAeDNAM4D\nMBnAahFZo6rb7Z3KscI7314iogAz/kQUafkW8GJXn4rZD+BE6/p8b5ttH4BHVbVfVQ8BWAHgjbkP\nVM4V3rlyLxERA38iirj8GX/vAuP+clsLYJGInCwijQAuArA0Z58HALxNRBIiMgXAWwBsrcRg+PYS\nEQVY6kNEkbFsSxvaeobwf956kr8tqPHP3pelPpWhqmkR+TyARwHEAdyiqs0icrl3+02qulVEHgGw\nEe4q7z9X1c3VGzURUX1g4E9EkXHbM7vxyuGB0MB/RIDPPv4Vo6oPAXgoZ9tNOde/D+D7EzemiXom\nIqKjF0t9iCgyWo8MonswlbVtrD7+bOcZbTyhQ0QUYMafiCJBVdHaPYjhtAPHUcS8iN7JN7nX+8lS\nHyIiqhfM+BNRJHQNpDCUcqAK9A6l/e1mcm9uH38/4GfcXxdY6UNExMCfiCKi9cigf9ku98mX8Y+x\nnWddEB7ZERH5GPgTUSTkC/w17+Re1vjXE+XsXiIiBv5EFA124H9kMOlfdhz3Z74+/swIRxtP6BAR\nBRj4E1EktHYP+ZfDSn3y9vHnpyAREdUJfuURUSS0HhnE1MY4gNzA3/2Zp40/OLu3PrDQh4iIgT8R\nRUT3YAonHjsFAHBkYOwaf9PukzX+RERULxj4E1EkOKqY2pRAYyKGHntyr/czX/cedvWpD5zbS0TE\nwJ+IIiLjKOIimDW5ocB2nu4Gxv3Rlrt+AxFRPePKvUQUCY7jTtSdObkBRwZSeHJbOx5rPog/f81s\nAGELeJmfDAyJiKg+MONPRJGQUUU8JpjpZfxX7TiE+57fb9X4Z+/vL9zLuL8+sNSHiGjswF9EbhGR\ndhHZbG07VkSWicgO7+cx1m1fFZEWEdkmIudXauBERLaMo4iJYNYUN/B3FHActUp9cjP+XqkPu/pE\nGt9dIqJAIRn/WwFckLPtCgBPqOoiAE941yEipwG4CMDp3n1uEJF42UZLRJSH42X8p09qQM9QCo4q\nMqp5F/ASrtxbV5QpfyKisQN/VV0B4HDO5gsB3OZdvg3AB6ztd6vqsKq+DKAFwNllGisRUV5mcm8i\nJsg4iozeAwfVAAAgAElEQVSjUHW3A/n7+LPUJ9r4/hIRBYqt8Z+rqge8ywcBzPUuzwOw19pvn7eN\niKiiMo4iFhPEvcDflPikvJR/LMbJvUREVN9Kntyr7sy5cZ9DFZHLRGSdiKzr6OgodRhEVOccdTP+\nIuLW93uBfzqTb3KvZP2kaGMffyKi4gP/NhE5AQC8n+3e9v0ATrT2m+9tG0FVl6jqYlVdPGfOnCKH\nQUTkcjP+QDzmBv2mxCeV8TL+edp5Mu6PNr69RESBYgP/pQA+4V3+BIAHrO0XiUiTiJwMYBGA50ob\nIhHR2FTd4D4uptTH3Z7K5Knx5+TeusKEPxFRAQt4ichdAM4FMFtE9gG4CsB3AdwrIpcC2APgwwCg\nqs0ici+ALQDSAD6nqpkKjZ2IyGf6+MdiAkcVjmNKfcIz/n4ff+aEI42lXEREgTEDf1W9OM9N5+XZ\n/2oAV5cyKCKi8TJdfWIicBy3lScApJzR+/gz409ERPWCK/cSUSQ4dlef0Br/7P2DGn9G/vVAObuX\niIiBPxFFQ0btjH/QxcWU+uSW9JjrjPujje8vEVGAgT8RRULGgZfxz+7qkzYLeOV82gn7+NcV5vuJ\niBj4E1FEOKqIx+B29dGgxj/o4587uZc1/vWAby8RUYCBPxFFgpncKyJQhd/VJ5lmjT8RERHAwJ+I\nIsKe3AsE3XySXo1/Ipb9cRcT1vjXE87tJSJi4E9EEWEm9/qBv5fpNxn/RIx9/OsSj+yIiHwM/Iko\nEjKOt4CXF+ilHTfgH05nIOJO/LWxxp+IiOoNA38iigRH3VIfE8gnvUm9w2lnRLYfCAJ+dvWpD8q+\nPkREDPyJKBoyjiIm8Et9TP/+ZNrxt9nYx78+8O0lIgow8CeiSHAU/gJeQLBibzLjjJjYC7CrT91h\nwp+IiIE/EdU+07rT7upj+vcPp/Jk/P1Sn4kZI1UHj+uIiAIM/Imo5pnFutyMv7st5dgZ/7DA3yv1\nmZghEhERVR0DfyKqeRkr4x/z23kGC3iFZfxNSVButx+KJlb6EBEx8CeiCHBMxj8miIe08wzN+Of8\npGjiOg1ERAEG/kRU80zGPy5Wxj9jZfzjIRl/79OPk3vrA1fuJSJi4E9EEeAl993JvbldfdLhXX2C\nBbwY+JebiFwgIttEpEVErhhlv7NEJC0if1+5sVTqkYmIag8DfyKqOY9sPohdHX3+9WByb5DJt9t5\nhvfx934yMCwrEYkDuB7AuwGcBuBiETktz37/A+CxiR0hEVH9YuBPRDWldyiFy3+5Hp++da2/zS/1\nidl9/NX/Gb5yr8n4V3rEdedsAC2quktVkwDuBnBhyH5fAHAfgPaJGBRX7iUiYuBPRDVm9c5OAEBj\nIvj4MpN77T7+ttH6+LPGv+zmAdhrXd/nbfOJyDwAHwRwY6UHw3eXiCjAwJ+IasrTLYcAAG+YP8vf\nZk/ujYcE8qNl/BkYVsWPAXxFVZ3RdhKRy0RknYis6+joKOkJObmXiAhIVHsARETjYQJ/O8C3+/iH\nZfBHX7mXoX+Z7QdwonV9vrfNthjA3d57NRvAe0Qkrar32zup6hIASwBg8eLFRYXufHuJiAIM/Imo\nprzSOQDAndDb0t6LqU2JoNRHwkt9Qrv6eLl+BoZltxbAIhE5GW7AfxGAj9o7qOrJ5rKI3ArgD7lB\nPxERlV/Rgb+IvA7APdamUwD8J4BZAP4RgDkv+zVVfajoERIReVQVaS+7n3EUX7hrA047YQY+/85T\nAQDxmPsvV/jKveYnI/9yUtW0iHwewKMA4gBuUdVmEbncu/2mqoyrGk9KRHSUKTrwV9VtAM4E/LZs\n+wH8DsCnAPxIVX9QlhESEXkcK3rLOIr+4TQGkumg1EckNJBPhC7gxYx/pXjJnodytoUG/Kr6yUqO\nhSv3EhEFyjW59zwAO1V1T5kej4hohLQTzAXNqCLjuP9MqY/dztM2eh9/Bob1QDm7l4iobIH/RQDu\nsq5/QUQ2isgtInJMmZ6DiOqcFffD8QJ+E/wDXlef0Br/sMm97ONfF/j+EhH5Sg78RaQRwPsB/Nrb\ndCPcev8zARwAcE2e+5WtTRsR1YesjL/j1vubzD/glu8UnPE3ffwZGRIRUZ0oR8b/3QCeV9U2AFDV\nNlXNeP2ZfwZ3FccRVHWJqi5W1cVz5swpwzCIKOoyVpG/owont9RnHF19uHJvfWGlDxFReQL/i2GV\n+YjICdZtHwSwuQzPQUTkd/QB3IOATG6pT0xCA/nRuvqwxj/a+O4SEQVK6uMvIlMB/DWAz1qbvyci\nZ8LtnrY75zYioqI5VuCfdhSZjFvu4/fxj4nfrccWWuPPPv5ERFRnSgr8VbUfwHE52y4paURERHmk\nc0p9Mn65j7stLpK1oq8RlvE/ec5UnPf64/GG+TMrNl6qPp7RISIKcOVeIqoZmdxSHzPB15/cGx7k\nh/Xxn9aUwM2fPKtygyUiIjrKlKudJxFRxWVl/B34E3vtyb2FdvWh+sLJvUREDPyJqIZkchfwUkU6\nkzO5N+RTLayrD9UHHvIREQX4bUhENSNjLeCVdhSqQa0/4NZzF1rjT0REVC0fvGEVfrN+34Q/LwN/\nIqoZ9gJeybTjbVOo2hn/wrr6UH1RsNaHiI4eL7xyBP/26xcn/HkZ+BNRzTAlPSJAykv/F9vVh+oD\nm/oQEQUY+BNRWX361rV4cOOBijy2mdzbGI9lZfztrj5hk3uZ8SdO7iUiYuBPRGX21PYObNjbVZHH\nNgt4NSZifsY/Yy3glW9yb5yTe+sWM/5ERAF+GxJR2ai62fdUpjLpVZPxb8oJ/P2uPiIF9/EnIiKq\nNwz8iahsTABuT8KtxOM3xGMYtkp9TMY/FmONP4VjpQ8REQN/Iiojk5FPVzjjb5f6OJqd8RfW+JNF\n2MmfiMjHwJ+IysYE5pUq9XGsUh9/cm/GyVrAKyy7z4w/KWf3EhEx8Cei8slkTOBfmVIfO+PvXYTj\nLeIF5C/1Yca/fpn/DhMZ9n/mtrV47X88PIHPSERUmES1B0BE0WFq+ytX4+8+bkM8yFmkHSerjz+7\n+lCYiUz4P761feKejIhoHPhtSERlk6lwqY/dx99wHCCjQR//0K4+zPgTEREx8Cei8gkm91a2q09j\nIjvj71iTe8MW8GKNf/0KJnuzxp+IiIE/EZWN6eZjDgDKLWNN7jUcDbbH8gT+7ONfv/ywn3E/ERED\nfyIqH1PbX+nJvXaNv/18MXb1oRzVmNxLRHS0YuBPRGVT6Rr/sIy/+3xu4B+PCcJifNb41y/Tx58Z\nfyIiBv5EVEaVrvFPh9T4A0Ayk72AV261D7v61K8g48/In4iI34ZEVDbpTGUz/k6+wD9tSn3c67m9\n/Jnxr19854mIAgz8iahsytHHP5VxcO+6vX6Qn/34o9f4m4A/lhPos8afWOpDRMTAn4jKKOOX+hQf\nZa3e2Yl//81GbNzfHfL4boCfL+NvAnxzAGAS/8z41y+/1IeBPxERA38iKh+TkU+WUOM/mMoAAIa8\nn2GP3xSS8RcJerabON8s9MWMfz3zJveyxp+ICIlS7iwiuwH0AsgASKvqYhE5FsA9ABYC2A3gw6ra\nVdowiagW+H38S8j4m7KdTEipTyaTv8bfrus3pT6N8RiG0w77+NcxZvyJiALlyPi/Q1XPVNXF3vUr\nADyhqosAPOFdJ6I6UI4af1O2E7YWQEbzdfVxsur6TYa/IWEy/jy5Wa94yEdEFKjEt+GFAG7zLt8G\n4AMVeA4iOgqVo4//qBl/RxGTkYF8bsbfXG7wMv2s8a9fpvyLGX8iotIDfwXwuIisF5HLvG1zVfWA\nd/kggLlhdxSRy0RknYis6+joKHEYRHQ0KEcf/yDjH97VJxGLIbdyJ5Vxsur4xQ/8WeNf7/jOExEF\nSqrxB/A2Vd0vIscDWCYiL9k3qqqKSGieRVWXAFgCAIsXL2YuhigCytHH3yzGFVYulHEU8ZiMCOST\nGSdrxV4z99dM7mXGnzi5l4ioxIy/qu73frYD+B2AswG0icgJAOD9bC91kERUG0ywnipDjX++Up94\nTEb06U+lNbvGnxl/8nByLxFRoOjAX0Smish0cxnA3wDYDGApgE94u30CwAOlDpKIaoMJ1lXDA/dC\nmBr/sLMGfsZfRmb8w7r6NCRMjT8n99YrP/Cv7jCIqMI6eoerPYRRqSo+e8c6PLPzUFXHUcq34VwA\nT4vIiwCeA/Cgqj4C4LsA/lpEdgB4l3ediOpA2gr2w7ryFMJk/MPmCaQdB4mwjH+erj4m4I+zneeE\nEpELRGSbiLSIyIjObiLyMRHZKCKbROQZEXljxcZi+vgz5U8UWWt2deKsqx/HQ5sOjL1zlfQnM3i0\nuQ3/eNu6qo6j6Bp/Vd0FYMSHtap2AjivlEERUW2ys/zpEjP+Yfc3Gf/cmv0Rffwl6OMPsMZ/IolI\nHMD1AP4awD4Aa0VkqapusXZ7GcBfqWqXiLwb7nyvt1RmQO4Phv1E0bXZW+l93e4uvOeME6o8mqMb\nz38TUdnYWfpUuriM//AoGf98k3tzu/r4K/cmWONfBWcDaFHVXaqaBHA33DbPPlV9xlrYcQ2A+ZUa\njHnnmfAnomIl0w6+/rtN6OwrvZyo2h9FDPyJqGyySn2KnOA7WsY/bSb35tb4px3YZfx+qQ/7+FfD\nPAB7rev7vG35XArg4UoNRoTvPRGV5sFNrbjz2Vdw9YNbi36Mo+WTqNR2nkREvqxSnyJbevo1/nlK\nfRJ52nlOSwQfZzF29akJIvIOuIH/2/LcfhmAywBgwYIFJT5btfNsRFSrzBnDKHyKMONPRGWTLkPg\n72f8Qyf3jpbxD6nxT5gaf37UTaD9AE60rs/3tmURkTcA+DmAC725YSOo6hJVXayqi+fMmVPUYFjq\nQxQNb/rWY/jYz9dUexg1j9+GRFQ2WTX+RZb6JEeb3JvJv4CXPbnX3N7g/WTGf0KtBbBIRE4WkUYA\nF8Ft8+wTkQUAfgvgElXdXsnBsJ0nUTR0DaSwqiU0R0DjwFIfohry0KYD2N81iH8855RqDyVUOTL+\nybTmvX9GFfFYzF+Z10h5BwSG38efpT4TTlXTIvJ5AI8CiAO4RVWbReRy7/abAPwngOMA3ODV4KdV\ndXElxhO086zEoxNRPYlCW2AG/kQ1ZOmGVmxr6z1qA/9MOfr4j9HO063xj43Ybpf/mLb97z7jVXjV\nzEkM/CeYqj4E4KGcbTdZlz8D4DMTMZZg5d7a/8ImotpX7Y8ilvoQ1ZBkxvEnvx6NyrGAV2rUBbzC\nV+4FkNPO0718xrxZ+PLfvK6ocVA0+DX+VR0FEUXBeLqEbWntwR2rd1v3Lf94isHAn6iGJNOOnxE/\nGtnBerELeI2e8Tcr9468Xyyk1IeZfjpqeuhRZFxy87N4y389Xu1hRM7a3YfRM5Sq9jDK5j3XrsSV\nDzRXexgjMPAnqiHJdB1k/P3A38na9tGfrcEzOzsRy5Pxt2N8czvjfjKqfXqdomPljkNo6yl9IScK\nDCYz+IebVuMzt62r9lAijzX+RDUkmXGKDqgnQln7+Fv37+xL4pmdbjeHsD7+AEK7+sQY+dc9f3Iv\ni32IjlqmC9zW1p4qj2R0pcwVuv+F1jKOpHjM+BPVkEpm/Nt7hrCva6CkxyhHxj+s1Mc+/RuPSWhA\nH1rqc7QUVVLVCIv8iWrG0fpnWo6vkq/9blPpD1IGDPyJakgy4yDtKJwi6+dHc9XSZvzrvS+W9BhZ\nffxLzvgHj9UzGAT+iTylPomsyb3mJwP/ese4P7qefKkdG/YeqfYwJswPHt2GNbsmvo/92t2H0T1Q\n2dr78XxSf+Snqys2jnzKWSpY7bOPDPyJaojJoldigm9nXxJHBpIlPUZWH/8iF/AyrzE1SsY/rNRn\nSmNQuejX+PMTru6ZLhys8Y+eT926Fh+4flW1hzFhrnuyBRctmdiVa1MZB/9w02p8/BfPTcjzFVJK\n8+zLhydgJOHG09XnaMWvRaIaYrLhlajzH0ilMZTK/7j7ugbwb79+cdRSo3LW+Ges+/cMpv3L+QL/\naU1x/zJLfcgIVu5l5E80Xo4XiB/ttfeGo4rvPfISDveXlsSKMgb+RDXEBMWVqPMfSGYwlMrkvf2Z\nlk78Zv0+vHK4P+8+aUfRlHA/Vorv6uOt3GudMbAz/olYzA/8GxPBR9iUppCMPwP/usf/AUSlq5UD\n56e2d+CG5Ttx5f2bK/L4UVgIMHKBP4/yKMpMiU+x9fOjGUxmMDzKAcVw2j0oGO2sQCajmNTgZt5L\nrfFPZWX8cyb3egF9Uzz4CJvaaGf8zU+GfeSKwPc10YTzu2JV+O+nXCU0JmE02ndZMaKUQ4pU4P9M\nyyGcffXjONA9WO2hEFVENTP+JuAfbZ+042BSQ8y/PF6q6h/cZLJq/MNLfbIy/laNf0zCy4Go/gSl\nPlQp512zHO/7ycpqD4MioNC/03yZdx7gjy1Sgf/Ojj6kHUU7F9agCLKD4kpM7jUZ/3wfqCbgHy3j\nn3YUk0vI+Nv3SeXp6hO3uvo02Bl/q8bfPSsw7qenSDIZS0YElbKzox+b99dGDTgdncr9cT3eDH1H\n7zBe+x8P5+0SNdbHx6qWQ1h4xYMFNchQBQ71VS9OjVTg3+mV+QyOkpEkqlUZR/0Pn3Jn/NMZxz+Y\nyHeK1GwfLeOfcYJSH7sd5/Jt7Vi+rX3McdjBvp3x77Uy/jEJSnnsjP/UnBp/1vcTwIw/USkm+mPU\nfMcNpzM475rlWLmjY9T9AODIQBK/XLOnqOdr6xnCpbeuxWNbDiKZdnDz0y+Puv/9G8IX4bpx+U4A\nwF3P7cUDG/aP+byLv/P4+AdbJpEK/LtM4J9k4F8JHb3D2H+EZVTVkszqkV/ewN8+WB7Ok9H3M/7p\nUUp9rBp/u7Xn/z6xAz96fAc6eofxSmf+RcLsA5p87TyHUg4SXuSfiAffSlOtUh9hqQ95/P8FjPyJ\nxs0OsFUVD2zYPyGrx+/rGsTOjn5c9UDzmPt++d4Xsdv7Xhnvib0fP74DT7zUjqVeQJ/vzGChB0D/\n88hL+OLdG8Y3iAkWqcCfGf/K+ubvm/GFXz1f7WHULTsoLnepj32wnC+wHypgcq9d42+Pt2cwhfae\nIXzrD1vwsZvX5P1wtb9Q8i3gNZDM+Bl/u13nlEa71Icdfcjl9/Fn5E9Ukoc3H8QX796A659smbDn\nzPdXa28/HFJeU+5P/3+5Z+Tilo6j+PrvNmHrgdoqc4tU4H+YGf+K6uxL+gdXNPGysuHW5WVb2nDO\n9570u+4UYyA5dsZ/uKDJvYqGuNtu057c2zuURnvvMHZ19GHv4UHsOhTeEtQuM8qd3NvgZfcHU2k/\n4Lez+lmlPqzxJw//G1ClrN1dvYWkJpoiiLHaKjCPMjdP46+4XeG5OeZ5i1kUrLV7EHc++wouvXVt\nyeNIZZysM9uVVHTgLyInisiTIrJFRJpF5Ive9m+IyH4R2eD9e0/5hjs6859ygBn/ihhIZbICRJpY\ndpbfvrztYA9eOTxQUivbgYIy/oXV+CdigkRMshbw6htOI+Motrf1AgCe3nEo9P6pPOVMPYMpnDBz\nsj9WE/Dbgb+d8WdXH8rFub2V11VniaHnqriCbDVMxEnUQs/MjfeAYNvBXtywvLAzFc/u6sRQKoOv\n/GYj3vztZXn3C84mlu7/3vUC3vCNx8rwSGMrJeOfBvBlVT0NwFsBfE5ETvNu+5Gqnun9e6jkURbI\nZKOHGJxWxGAyjYHh9Ng7UkVklfpYl/u9/+9HBorPFgymgvc1N7DvHkjh8S1t/vbR+iOnM4p4LIaG\neCxYiCvj+AcWZttKL/A/1DeMp7YHk7fMAU1jPOZn/FUVPUMpzD/GDfwHk5lgZV4ruJ9mZfzfe8YJ\n+MzbTyn49VN0+ZN7GfhX3DbvwJ6iww7E8/0Nrd9zuORyF6nwubn3X/c0vvfINgylMnho0wH05smu\nt7T34SNL1uAHj27DPev2jlrlUOyIw36ND28+WOSjjV/Rgb+qHlDV573LvQC2AphXroEV4nB/Epv3\nd5vxBJN7mfGviIFkBgOpDNviVUn25N7gPTClbd2DxQf+WaU+OYH9nc/twWduX4dOr/3YWH38EzHB\n5MY4+r2DxL6cg8XGRAzNre7f7a+efQWfvnWtn91Ppd3XNbkx7r/GoZSDVEYxb5aV8Q8p9bFX7v2L\nU2fjc+84tdCXTxHmL0BU5XEQ1bLRvvf/7sbVePf/lraOQ26mf6xs+lh/z7lnJ8z32lv/+wn8853P\n499+7dbs5wbvps3mJi+2HM3uTrdktZCQ6OFNB/LetvTF8E5BlVKWGn8RWQjgTQCe9TZ9QUQ2isgt\nInJMOZ4jzM1P78KF16+C4yh6htJ+FxGWo1TGYDID1dEnd1LlmKAYAJKZ4P+4CbDLFfjnBvb7utxO\nTof6kqG329KOIhEXnDBzEg70DAHIbsUJAKfMnuqPuWcwhYyj/j7m4GZKY9yfI2DqHl89K6TUx/p0\nt1fuJTLMf5EXXumq7kAmwOV3rMcHrl9V7WFQhf33w1uPmgSc3env5yt3Ff04z7R0jmt/O8GbHsea\nMebMuPley2WmlhXSHOKjP3vWu8/oz6+q+Kc78zdGeTpPy9JKKTnwF5FpAO4D8CVV7QFwI4BTAJwJ\n4ACAa/Lc7zIRWSci6zo6invRc2dMQsZRdPYns+qbRwtMqHgmOBxIstynGuxg3z4IMHNauksp9ckK\n/LMP7Fq9D/Yg4+/4z/d3Nz6D3dZEXVPj/+qZk/375U5YWjR3OgaS7pkjU6ZkTruaEqbJjXG/1Mcc\nFLxq5iQA7kJdJvCPZdX4Bxl/olw3eH22o+yR5oN5FyCi6PjpU7vQMcELQNmhrR0Tf+xna/zL33lw\na9GP/5nb17nPU2AMf/szu/3LLe19436+fM9jvndiJUbH53zvSf+yk/NcuevwTPQxXEkvTUQa4Ab9\nd6rqbwFAVdtUNaOqDoCfATg77L6qukRVF6vq4jlz5hT1/MdPdwOBtp4hHO4P/gjyBaa3PbMbew/n\n7yFO+TmO+kfYhZxRSWUc/OSJHTxIKCO7BGfYKvsZKHPGP7c7kAngTZBuDqy3tfVi/Z4uPG9lUk2N\n/wmzJqH1yCBUFX1Wxl/EzfinHXcV4kHv/4cJ7lNWxt+U+phSoVfNmISrP/inuOPTb/Ez/Qkr8LcX\n8yIy2NWVjiZXPbAZ978w9gJPY6l0TbxhglI7OB1KZeB40WxnX2UmdAddfcJvt9eJKaVVb+7ngzmT\nsu1g74ht+SiAlw5mz295xYo1xzojMNFK6eojAG4GsFVVf2htP8Ha7YMANhc/vNHNndEEAGjvHcr6\nzzcYUorSM5TCVUub8dvnS/+Dq0d20FlI4P/8ni5cs2x73u4tNH52XX8qZHJvKYG/fdo0N+N/4MhQ\n1nXT3afL653cZZ1pMBn/ebMmYyCZQc9g2g/qYwLMntaEY6Y0uOMezlgZf6/Ux3vsKQ0JP/NiDhym\nNiXwsbechAXHTQmd3EtE9amlvXYmFd+2eg++dM/RvcBTPiYO+O3z+/H1+ysT2hUaImcvLGZfLrAr\n0BiPe8iKKcdaFVhV/bkBYcYK/O2Dj4ko4SolRfaXAC4B8M6c1p3fE5FNIrIRwDsA/Es5Bhpm7gw3\n43+wexhLX2yFCHD89KbQPv5m4u9E9UmNGjtz319AFt8Ehbn13VS8fAt4mf/vRwaLz7wMJsO7+vQM\npdCbMznX3H7Ee4+PWIunpB0Hibj49fj7jwyid9j9mztj/iy8/lXT/X77/cNpf+ym1Md8sUxpivvZ\n/z7v/nbXHsD06mfgT6ObqMxomHprb1ktv16/r9pDqDkb9x0ZdRX1MN/+wxb/8l3PvVLuIWUxrZ9N\n5nxXR3Y5T74AubV7KHR7ocKC9CtzVg/+xC3PZcWZY8Xq3/r9ltF3yHr+gnctWildfZ5WVVHVN9it\nO1X1ElU9w9v+flXNP5W5RHOmuxn/X6x6GX/YeAD/7/zX4cRjp2BbWw/e/O1leNmqPTYTOnpKyIrW\nMzvLn2+BtBde6cJZVz+Orv6knwXO1zKLxi/fAl7mQKx7sPiDrHxdfXKz/UAQ+Jv3uCsr8Hcz/id4\n9fgHugf9g7/rP/omLLlksR/4DyQz/tjNPub/y7FTGv1TuX3D7vNNn5QT+ItklfoQhanmsaE98XEi\nORMRPRxN6uzllsP7r1uFc77/5Ng75jHa39X+I4N41w+fwoHuQSxZsbOoOGCHVbfvOIp3XvNU1u3X\n/rEF7T35g/yxDvjNgUPufoX86Ty1vQM/XLbNvz7WwqZ3Pjv6QdL9LwRdfY72jH/VNcRjmD2tETva\n+zB9UgKXn/MaTG6IY+/hQXT2J7NqtExwwox/cexSkP48vfw37utGR+8w9nYN+L/v3FaOVLxUngW8\nytXOs8mrkbcz/q0hgYtZwfeIH/hbpT5ejb9pvdl6JAj8Z09rwuTGuL/QVn9yZMbf/H0eO7URGUe9\nOQLhGf9YLHtyL1GYejwptL4OOhhROFXFI5sPZH1fFKqtZ6gsgeevnt2DlvY+XLRkDf7roZf8MwUt\n7b345C+eK+hAwB7HKV8LXw7qmse243B/Mu9x389X7sqag1aIQuvxf7by5XE97mjs7/OJOIat6cAf\nCCb4vmH+TMRigkkNQUs/uwQhyPgzEC1GVsY/T9ckU+PWM5j2O8yUWupzuD+J5dvaS3qMKNh6oAcv\n7gu6ddgfFH47z4FSSn0ymDnZrb23M/5hGUuzsq/5+7K7CZl2nrOnNaEhLth/ZAi9Q2k0xmP+36af\n8R8OMv4HeobwWPNB9AymERNghjeWtKP+wePU3FIfZvypANUs9anWnL66y/gTALcxwx82HsDlv3we\n1/3RXaX2cH8SW1rHXlxrR1sv3vJfT+CWVbsLeq7R/qrM39wer5ToyEAKB7uH8K4frsDybR34w8ZR\nCkHUPbtdyIJW96zbiz/79rIRXXKM7zy4FR+64ZnQ23In4xrVnog7Ec9f84G/meB7xrxZAOBnEwHg\niBnb3GoAACAASURBVJUBZca/NFk1/sNjBP5DKev3XVrgf+eaPfj0rWvzlhfVi6uWNuMX1gey/UFn\nDsRKyvinMpjalEBTIoZh68CurWcIMcmeRDuUyuDIQNKa3Ov97E8i7Thu7X1MMHfGJK/UJ5VVpmP+\nRvusGv/bn9mDy+5Yjx3tvZgxuQENcfejKZ1R9A6n0ZSIjejaY57nlDlT8ZenHlf0a6doq8eMP9WP\noVQGV9y3EV39Sbzj+8vxhbteABCcrX3vtSvxnmtX4vonW/AnVz4y4v67D/XjpYM9fmn06p3ja8hR\nSJg6nHZw5QPBZOCx/iR/8Ng2NBdwsDKaIas7XSbPgXB779CIz4fUONYEqISJOO6o+cbXZoLvG+bP\nBABMtjL+du2xKUdg4F8cO/DO16Kzo9cL9gdTZavxP9Q3DEfd93Jy4+SSHquWtVm1jPGY+Kdxk2nH\n/6AqqatPMo3JDXFMaohnlfp09A7j2KlNSKYz/kHc9rY+/Nm3l+HYqY0A3GzOQDKNN317GQCgwTtI\neNWMSWjvGYZOB6ZZgf80v8Y/7R9EmoOXXR39mDGpAQ1x9zHSjoO+ofSIMh/ze4iL4I9fPrfo103R\nV824v5Q2g5XwxNY23LFmD279VGiXbapBv1m/D3ev3Yt4TLImtrZ2D+Jzv3oeB7xt33/UrUk/++rH\n8eiXzvH3O/cHywEAP73kzQCAx7e2o384PeIMay4Z5Yg696antmev1XTX2r14cls7Tn/1TLznjBOy\nAnOFYn+exbXGY/m24DnzdeU5++onSn6ecpuIwL/mM/7He4H/GfO8wN/O+PcHgZApg7BLfa56YDPu\nGKNN04rtHfj4Lc/lPWKshi/c9QL+sLFySzx/4a4X8GDOqbiBrMB/7Iz/kTJ19QmbQFqPOnqDVmHT\nmhJ+xt8ckE1pjKN7MFX0Kf6eoTSmNMYxqSGG4bSDBzbsx5fvfRGH+oYxe1rjiMDb0aDdWddAEget\nLxxzgDB3xiS09Q6FZPzdy33D6RFlY3sOD2DG5IR/hiGdcUt9pk0KD/xZ6kNUuEtvW4fl2zqOmlVf\no+ozt63F9U+2jNheSnImn10dbqY+9x1d1dI54nscANp7h/0kje2zd6z3L7/32pX+5dEWunp2V/hq\nu2N9Kr+49wgebW7DD5dtx/t+shLn/3hF9v3L/LF+1dLmsXc6SkxEsqDmA/+//7P5uPJ9p2H+MW42\n2A78wzL+vUNBcPTAi634/YbRA+i7176CFds7shZjqKb+4TR+/2Jr1izwchpOZ/D7F1vx0ObsD4zB\n8QT+g2n/913q5N4uv2Vk/Z6pGUims37n05oSfpbf1MifMHMSHAX6ilgwraN3GM/v6cKZJ85CU8LN\n+D+y+SB+98I+HOwZwpzpTX72Z6r19xWML4NWr/vPX7zmOFzy5ycBAI6f0YT2nmH0DqUxvanB339q\nk/sYh0IWfkmmHcyY1ICEKfVxNG/GPybCyb00pmqW+hQSX3cPpvCV32zM2zShEo6iPFZZ5LYcrrbH\nt7b7GXZbuogJt6PpH07jllXuJNNyHsvt7hwoKNn5kSVrRnzHX3b7Olz7x5EHPfnkrhsDjH42IerK\ncbZjLDUf+C84bgoufdvJ/n+UyVmTe0fW+DsKNLf24FDfMI4MpLAjz8If3YMpvHSwB6ta3CPaHW3j\nXyBkV0cfHm0+mLflVP9wuqClpptbu/3SDnMAsml/ZZZlb+9xg/eWtuxxmfIekexSn6FUBtsO9mYt\nYJGd8S8tYDcZknrO+B/qzX7tkxpifsbfHBCceOwUb9/Rl3Fvae8dUap177q9SDuKi9+yAJMaYhhK\nOdjTOQBHge0H+zB7WhD4z5rSmHVfk8k3C+j8x3tPw2vmTAPgZvz7htPY3dmP46YF95uUiEMk+yyG\nbcakBj+Tn3YcN+MfEvj/9Wlz8dZTWNtPowsLLCbKTU/tBAAsvOJBfOnuF/Luc8+6vbht9e6Kj8fE\nU5WaQFitEwm/GqNdYqWEnWF9cW/w3WwHzw9vOoA3f+fx0MfZeqAn9AzBUCqD9167Emt3Hx5x21lX\nP47rrPuUu6/+cDqDhVc8iD/5z5HzAkbz2Ja2kp43lVHUcz5nIsrRaz7wzzUlT8bfPgj42+uexpXe\nqnNdAyl0hqy49v1HX8IFP17pB54tHWMH6LkuvW0dPnvH+qxJLbYlK3bhb3/yNIbT+SeuPvfyYbz3\n2qfxC++o3gT+bT3DWXXf5dLe6z7my4f6s7ITZjXkY6Y0ZmWfb1i+E3/7k6dxsGfI/4LtHkz5v+/S\nS31Grg5bbzpy/n82JuJ+Vx8TxJ/qBdsHRlm8pG84jfdc+3TWJGEAeGTzQZy18Bi8Zs40N+OfzmCv\n9/8smXGySn1M5x/jlNlTAQQ9l2dPDwJ8M/H+UF8Si46f7m+PxQRTGuL5A//JiSDw90p9cnv4A8DV\nHzwDf//m+XlfLxFQ3ZbCD28+iFu9z+77Q84uL/7OMty43D04mIig2cRT1epc0jecxpX3b847T8yW\nzji4d+1e/MNNz4yrJePqnZ0lleY+/0pXVkfA0VyzbLv/nWme+8LrV/nXV+4I6syXrNyV93He/b8r\n8f1Ht2HhFQ/ihuVBMN/S3ofm1h5c9UB4qYr5v1MJ1Zzk+sAYlRhR1hgfeVa93CIX+E/KmtybnfG3\nzwasaglmru9oH7kinD0xZMakxIgM+N7DA1j8nWXYvL87dBzJtIM9nW7t3c6O/tB9Wtr7MJjKYPeh\n/GVE967b6+8LIGulvU37wp+7FG1exj+ZcbLKmwaTaYgAx0xpyPrQXr6tHcmMk/X7aj0yiLTjHrWX\nGvibeRqltKqsdblLgTfGZUTG/9Tj3cA/rO++0by/G8m0M+Ls1d6uAbx2rhuYT2qI4WD3UNapc7fU\nx/3byQ38T/YC/5b2Poi4C28Zc71Wu/b4jKlNCf+AJjeodyf3WqU+eTL+RIWo9jyQb4yyamdYuVs5\n5CuVMCtdVyruN/XJf9jYivtCVtNdsmIX7lizZ0TyIcxNT+3Ev9+3EWt3d41oyTiUyoQm7Fbu6MDF\nP1uDy25f55+FDDNal7gP3fAMPvbzZ8ccn3H21U/gh8u2Q1Vx8c/WZN1mH4CEfTZvb+vFmpw6+bAS\noWo4muY11hOzMG0lRS7wN8H9nOlNODKQ9CcxHRlIYYFXDgFkt5nMLbfZ0zmAfV2DuPyvXoOfXvJm\nvPHEWSMy/qt3duJQXxIrdnQgmXZwxX0b0dwaBOL7utxSidnTGrH38EDoKcE9h/tDn9/oHUr5k3hb\njwzhyvs34/cbWzG1MY6YABvzHHSUwj6LYB8QDSQzmNwQx9SmhB9sdvUnsckbwx9fcnvtxyTo3XvC\nzMnoG04X/QGSyjh+AFrPGf8RgX8i5pd+mYOwU+aYwD9/xt+8V68cHsAPl23Hiu0dGEimcWQghVd7\nC25Naoj7S6UbdqmPCfyne9cXWhn/Y6c0+rX5gFvjbyyaOzLwN2VJr5oxKeu26ZMarMm9blefsTpM\nENWKf7x9HX761MhMrZ1QGUxmsK+r+HllH/7parS09+Ff7tmQVQJS6VKfj/7sWQylMvj8r17Al3/9\n4ojbM477uZW7Nkv3QAo/fWpn1qTjfXlqnZNpB6+/8pERZTOq6p/xfOKldrzrh8GE0c37u/GjZdtx\n8ZI1fvmKycav3tnpn+E0z9/c2oNvLG3GMqtsxc7s57r2iR04GHIG/tLb1mHhFQ/i1lUv+0k142D3\nEP7mRytw0ZLsgwVVjHjvM45OeCD+eIklO1SctFP50sTIBf6m1GfR8dOQdhQPbGjFkYEk+obTWHDc\nlBH7J2KClvY+vHSwx1/kYqV3NuDDi+fj/NNfhVOPn4aW9j44jmI4ncEDG/Zjg7eY0ub93Vi2pQ13\nr92La5/Y4T+uyZa/7dTZGE47aLfKGl4+1I9VLYf8ANkE/oPJDJa+2OofJKzf04WhlIO5M5qw5v+3\nd+bhUVTZAv/ddNJZyR7IRkgCAWXfFwFXVGBwH5+iMjqu476M+/gcfTrjMo7jNqMzo6MiiyIqgqjI\nAKIIAcKWQCCQBLLvIQvZ033fH1Vd6SZpCDEx6eb+vq+/VFVXV5+T21X33HPPOTe7go+Sc0jNryYx\nIoCk/v1Iyz95nH9prbYwko2mVgsrdhU4XdWvpKbJMLrsByT1LRb89FVX6/USjD9llSOl9j/coBv+\nsSF+hq4DQzVj8mRT7an5VRwodqzZW1jVwLd2C3ioGP82vEztY/xD/LwID/A+occ/VZ8hOlR6jDfX\nH+KfP2QZAwXbSrvenh7tEv/CA7wNj7vNYJg3JorLx0Zz+dgYQFskJjzA0VNhq7hl8hDEh/k7vOdn\nNhkef1tJXtsAINDX066cp1bHv6OqPgqFqzH7tR9Ym17CC98caPcM/vuGtsHAje9vY8ZLG5BSdjnm\nd9arG/liVwFPfJ5GRnEtlXXNRvjGOxudh510BotV8sn23A6TVZ/8Is3YXpSc0+E5248cNerGAzz1\n5V5e+OYAm7M6rhJjz0vfHujw+HmvfM+jy1Mdjq1OLSKnoo55b27i9XWH2GLnXf9kuzabPv/fycx8\neQNH65odnn0fbD7CbQtTyKusJ/7x1Uz+0zq+2NV+FsPG75e1H+jY6GjWZ+oLzktJznhpA9A2UMso\nqWXem5ucnt8TPPpZ6slPUnQ7K/f0fJiT2xn+g/sHEOLnxeSEUAAe+GQ3v3pDu2HGDgx2ONds8mBK\nYiif78znqn9s5qb3t9FisbLtcCVRQT5GGMOI6CDqmy3szq9iUXIu93+8m0/1EJzU/GoWb9VKgv53\nf6lR1tAw/JMiHPZLaxuZ/69kfvv+diMMxpZg/N6mbO5buot1B0o51tRqhPJcMS6WVrsnkhAwKjaI\ntIJqp2XZrFbJwZJabvzPdm7/aIdhEK5OLeKBT3bzkN1D6mhd28xIaU0jkYE+xIb4OoQSNTRb8DWb\n8DN7Ut+iyb3tcCX+ZhNnD42g1Srx9vRgeFSg8RmbF9qZ4V/f3EpTq4WbP0jhkU8dHzIvfnPAWIgE\nur+qT4vFSmZprcOqs6dKfXNrl8tndjaGFDSPf7BfW4iNl8nO468PwnzNJmKCfSisbm/426qF2Dz+\ntY2tSKl1vtn6TJbN4x8b0jY4jtcHyvYef1sVofgwf167dhzx4f4Msp3X77jEX29PfL1MDArza7f4\nlr+5rSTp+EEhxIb4MjJG++0E+nhh8tDOr2+20NxqNWYYFIpT5fiE9N7EfrXQjjyqj+he8m2HtWTO\n9zYdZvQz3/HMzyxHuDa9mLXpbY4UeyeVlJK3v8/i+neTueWD7UaITGbpMeIfX+3g9bZx39JdPPZZ\nGkP+8E27Z+DnOwuM7adW7GWJPuPw6tqDDoOb5TvyWPDeVgqqGoxnVGOLhZ25R1mWktdhNaaGZovT\nUtZHKtrPkNy9ZCfn/OX7Ds//KrWIZ1e1/V9vXZhi5NLZM/PlDcb2g5/sYZeTfIPODFpOhbNeWGfY\nLqAlACvcn19isVK3M/yHDujHrqcvcjBAC6oamDsqkvmT4xzO7R/ozQtXjMbDQ9BqlZTWNrFufwlp\n+VWMiQ024iRnj4zE32xicXIuS3Qjv8Ui8TObyD/awOasCq6ZOBCrlPxlTQZSSnIr6vHx8mDCoBAA\nI97/2VXpFNc0GsmZZk8PMkuPYbFKlm7TBhNPrUhj7LPf8fH2PBIj/I0Bi80rOyQigNGxQZQfa+5w\netFqldz+0Q4u+tsPxsNi0yFtFmNvgba/ak8hKUcqyT9az6Q//Ze/fncQgJLaRgYEenPW4DA2Z5Ub\n04v1za34eXkS7OtFbkU9JTWNpOZXMzImiHG6fC9cOcooq+rpIZgUr+neUWUfi1Uy781NnPuX7yk/\n1sTewmoq65oN+TfZ5WCYPES3e/yf/yqdWa/+wPSX1rebbegMrRYr897YxPXvbnU6e+KMbYcrGf/c\nWqNzPxlltU0O3nSzp1ZrH9oMcX+zJ9HBvu08/odKahn/3Fru/3gXh8vrGBHddl80t1qNJKqoIM3b\n/uCFQ433bdV5wvu1JffaknRHRAcZ59nW0Dje4y+EIC7Uz+FetGHLGQC4clwMmx47n8ggm8ffC0/d\n41/doLW7ivFXdJUh/QM4I7LfyU/8hXl+9f52xz7dke9QQ93mlf5g8xHWppdQ3dDCDwfLOqwUd6JS\nka98d5DHPktzOFZa00h22TFS86t56dsD/JRZYYTIxD++mlmvbgRgxe6CdtdbndZW7jnxya9PqOdz\nX6WTV1nvMNgAbYbjx0PlTH9xvdE/3rl4J1f+YzOPLk81+kN7ec98+tt2ITM/B/tcgx05Rztsk+O5\n4rh8g56i8ASFGhTuS//jQl97ArftTW0eynOGRvDghUM5M6ofpuNcCP37eRMX5seqe2YgBPzPO1t4\ne2M2RyrquXriQOO8AG9PLh8Xw5JtuUgJI2MC2VtQw2Vjo1m6LQ9PD8EfLx3OgEBv3lifSXJ2BQVV\nDQwdEEBsiC8eQksGLq1pZM3eYuZPjmNZSh4Wq2RaYhhbsitYlJxDQVUDo2KCDM9sQVUDl4+NNuKj\n546K5KoJscQE+xphOHvyqnhzfSYb7ZJrWyxaaNFd5w5mzsgobv5wO+9uymZVaiFltU0MHRBAVlkd\nGzJKiQ72pdUqeWtDJiv3FJJbWc/sEZHMSIpgWUo+qflVjB0YTHFNE75mE3ecM5hv9xVz9+Kd7C+q\nYcHUQdw6M5GZQyMYOzCY/KPawz0xwt8wBGe/9iPxYX7cfd4QVuwuIKeinrEDg42FR3y9TDS0WPgp\ns5xLxkSTXlRjDAIA4kL9qKpv4afMct7+PosHL0zikeWpNLVY8fHy4PnLRzFtcBjV9S3cuXiHEUL1\n2+nx3Dw9gSe/SOPHQ+Wcd0YEof7eZJcdY8OBUs4ZGkF6UQ23L9zBynumG57BI+V1PPF5GvOnxPHR\nliP8/qJh7cpGbsgoI7u8juzyOv5vVTpeJg8CvE08dNEw3v4+i0XJOZwR2Y+3rhvPhoxSPk3J4x/X\nT8DXbGJtejFWCR9uOUJkoA8PL9/DPecN4eyhERxrauXeJTuZNXwA10wcyMOf7mF9Rinj44LJ1MNi\nzSYPDhTXMv3F9cagytdsIjrYl+8zytiVe5TH9Gna/v189AW5CokP8+OJOWdyw3tbCQ/wpqahhdVp\nRQiBYXQH+XqR8tQsjtY1s3BLjpGwazO8bzs7kSvGxTDGbvZsdGwQX6UWtTP8Af79m4n4ebevUuBn\nZ8jb3g/11z4f6NO2ToGtLVWMv2shhJgNvA6YgHellC8e977Q358L1AM3SSl39pQ8Zw0O50BxLRnF\ntQzrI4OAAidhefsK2xwR9nlWty1MaXfugedmsywljzkjo5j0p45LRTpj8p87t2rp6tQiVqeuBuDt\n68e3S/A/GS0W6eA17whbAQzbLGBHdFZehcKVie8gJL27cdvedFJ8KPedP4SbZyQ4TPU+d/lIJsWH\nMO+NTUZssa0G+rWT43h1reb5Hh0b5HC9O88djFVKfL08ufu8wbz/0xFum5lImL8386fE4Wf25IFZ\nQ/H2MrFufwkFVQ1IqYVlRAf7sjqtiJSco7RaJbfNTCCztJbtR45y7/lD2JJdwR9X7mNMbBD/uWkS\nS7bmUljdyNJtuYyKDSYx3J9HLh7G5eNiDK//mVGBeHoInl2VTlF1I7POHOAQDjIqJojfTBuEEIIZ\nQ8L5YlcBB/XKRDedFU+gjxebDpUTHexLVJAPV42PNRJ0TR6C6YM1Q3fjwTK2Hq5kT14VT/3qTIZF\n9uPRi4cZMYujYoPwNZuMWQlbhZb4MH8HL63Z04NHlqfiZRIMj9YMxf79vHnk4mFEB/ty56IdvP19\nFsnZFWQdl0gdH+ZHcnYldy7aQU1jK/sKq6lrsnDp2Gi2Ha7kzsU7WHHXdJ5euY/tRyq5ZEw0eZX1\nPL96P+sPlLI5q4JJ8SEsSnasc/zArCSsEub/K5l7luzixatGsXJPIYfL6tiSXWHEg965aAdzR0Xh\nZfJgwbRB/HCwjCVbcxkQ6M280dG8t0mbHvb0EAT5mXnp2wNMGBTC+oxSFry3lbSCapparby3KRsP\nD2FUQFqzt5j9hTVkl9dxd9FOLhkTzf6iGnblVrHxYBmr9hSSnF3J3FGRXD1xIPedn0Rds8XwnEUF\n+TBtcBgJ4f74eJmICvKhocXCLR+m4KOH1hwsKWfuqEiGDQjk0rHRRtWdCYOCsUpYm15CqJ/ZqKID\nmuc+PMCbG6YO0n5nJg/mjIykocVCdJCP8Ru0MdKJxx/oMK8GHBcCs63kG+avyRbo62WEdj27Kp1g\nPy9mJIV3eB1F30MIYQL+DlwI5APbhRArpZT2gc5zgCT9NQV4W//bI9gWObr4tR9YMHUQt5+daDz3\nXZkz/lersf60k3KP3c2di3tsbKZQuA1f3TujyzkZZ0S2nyHvbtzW8Dd7evDQRcPaHV8wVVtV9LKx\nMUxJDHV475pJA3l93SEsVmmEL9iIDfHjhStHG/sPXzzM4S9o9cnvPm8IN09P4I5FO5g7MhKA2SMi\nWbG7gOqGFq4aH0tiRADXTYkjPMCbifGh/PXqMbz/02H+fv14wgK8ufeCJPKP1rO3oJrzhkUghHZd\ne3y8TMwbHcWmzHJum5nAk3PPdFrC7crxMWSX19HYbCGjpJbRsUEE+3nx+rpDHCo9xrzRUTx88TBu\nmZHAbQtTuHRsNGEB3swYEs47G7NoarUyb3QUt8xIAOCqCbG8vCaD+mYLo2Md8yZseQvx4f4khgcw\nMiaQ3184jBExgdy1aCfzJ8cxZ1Qk9y3dxYXDBxgzK/OnxPHZjnzW6InIs0dE8q2+nRAewIaMMny8\nvBgeFUh6UQ1XjIvhlavHcKS8jkvf2sSv3viRumYLL1w5ivmT44zKErvzjnLH2Yk8PucM/u+rdCrr\nmhkcEUBW2THGDtTCuZ6/YiSPLk/l4r/9QJ0eXzdxUAgtFisLpsXz7o/ZrNlXTE1jK5+m5FHXbCHU\n38yDs5K4bsogjtY102Sxsjq1iOe+SmdSfAiLb53K0m25/H1DJiOiA6msa+YVPZwK4NpJA9mcVUGL\n1crLvx7Nh5uP8N2+YkwegqfnDWdDRin7i2q5/4Ikh/Ab0PJEgtK8ePfGiQ7laycnhBId5IOftydv\nXTcOgD98sZd7zktiuF2Iz6wzB3DJmGgCvD1Zm15CRV3HYVTDIvsZ3tH+gT787pzBHZ43dmAwk+ND\n291PJ+JMu/AfWyWuyQmhTBgUQlyoHyF+rcSH+dFikfzl16OJCvJ1dilF32MykCmlzAYQQnwMXAbY\nG/6XAQulllyULIQIFkJESSmL2l/u5/M/E2NZlqIlZn6UnMNHyTnMnxxHWW0jc0ZG4amXyDXKXQJZ\nZcc4VHKMgaG+JPXvx+HyY0T086aoupHRsUHGYLm+2YLZ5GGEpwl6t3yoQuFO/O2aMUaBiz99ffJQ\nrJMxOT6Uc4ZFMC4umIeX7WHa4HA+25lPmL/ZaV8I8PicMyisamDhlhzGDgxmfFwIV0+MZc7rbaF5\ni26Z0qGTKu2Zi/gmrZgPNh/hvguG8LtFjgPoL++ebqz/8EuU8xTOkkN/SSZOnChTUtpPY/YG9y7d\nxcHiWtY8eHZvi9LtrNhVwEPLdrPh4XOpaWjlkre0Eek7N0xgtj5Isae0ppFL3/qJUH8zy++cZnhm\nAZ7+ci/f7i0m+YkL8LCrk71mXzF3fLSDZXdMMxKsu8pdi3fwdVoxL181mie/SGPhLZOprm/hriU7\n+ezOsxgfp+UQbDxYxs0fbOe6yXE8d/nILn3Xs6v28eHmI8wZGcW3+4pZ/rtpjNOvb2N3XhXX/HML\ns4YP4K3549oNtH77/jYOlhxjxd3T2928i5Jz+OPKfcweEcnqtCK+uX+mg/HbG1itksQnv2ZmUjgf\n3dJjztYOsVgl576ygcKqRrL+PPcX/W5XRAixQ0o5sbfl6AxCiF8Ds6WUt+r7C4ApUsp77M75CnhR\nSrlJ318HPCalTDnuWrcDtwPExcVNyMnJ6ZJMFqtk8Eli0bub0bFBxAT7klFcS3Z5x2u5uBMvXjkK\nP29PPk3J02Yol3a8UjFoZZ+7qzrlTWfFMyUhlPs+3uWw6FR4gLdDKWQvkzDenzAohMRwf3Iq6rn3\ngiE8+Mkeh3PDA8zMGBLOLTMSjX7SnvAAM98+cDYrdxeyOq2I5y4byatrMzhcXseNZ8Xz9Jf7WDB1\nECYPwRXjYrSyxxKWbMslwMeT3blV5FbWMaR/AEu35REb4kuQr5dDmJeNfy6YwKo9haTrs8MHn5+D\n2dODfYXV1DS0MjUxlJSco9Q3WyipbqSmsYWrJwzkx8wyDpUco7HVgpRabpkQMDDEDw8B95w/hDX7\nSrhhyiAOFNdgkZIb3t3K0/OG4+/tyc7co4yLC6GwqoGZSeFMGBRKbkU9WeXHiAryMTzTR8rr8DQJ\nHl2eyoykcHbmVPHo7GHEhfpxz5KdnDU4nGGR/Wi2WGm1SM6I7MeOnKMEeHtS09hCq0UybXAYQX5e\n5JTXc9P72/jzlaMwe3qQUVzr4HCqrm+hsr7ZKLxi1XMzbaGqVqtk+c58ZiaFExXkS2OLha/Tirhi\nXAz1zRaaWq2E+nec7C+lpKCqgZqGVoprGpiSEMbegmomxYc62Dg2m9nW/ze3Wnl+dToPXTjUIbok\nNb+KJVtzmT0yknOH9Xf4ro0Hyxgc4c8n2/MYER3I7JFRtFqsmDyEUwfuyTiVPkIZ/sfR2KL9OE41\njtFVKK1tpL++sNLh8jpaLFaS+gc4/bHVNrZg9vTA29MxTru51UpdUyshHdxE5ceaOgz7OFVaLNp3\nBPp4UVnfVi6ytKaxXQJMxbEmQv3NXb5ppJRU1jUTFqCt/+CsEsjRumaCfL0cHgQ2Glu02QJ7L7w9\ntuuW1DQaYWa9TU1jC2aTh1OZe5JWi5XqhhbCuuG34u6croa/Pd3RT9Q3t1JS04SXSVBa20R9709y\nzgAACMBJREFUkwVfswdBvl4IIYzyxBLtmeLtaaLZYsHHy0SrRRozA23lbbUSyfb3T1OLFQ8PLYQ0\n0Mc9+xGFQtG3OJU+wm1DfbqKj5epV4ygX4r+dqup2kbNJ6Kfk47L7OmB2bNj47g7jH7Q8iNsBrj9\nNTvKev+5xqMQwrjGicr/dTTQsXGy343tun3F6Ad61TDxNHkoo989KQAG2u3H6sdO9Zxux8/sSUK4\n1u3Zl67tiM48HxUKhcLVcLtyngqFQqHoVbYDSUKIBCGEGbgWWHncOSuB3wiNqUB1T8X3KxQKhaIN\n5fFXKBQKRbchpWwVQtwDrEEr5/kfKeU+IcTv9PffAb5GK+WZiVbO87e9Ja9CoVCcTijDX6FQKBTd\nipTyazTj3v7YO3bbErj7l5ZLoVAoTndUqI9CoVAoFAqFQnEa0GOGvxBithAiQwiRKYR4vKe+R6FQ\nKBQKhUKhUJycHjH87VZunAMMB+YLIYb3xHcpFAqFQqFQKBSKk9NTHn9j5UYpZTNgW7lRoVAoFAqF\nQqFQ9AI9ZfjHAHl2+/n6MYVCoVAoFAqFQtEL9FpyrxDidiFEihAipaysrLfEUCgUCoVCoVAoTgt6\nyvA/6aqMUsp/SSknSiknRkRE9JAYCoVCoVAoFAqFAkBo5ZS7+aJCeAIHgQvQDP7twHVSyn1Ozi8D\ncrr4deFAeRc/2xdxN33A/XRS+vRt3FEffynlae0hUf3EKaH0dW+Uvu5NV/Qd1Nk+okcW8HK2cuMJ\nzu9yhyaESJFSTuzq5/sa7qYPuJ9OSp++jZvqE9/bcvQ2qp/oPEpf90bp6970tL49tnJvRys3KhQK\nhUKhUCgUit5BrdyrUCgUCoVCoVCcBriD4f+v3hagm3E3fcD9dFL69G2UPorjOd3+h0pf90bp6970\nqL49ktyrUCgUCoVCoVAo+hbu4PFXKBQKhUKhUCgUJ8GlDX8hxGwhRIYQIlMI8Xhvy9MVhBBHhBBp\nQojdQogU/VioEGKtEOKQ/jekt+V0hhDiP0KIUiHEXrtjTuUXQjyht1eGEOLi3pHaOU70eUYIUaC3\n0W4hxFy79/q6PgOFEBuEEOlCiH1CiPv14y7ZRifQxyXbSAjhI4TYJoTYo+vzrH7cJdunL+Im/US3\n3cdCiAl6n5MphHhDCCF6Q6fOIIQwCSF2CSG+0vfdVl8hRLAQYrkQ4oAQYr8QYpqb6/ug/lveK4RY\nqj8L3UZf0U22kTP9hBDeQohP9ONbhRDxnRZOSumSL7QyoVlAImAG9gDDe1uuLuhxBAg/7tjLwOP6\n9uPAS70t5wnkPxsYD+w9mfzAcL2dvIEEvf1Mva1DJ/R5Bni4g3NdQZ8oYLy+3Q9tfY3hrtpGJ9DH\nJdsIEECAvu0FbAWmumr79LWXG/UT3XYfA9v035gAvgHm9LZ+J9D7IWAJ8JW+77b6Ah8Ct+rbZiDY\nXfUFYoDDgK++vwy4yZ30pZtsI2f6AXcB7+jb1wKfdFY2V/b4TwYypZTZUspm4GPgsl6Wqbu4DO0h\ngP738l6U5YRIKX8AKo877Ez+y4CPpZRNUsrDQCZaO/YZnOjjDFfQp0hKuVPfrgX2oz10XbKNTqCP\nM/q6PlJKeUzf9dJfEhdtnz6IW/QT3XUfCyGigEApZbLULIaF9NH+RQgRC/wKeNfusFvqK4QIQjMU\n3wOQUjZLKatwU311PAFfoS346gcU4kb6dodtdBL97K+1HLigs7Mdrmz4xwB5dvv5nNgA6KtI4L9C\niB1CiNv1YwOklEX6djEwoHdE6zLO5HflNrtXCJGqT9/ZpudcSh99KnAcmlfZ5dvoOH3ARdtID2fY\nDZQCa6WUbtE+fQS3+3/9zPs4Rt8+/nhf5DXgUcBqd8xd9U0AyoD39dCmd4UQ/ripvlLKAuAVIBco\nAqqllN/hpvra0Z36GZ+RUrYC1UBYZ4RwZcPfXZghpRwLzAHuFkKcbf+mPspz2dJLri6/zttooQJj\n0R5Sf+1dcU4dIUQA8BnwgJSyxv49V2yjDvRx2TaSUlr0Z0Asmpdn5HHvu1z7KHoGd7uPnSGEmAeU\nSil3ODvHnfRF836PB96WUo4D6tBCQQzcSV/dMXMZ2oAnGvAXQtxgf4476dsRvamfKxv+BcBAu/1Y\n/ZhLoY98kVKWAl+gTU2X6FM86H9Le0/CLuFMfpdsMylliW6cWYF/0xZa4RL6CCG80IyFxVLKz/XD\nLttGHenj6m0EoE/tbwBm48Lt08dwm/9XN93HBfr28cf7GtOBS4UQR9DCs84XQizCffXNB/L12T7Q\nQjfG4776zgIOSynLpJQtwOfAWbivvja6Uz/jM3q4VBBQ0RkhXNnw3w4kCSEShBBmtOSGlb0s0ykh\nhPAXQvSzbQMXAXvR9LhRP+1G4MvekbDLOJN/JXCtno2eACShJa70aWw3qs4VaG0ELqCPHvP3HrBf\nSvmq3Vsu2UbO9HHVNhJCRAghgvVtX+BC4AAu2j59EJfvJ6D77mM9zKBGCDFVv+Zv6IP9i5TyCSll\nrJQyHq3N1kspb8B99S0G8oQQw/RDFwDpuKm+aCE+U4UQfrqcF6Dlrbirvja6Uz/7a/0a7R7p3AyC\n7APZz119AXPRqhtkAX/obXm6IH8iWib3HmCfTQe0OK11wCHgv0Bob8t6Ah2WooVWtKB5LW45kfzA\nH/T2yqCPZN93Qp+PgDQgVb/ZolxInxlo04mpwG79NddV2+gE+rhkGwGjgV263HuBp/XjLtk+ffHl\n6v2ErkO33cfARP23lgW8hb6QZ199AefSVtXHbfVFC1NM0dt4BRDi5vo+i+bk2Ks/v73dSV+6yTZy\nph/gA3yKlgi8DUjsrGxq5V6FQqFQKBQKheI0wJVDfRQKhUKhUCgUCkUnUYa/QqFQKBQKhUJxGqAM\nf4VCoVAoFAqF4jRAGf4KhUKhUCgUCsVpgDL8FQqFQqFQKBSK0wBl+CsUCoVCoVAoFKcByvBXKBQK\nhUKhUChOA5Thr1AoFAqFQqFQnAb8PxfhSH04HnA5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdac759c3d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 10000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = current_model.act(state, epsilon)\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > batch_size:\n",
    "        beta = beta_by_frame(frame_idx)\n",
    "        loss = compute_td_loss(batch_size, beta)\n",
    "        losses.append(loss.data[0])\n",
    "        \n",
    "    if frame_idx % 200 == 0:\n",
    "        plot(frame_idx, all_rewards, losses)\n",
    "        \n",
    "    if frame_idx % 1000 == 0:\n",
    "        update_target(current_model, target_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><hr></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Atari Environment</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.wrappers import make_atari, wrap_deepmind, wrap_pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env)\n",
    "env    = wrap_pytorch(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CnnDQN(nn.Module):\n",
    "    def __init__(self, input_shape, num_actions):\n",
    "        super(CnnDQN, self).__init__()\n",
    "        \n",
    "        self.input_shape = input_shape\n",
    "        self.num_actions = num_actions\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(self.feature_size(), 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, self.num_actions)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "    \n",
    "    def feature_size(self):\n",
    "        return self.features(autograd.Variable(torch.zeros(1, *self.input_shape))).view(1, -1).size(1)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            state   = Variable(torch.FloatTensor(np.float32(state)).unsqueeze(0), volatile=True)\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0]\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_model = CnnDQN(env.observation_space.shape, env.action_space.n)\n",
    "target_model  = CnnDQN(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    current_model = current_model.cuda()\n",
    "    target_model  = target_model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(current_model.parameters(), lr=0.0001)\n",
    "\n",
    "replay_initial = 10000\n",
    "replay_buffer  = NaivePrioritizedBuffer(100000)\n",
    "\n",
    "update_target(current_model, target_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Epsilon greedy exploration</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 30000\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fdac7610dd0>]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD8CAYAAACW/ATfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGANJREFUeJzt3XlwXWd5x/Hvc+/V1W4ttpzIthLbwVmcQgJRtrIkBMjW\nFrcztHVoWVIgk5IwQDuFADMtHf5hnaEMARPSlKUFE0IGDGMIS9MSSLMokM2JTRQ7sWUnsRzvlm1t\nT/84x/a1rKt7JF/p6Jzz+8xodM97zr3neb389Oo9m7k7IiKSbrm4CxARkemnsBcRyQCFvYhIBijs\nRUQyQGEvIpIBCnsRkQxQ2IuIZIDCXkQkAxT2IiIZUIhrx/PmzfPFixfHtXsRkUR65JFHdrh7x2Tf\nF1vYL168mJ6enrh2LyKSSGb2/FTep2kcEZEMUNiLiGSAwl5EJAMU9iIiGaCwFxHJgIphb2Z3mNl2\nM3uyzHozsy+ZWa+ZPW5mr6l+mSIicjKijOy/AVw9wfprgGXh1w3AV0++LBERqaaKYe/uvwZ2TrDJ\nCuBbHngAaDWzzmoVONaGF/fx+Xs2sPPA4HTtQkQkdaoxZ78Q2FKy3Be2ncDMbjCzHjPr6e/vn9LO\nNvbv58v39vLS3kNTer+ISBbN6AFad7/N3bvdvbujY9JX+wLQWBtc9Hvg8HA1SxMRSbVqhP1WoKtk\neVHYNi2OhP1+hb2ISGTVCPs1wDvDs3IuAfa4+wtV+NxxNR0d2Y9M1y5ERFKn4o3QzOy7wOXAPDPr\nA/4FqAFw91XAWuBaoBcYAK6frmIBGmvzABwY1MheRCSqimHv7tdVWO/ATVWrqILGoubsRUQmK3FX\n0OoArYjI5CUu7IuFHMV8jv2asxcRiSxxYQ/BvL1G9iIi0SU07As6QCsiMgnJDPtiQSN7EZFJSGbY\n1+Z1nr2IyCQkNOwLuoJWRGQSEhn2TbWaxhERmYxEhn1jbYGBQU3jiIhElcywL+Y1jSMiMgnJDPtw\nGie4U4OIiFSS2LAfHnUOD4/GXYqISCIkMuyP3OZY8/YiItEkMux1MzQRkclJZtgXg3va6yCtiEg0\nyQx7jexFRCYl0WGvkb2ISDSJDHsdoBURmZxEhn2D5uxFRCYlkWHfpDl7EZFJSWTY6wCtiMjkJDLs\n9RxaEZHJSWTYQ/AAkwE9mlBEJJIEh32B/YcU9iIiUSQ27JtqC+zTnL2ISCSJDfs5dTXsOzQUdxki\nIomQ2LBvriuwT9M4IiKRKOxFRDIgwWFfw15N44iIRJLgsA9G9no0oYhIZQkO+xpGRp2DQ7qwSkSk\nkkhhb2ZXm9kGM+s1s1vGWd9iZj82s8fMbJ2ZXV/9Uo/XXBfcMkHz9iIilVUMezPLA7cC1wDLgevM\nbPmYzW4CnnL384DLgS+YWbHKtR7nWNhr3l5EpJIoI/uLgF533+jug8BqYMWYbRxoNjMDmoCdwLQO\nuefU1QCwVyN7EZGKooT9QmBLyXJf2Fbqy8A5wDbgCeCD7j469oPM7AYz6zGznv7+/imWHJhTr2kc\nEZGoqnWA9irgUWABcD7wZTObM3Yjd7/N3bvdvbujo+Okdtgcjuw1jSMiUlmUsN8KdJUsLwrbSl0P\n3O2BXmATcHZ1ShyfDtCKiEQXJewfBpaZ2ZLwoOtKYM2YbTYDbwIws1OAs4CN1Sx0LI3sRUSiK1Ta\nwN2Hzexm4B4gD9zh7uvM7MZw/SrgU8A3zOwJwICPuvuOaaybxmKenGlkLyISRcWwB3D3tcDaMW2r\nSl5vA66sbmkTM7PgNscKexGRihJ7BS3o/jgiIlElPOw1shcRiSLRYa8HmIiIRJPosNfIXkQkGoW9\niEgGJDzsNY0jIhJFwsNeDzAREYki4WFfw/Coc2johHuuiYhIiYSHfXBNmM61FxGZWKLDvrUhuD/O\nnoMKexGRiSQ77OuDh2HtHlDYi4hMJNFh31Kvkb2ISBSJDvsj0zi7BwZjrkREZHZLdNjP0cheRCSS\nRId9c22BnCnsRUQqSXTY53JGS32NDtCKiFSQ6LCH4CCtRvYiIhNLftg3FNmtsBcRmVDiw761voY9\nOhtHRGRCiQ97TeOIiFSW+LBvbajRNI6ISAXJD/twZD86qtsci4iUk/iwn1NfgzvsO6wnVomIlJP4\nsG9tCG6Gtkfn2ouIlJX4sD9yM7TdB3VGjohIOYkPe93TXkSksuSH/ZGRvaZxRETKSnzYH5vGUdiL\niJST+LA/cpvjvQp7EZGyEh/2dTV56mvy7DqgA7QiIuVECnszu9rMNphZr5ndUmaby83sUTNbZ2b/\nW90yJ9bWUMMuzdmLiJRVqLSBmeWBW4G3AH3Aw2a2xt2fKtmmFfgKcLW7bzaz+dNV8Hjam4rsPHB4\nJncpIpIoUUb2FwG97r7R3QeB1cCKMdu8Hbjb3TcDuPv26pY5sfbGWnZqGkdEpKwoYb8Q2FKy3Be2\nlToTaDOz/zGzR8zsndUqMIr2hhp26jbHIiJlVZzGmcTnXAC8CagH/s/MHnD3P5RuZGY3ADcAnHba\naVXadTiy36+wFxEpJ8rIfivQVbK8KGwr1Qfc4+4H3H0H8GvgvLEf5O63uXu3u3d3dHRMteYTzG0q\ncmBwhENDI1X7TBGRNIkS9g8Dy8xsiZkVgZXAmjHb/Ah4nZkVzKwBuBh4urqlltfeGNwMbZemckRE\nxlVxGsfdh83sZuAeIA/c4e7rzOzGcP0qd3/azH4GPA6MAre7+5PTWXiptvDOly/vH6SzpX6mdisi\nkhiR5uzdfS2wdkzbqjHLnwM+V73SopvbFIS9zsgRERlf4q+gBU3jiIhUko6wL5nGERGRE6Ui7Fvq\na8jnTNM4IiJlpCLsczmjTRdWiYiUlYqwh+CMHF1YJSIyvtSEfXtjUdM4IiJlpCbs5zYVNY0jIlJG\nasK+rUEjexGRclIT9nMbi+waGGRk1OMuRURk1klN2M9rrsVdV9GKiIwnNWE/v7kWgP59emKViMhY\nqQn7jiNhv19hLyIyVnrCvqkOgO17D8VciYjI7JOesNfIXkSkrNSEfX0xT3NtQXP2IiLjSE3YQzC6\n366wFxE5QarCfl5zrUb2IiLjSFXYdzTXskNhLyJyglSF/XxN44iIjCtVYd/RXMv+w8MMDA7HXYqI\nyKySrrBvCk6/3LFPt0wQESmVqrCfPye8sGqfLqwSESmVqrA/MrLXGTkiIsdLV9jrKloRkXGlKuzb\nG4sUcsZLuj+OiMhxUhX2+Zxxypw6XtijsBcRKZWqsAc4taWOF3Yr7EVESqUu7Dtb6nhhz8G4yxAR\nmVVSGvaHcNezaEVEjkhh2NdzeHiUXQNDcZciIjJrpDDsgwurNJUjInJMpLA3s6vNbIOZ9ZrZLRNs\nd6GZDZvZ26pX4uR0ttYD6CCtiEiJimFvZnngVuAaYDlwnZktL7PdZ4CfV7vIyTg6ste59iIiR0UZ\n2V8E9Lr7RncfBFYDK8bZ7gPAD4DtVaxv0uY11VLIGS/s1jSOiMgRUcJ+IbClZLkvbDvKzBYCfwF8\ntXqlTc2RC6te1IVVIiJHVesA7ReBj7r76EQbmdkNZtZjZj39/f1V2vWJOlvq2KYDtCIiR0UJ+61A\nV8nyorCtVDew2syeA94GfMXM/nzsB7n7be7e7e7dHR0dUyy5ss7Weo3sRURKRAn7h4FlZrbEzIrA\nSmBN6QbuvsTdF7v7YuAu4P3u/sOqVxvRgpY6tu05xOioLqwSEYEIYe/uw8DNwD3A08Cd7r7OzG40\nsxunu8CpWNTewODwqG51LCISKkTZyN3XAmvHtK0qs+27T76sk9PVFpxrv3nnAKeET68SEcmy1F1B\nC3BaewMAW3YOxFyJiMjskMqwX9hWj1kwshcRkZSGfW0hz6lz6hT2IiKhVIY9QFdbA307da69iAik\nOezbGzSyFxEJpTjs63lp3yEODY3EXYqISOxSG/antTfgDlt1QzQRkfSGfZdOvxQROSq1YX/kXHvN\n24uIpDjs5zfX0lDMs7H/QNyliIjELrVhb2Ys7Wjk2f79cZciIhK71IY9wNJ5TRrZi4iQ8rA/o6OJ\nbXsOcnBQp1+KSLalOuyXdjTiDpt2aHQvItmW+rAH2LhD8/Yikm3pDvt5TQCatxeRzEt12NcX8yxs\nrWejzsgRkYxLddgD4emXGtmLSLalPuzP6Giid/t+PXxcRDIt9WF/TmczB4dGdNsEEcm0DIT9HADW\nv7g35kpEROKT+rBfNr+ZnMFTL+yLuxQRkdikPuzri3kWz2tk/Qsa2YtIdqU+7CGYyln/okb2IpJd\n2Qj7U5vZvHOA/YeH4y5FRCQWmQj7s08NDtJu0EFaEcmoTIT9OQuCsF+3TWEvItmUibBf0FLHvKZa\nHt2yO+5SRERikYmwNzPO72rhMYW9iGRUJsIe4PyuVp7tP8Ceg0NxlyIiMuMyE/bndbUC8ETfnpgr\nERGZeZHC3syuNrMNZtZrZreMs/5vzOxxM3vCzO43s/OqX+rJedWiIOwf69NUjohkT8WwN7M8cCtw\nDbAcuM7Mlo/ZbBNwmbu/EvgUcFu1Cz1ZLfU1LO1o5PebFfYikj1RRvYXAb3uvtHdB4HVwIrSDdz9\nfnffFS4+ACyqbpnV8equNn63eRfuut2xiGRLlLBfCGwpWe4L28p5D/DTkylquly8tJ2dBwZ5Zrue\nXCUi2VLVA7Rm9kaCsP9omfU3mFmPmfX09/dXc9eRXLp0LgAPbHx5xvctIhKnKGG/FegqWV4Uth3H\nzF4F3A6scPdx09Tdb3P3bnfv7ujomEq9J2VRWz0LW+sV9iKSOVHC/mFgmZktMbMisBJYU7qBmZ0G\n3A28w93/UP0yq8PMuHhJOw9s3Kl5exHJlIph7+7DwM3APcDTwJ3uvs7MbjSzG8PN/hmYC3zFzB41\ns55pq/gkXbJ0rubtRSRzClE2cve1wNoxbatKXr8XeG91S5sel54RzNvf98wOzjylOeZqRERmRmau\noD2iq72BV8xv4t712+MuRURkxmQu7AGuOHs+D256WQ8zEZHMyGzYD404v3lmR9yliIjMiEyG/QWn\nt9FcV9BUjohkRibDviaf4/Kz5vPLp19ieGQ07nJERKZdJsMe4E9e2cnLBwa5/1ldYCUi6ZfZsL/8\nrA6aawv8+LFtcZciIjLtMhv2dTV5rjz3VH627kUOD4/EXY6IyLTKbNgDvPX8Bew7NMx/P60DtSKS\nbpkO+9e9Yh4LWur4zkOb4y5FRGRaZTrs8zlj5UWncd8zO3hux4G4yxERmTaZDnuAlRd2kc+ZRvci\nkmqZD/v5c+q46txTWP3QZvYdGoq7HBGRaZH5sAe48bIz2HtomG8/8HzcpYiITAuFPfCqRa284cwO\n/v2+TRwc1GmYIpI+CvvQB654BS8fGOQb9z8XdykiIlWnsA9duLidN58zn1vv7aV/3+G4yxERqSqF\nfYmPX3sOh4ZG+MLPN8RdiohIVSnsSyztaOL61y5m9cNbuP9Z3eteRNJDYT/Gh99yJovnNvCRux7X\nk6xEJDUU9mM0FAt84a/OY9vug3z87idw97hLEhE5aQr7cVxwejv/eOVZrHlsG1+/b2Pc5YiInLRC\n3AXMVu+//Aye2raXT/90PZ0t9fzZeQviLklEZMoU9mWYGZ//y/Po33+YD33vUQo545pXdsZdlojI\nlGgaZwL1xTx3vPtCzlvUwk3f+R3f+O2muEsSEZkShX0FTbUFvv2ei7ni7FP45I+f4qN3Pc4BnaUj\nIgmjsI+gsbbA195xATe98QzufGQL137pPp2HLyKJorCPKJ8z/umqs1n9vksYGXXe/vUHed+3elj/\n4t64SxMRqcjiOo+8u7vbe3p6Ytn3yTo0NMIdv93EV+59lv2Hh3n9snm845LTueysDmoL+bjLE5EU\nM7NH3L170u9T2E/d7oFBvvPQZr55/3O8tPcwc+oKXHnuqVx2ZgeXnjGXeU21cZcoIimjsI/R0Mgo\nv+ndwY8f28YvnnqJfYeCA7ivmN/EHy2Yw/IFczincw6ntzfS2VpHTV6zZyIyNVMN+0jn2ZvZ1cC/\nAXngdnf/9Jj1Fq6/FhgA3u3uv5tsMUlVk8/xxrPm88az5jM8MsqT2/by294dPPL8Lh7ctJMfPrrt\n6LY5g86Weha01jG3sZa2xiLtjTW0NRRpbSjSUMxTX8zTUJOnoVgIXhfz1ORz1OSNfM6oyefI54xC\nzgj+6EVEJlYx7M0sD9wKvAXoAx42szXu/lTJZtcAy8Kvi4Gvht8zp5DPcX5XK+d3tR5t23VgkPUv\n7mPLrgH6dh2kb+cAW3cfZOOO/ex8fohdA4OMjE7tN6wjoV/IGYV8DjMwgovCgu8ARs4I19lx2xBu\nU7ouF76XWfJzZJaUMSt+sMZfgVTDX1/YxXtfv3RG9xllZH8R0OvuGwHMbDWwAigN+xXAtzyYE3rA\nzFrNrNPdX6h6xQnU1ljk0jPmcilzx10/OursOzzMnoEhBoaGGRgc4WD4NTA0wsHBYQZHnOGRUUZG\nnaERZ2R0lKERZ3h0lOFRZzhc74A7OB5+D5bBGR09sd0JFoJlx4FRZ9bcAG52VMGsKMRnQxFSFXEc\nz4sS9guBLSXLfZw4ah9vm4WAwj6CXM5oqa+hpb4m7lJEJKVm9Eihmd1gZj1m1tPf3z+TuxYRybQo\nYb8V6CpZXhS2TXYb3P02d+929+6Ojo7J1ioiIlMUJewfBpaZ2RIzKwIrgTVjtlkDvNMClwB7NF8v\nIjJ7VJyzd/dhM7sZuIfg1Ms73H2dmd0Yrl8FrCU47bKX4NTL66evZBERmaxI59m7+1qCQC9tW1Xy\n2oGbqluaiIhUiy7lFBHJAIW9iEgGKOxFRDIgthuhmVk/8PwU3z4PyNrTQ9TnbFCfs+Fk+ny6u0/6\n3PXYwv5kmFnPVO76lmTqczaoz9kQR581jSMikgEKexGRDEhq2N8WdwExUJ+zQX3OhhnvcyLn7EVE\nZHKSOrIXEZFJSFzYm9nVZrbBzHrN7Ja466nEzLrM7F4ze8rM1pnZB8P2djP7hZk9E35vK3nPx8L+\nbTCzq0raLzCzJ8J1XwofB4mZ1ZrZ98L2B81sccl73hXu4xkze9cM9jtvZr83s59kob/hvlvN7C4z\nW29mT5vZpWnut5l9OPw3/aSZfdfM6tLYXzO7w8y2m9mTJW2x9tOCG1M+GL7nexbcpHJi7p6YL4Ib\nsT0LLAWKwGPA8rjrqlBzJ/Ca8HUz8AdgOfBZ4Jaw/RbgM+Hr5WG/aoElYX/z4bqHgEsInk73U+Ca\nsP39wKrw9Urge+HrdmBj+L0tfN02Q/3+B+A7wE/C5VT3N9z/N4H3hq+LQGta+03wcKJNQH24fCfw\n7jT2F3gD8BrgyZK2WPsZ/nmvDF+vAv6+Yj9m6j9Clf7QLwXuKVn+GPCxuOuaZB9+RPA83w1AZ9jW\nCWwYr08Edxu9NNxmfUn7dcDXSrcJXxcILtaw0m3CdV8DrpuBPi4CfgVcwbGwT21/w321EISfjWlP\nZb859nS69rCWnwBXpri/izk+7GPrZ7huB1AI24/LxXJfSZvGKff4w0QIfz17NfAgcIofu+f/i8Ap\n4etyfVwYvh7bftx73H0Y2APMneCzptsXgY8AoyVtae4vBKO4fuA/wumr282skZT22923Ap8HNhM8\nfnSPu/+clPZ3HHH2cy6wO9x27GeVlbSwTywzawJ+AHzI3feWrvPgx3MqTosysz8Ftrv7I+W2SVN/\nSxQIftX/qru/GjhA8Ov9UWnqdzhHvYLgh9wCoNHM/rZ0mzT1dyJJ6WfSwj7S4w9nGzOrIQj6/3L3\nu8Pml8ysM1zfCWwP28v1cWv4emz7ce8xswLBlMLLE3zWdHot8FYzew5YDVxhZv9Jevt7RB/Q5+4P\nhst3EYR/Wvv9ZmCTu/e7+xBwN/DHpLe/Y8XZz5eB1nDbsZ9V3nTOc03DvFmB4CDFEo4doD037roq\n1GzAt4Avjmn/HMcf4Pls+Ppcjj/As5HyB3iuDdtv4vgDPHeGr9sJ5pHbwq9NQPsM9v1yjs3ZZ6G/\n9wFnha8/GfY5lf0GLgbWAQ1hnd8EPpDi/i7m+Dn7WPsJfJ/jD9C+v2IfZuo/QhX/0K8lOKPlWeAT\ncdcTod7XEfyK9zjwaPh1LcG826+AZ4Bflv5jBT4R9m8D4RH7sL0beDJc92WOXRRXF/7l94b/oJaW\nvOfvwvZe4PoZ7vvlHAv7LPT3fKAn/Lv+YfgfNLX9Bv4VWB/W+m2CgEtdf4HvEhyXGCL4De49cfeT\n4IzEh8L27wO1lfqhK2hFRDIgaXP2IiIyBQp7EZEMUNiLiGSAwl5EJAMU9iIiGaCwFxHJAIW9iEgG\nKOxFRDLg/wF0AksxrTlRXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb864bb4d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(1000000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Beta Prioritized Experience Replay</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "beta_start = 0.4\n",
    "beta_frames = 100000\n",
    "beta_by_frame = lambda frame_idx: min(1.0, beta_start + frame_idx * (1.0 - beta_start) / beta_frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fdac674cf50>]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD8CAYAAACW/ATfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFtRJREFUeJzt3XuMXOd53/Hvw10urxIv4lKWeBEvkZPQReM4WyVOg1hI\n6kQS0gopUkAKCidqAkKJZaQt0EZugLRF/2niFnADqWaIQJHTi5WLXVs1mMpt2sZB4diiGlkWbTPm\nDG1eonpnSesyw8tyuU//mLPkcMTdHZKzHM453w+w4Mw57+w8Ly8/vvued94TmYkkqdyWDboASdLS\nM+wlqQIMe0mqAMNekirAsJekCjDsJakCDHtJqgDDXpIqwLCXpAoYHdQbb9q0KXfs2DGot5ekofTS\nSy9NZeb4tb5uYGG/Y8cODh48OKi3l6ShFBHfup7XOY0jSRVg2EtSBRj2klQBhr0kVYBhL0kVsGjY\nR8QzETEZEa/Ocz4i4rci4khEvBIR7+l/mZKkG9HLyP5Z4IEFzj8I3Ft87QU+duNlSZL6adF19pn5\n+YjYsUCTh4Hfy/b9Df88ItZHxF2Z+VqfarxlfKF2ii/UpgZdhqQhN7FjIz/6zmv+XNQN6ceHqrYA\nxzuenyiOvS3sI2Iv7dE/27dv78Nb31y//plX+cZkk4hBVyJpmD3+vt1DGfY9y8z9wH6AiYmJobrT\n+czFWb55qsXj79vNkw9+z6DLkaRr0o/VOCeBbR3PtxbHSuX4d85y4WKye3zNoEuRpGvWj7B/HvhA\nsSrnh4A3yjhfX280Adi9ee2AK5Gka7foNE5EfAK4H9gUESeAfw4sB8jMfcAB4CHgCHAGeGypih2k\n2lzYbzLsJQ2fXlbjPLrI+QQ+2LeKblH1RotNa8dYt3r5oEuRpGvmJ2h7VGs02eWoXtKQMux7VG+0\n2L3Zi7OShpNh34PXz0xzqjXtyF7S0DLse1BrtAAc2UsaWoZ9D+ZW4jiylzSsDPse1BpNxkaWsXXD\nqkGXIknXxbDvQb3RYsem1YyO+NslaTiZXj1w2aWkYWfYL+LCxVmOnTrjxVlJQ82wX8Sx02eYmU1H\n9pKGmmG/iPqlZZeGvaThZdgv4tKyS7c2ljTEDPtF1CabjN+2gttXugGapOFl2C+iPtXyhiWShp5h\nv4hao8mucefrJQ03w34Bp1vTvH7mArsNe0lDzrBfgBdnJZWFYb+AufvOfpcje0lDzrBfQK3RYmx0\nGXevdwM0ScPNsF9AvdFk16Y1jCyLQZciSTfEsF9ArdFyvl5SKRj285iemeXY6TOuxJFUCob9PI6d\nbnFxNg17SaVg2M/jyGR7AzSncSSVQU9hHxEPRMThiDgSEU9e5fyGiPgvEfFKRHwpIv5a/0u9uepT\nc2vsHdlLGn6Lhn1EjABPAw8Ce4BHI2JPV7N/BrycmX8d+ADw7/pd6M1Wm2xx5+0rWLtidNClSNIN\n62Vkfx9wJDPrmTkNPAc83NVmD/A/ATLz68COiLizr5XeZPWppvP1kkqjl7DfAhzveH6iONbpy8Df\nBYiI+4B7gK3d3ygi9kbEwYg42Gg0rq/imyAzqU02na+XVBr9ukD7r4H1EfEy8CHgL4CL3Y0yc39m\nTmTmxPj4eJ/euv+mmtO8eW7Gkb2k0uhlQvoksK3j+dbi2CWZ+SbwGEBEBHAUqPepxptubk8cw15S\nWfQysn8RuDcidkbEGPAI8Hxng4hYX5wD+EXg88V/AEOp1nDZpaRyWXRkn5kzEfEE8AIwAjyTmYci\n4vHi/D7ge4GPR0QCh4BfWMKal1y90WTl8mXcvc4N0CSVQ0/rCjPzAHCg69i+jsdfAN7Z39IGp9Zo\nsnPTWpa5AZqkkvATtFfhfWcllY1h3+XchYscP33GT85KKhXDvsu3Tp1hNnFkL6lUDPsuLruUVEaG\nfZe5m4zv3OTIXlJ5GPZd6o0Wd69byRo3QJNUIoZ9l1qj6cVZSaVj2HfITOoNl11KKh/DvkPjrfO8\ndX7Gkb2k0jHsO8ztieNKHEllY9h3mFuJ4wZoksrGsO9QazRZPTbCO25fOehSJKmvDPsO9UaLXeNr\n3ABNUukY9h1qjSa7NjlfL6l8DPvCuQsXOfn6WS/OSiolw75wdKpFphdnJZWTYV+ou+xSUokZ9gU3\nQJNUZoZ9odZosmX9KlaNjQy6FEnqO8O+UG+02L3ZKRxJ5WTYM7cBWpNdTuFIKinDHvj2m+dpTV90\nZC+ptAx7Ll+c3e3IXlJJ9RT2EfFARByOiCMR8eRVzq+LiP8aEV+OiEMR8Vj/S106l+4768heUkkt\nGvYRMQI8DTwI7AEejYg9Xc0+CHw1M78PuB/4txEx1udal0yt0WLN2Aibb1sx6FIkaUn0MrK/DziS\nmfXMnAaeAx7uapPAbRERwFrgNDDT10qXUK3RZPfmtbTLl6Ty6SXstwDHO56fKI51egr4XuCvgK8A\nv5KZs32p8CaoN1quxJFUav26QPuTwMvA3cC7gaci4vbuRhGxNyIORsTBRqPRp7e+MWemZ9wATVLp\n9RL2J4FtHc+3Fsc6PQZ8KtuOAEeB7+n+Rpm5PzMnMnNifHz8emvuq6NTxZ44XpyVVGK9hP2LwL0R\nsbO46PoI8HxXm2PAjwNExJ3AdwP1fha6VObuO+tul5LKbHSxBpk5ExFPAC8AI8AzmXkoIh4vzu8D\n/hXwbER8BQjgVzNzagnr7pt6o0kE7LjDsJdUXouGPUBmHgAOdB3b1/H4r4Cf6G9pN0et0WLrhlWs\nXO4GaJLKq/KfoK03ml6clVR6lQ772dksll0a9pLKrdJh/9qb5zh74SK7NztfL6ncKh32l/bEcRpH\nUslVOuxrk+2wd9mlpLKrdNjXp1rctnKU8bVugCap3Cod9rVGk13jboAmqfwqHfb1RovdTuFIqoDK\nhn3r/AyvvXHOi7OSKqGyYX9pAzRH9pIqoLJhP3ff2V2O7CVVQHXDfrLJsoB77lg96FIkaclVN+yn\nWmzfuJoVo26AJqn8qhv2k02ncCRVRiXDfnY2OTrlsktJ1VHJsD/5+lnOz8w6spdUGZUM+/qlZZeG\nvaRqqGTYuwGapKqpZtg3mqxbtZw71owNuhRJuikqGfZze+K4AZqkqqhk2M/tdilJVVG5sH/r3AUm\n3zrvxVlJlVK5sK832itxvDgrqUqqF/ZT3ndWUvX0FPYR8UBEHI6IIxHx5FXO/5OIeLn4ejUiLkbE\nxv6Xe+Nqky1GlgXbN7oBmqTqWDTsI2IEeBp4ENgDPBoRezrbZOZHMvPdmflu4MPAn2bm6aUo+EbV\nGk3u2biasdHK/VAjqcJ6Sbz7gCOZWc/MaeA54OEF2j8KfKIfxS2FeqPlShxJldNL2G8Bjnc8P1Ec\ne5uIWA08AHzyxkvrv4uzydFTboAmqXr6PZfxt4H/M98UTkTsjYiDEXGw0Wj0+a0Xd/I7Z5memfXi\nrKTK6SXsTwLbOp5vLY5dzSMsMIWTmfszcyIzJ8bHx3uvsk8u34rQkb2kaukl7F8E7o2InRExRjvQ\nn+9uFBHrgPcBn+lvif0zF/aO7CVVzehiDTJzJiKeAF4ARoBnMvNQRDxenN9XNP1p4HOZ2Vqyam9Q\nrdFiw+rlbHADNEkVs2jYA2TmAeBA17F9Xc+fBZ7tV2FLod5oOqqXVEmVWmxea7Scr5dUSZUJ+zfO\nXmCq6QZokqqpMmFf9+KspAqrTNjX3O1SUoVVJuzrjSbLR4JtboAmqYIqE/a1RpPtG1ezfKQyXZak\nSyqTfO37zjpfL6maKhH2Mxdn+eYpd7uUVF2VCPvj3znLhYvpbpeSKqsSYX9p2eVmR/aSqqkSYX9p\nA7RNhr2kaqpE2NcbLTatHWPd6uWDLkWSBqISYV9rNNnlqF5ShVUi7OuNFrs3e3FWUnWVPuxfPzPN\nqda0I3tJlVb6sJ/bE8eRvaQqq0DYF/eddWQvqcIqEfZjI8vYumHVoEuRpIEpfdjXGy12bFrNqBug\nSaqw0iegyy4lqeRhf+HiLMdOnfHirKTKK3XYHzt9hpnZdGQvqfJKHfb1S8suDXtJ1VbqsL+07NKt\njSVVXE9hHxEPRMThiDgSEU/O0+b+iHg5Ig5FxJ/2t8zrU5tsMn7bCm5f6QZokqptdLEGETECPA28\nHzgBvBgRz2fmVzvarAf+PfBAZh6LiM1LVfC1qE+1vGGJJNHbyP4+4Ehm1jNzGngOeLirzc8Cn8rM\nYwCZOdnfMq9PrdH0VoSSRG9hvwU43vH8RHGs0zuBDRHxvyPipYj4QL8KvF6nW9O8fuaCNxmXJHqY\nxrmG7/MDwI8Dq4AvRMSfZ+ZfdjaKiL3AXoDt27f36a2vzouzknRZLyP7k8C2judbi2OdTgAvZGYr\nM6eAzwPf1/2NMnN/Zk5k5sT4+Pj11tyTufvOfpcje0nqKexfBO6NiJ0RMQY8Ajzf1eYzwI9ExGhE\nrAZ+EPhaf0u9NrVGi7HRZdy93g3QJGnRaZzMnImIJ4AXgBHgmcw8FBGPF+f3ZebXIuK/Aa8As8Dv\nZOarS1n4YmqTTXZtWsPIshhkGZJ0S+hpzj4zDwAHuo7t63r+EeAj/SvtxtSnWuy56/ZBlyFJt4RS\nfoJ2emaWY6fPeHFWkgqlDPtjp1tcnE2XXUpSoZRhf2SyvQGaI3tJaitl2Nen5tbYO7KXJChp2Ncm\nW9x5+wrWrujXZ8YkabiVMuzrU03n6yWpQ+nCPjPba+ydr5ekS0oX9lPNad48N+PIXpI6lC7s5/bE\nMewl6bLShX2t4bJLSepWurCvN5qsXL6Mu9e5AZokzSld2NcaTXZuWssyN0CTpEtKF/bed1aS3q5U\nYX/uwkWOnz7jJ2clqUupwv5bp84wmziyl6QupQp7l11K0tWVKuy9ybgkXV2pwr7eaHH3upWsHnMD\nNEnqVKqwrzWaXpyVpKsoTdhnJvWGyy4l6WpKE/aNt87z1vkZR/aSdBWlCfu5PXFciSNJb1eisHcl\njiTNp1Rhv3pshHfcvnLQpUjSLaensI+IByLicEQciYgnr3L+/oh4IyJeLr5+vf+lLqzeaLFrfI0b\noEnSVSy6ID0iRoCngfcDJ4AXI+L5zPxqV9M/y8yfWoIae1JrNHnP9g2DentJuqX1MrK/DziSmfXM\nnAaeAx5e2rKuzbkLFzn5+lkvzkrSPHoJ+y3A8Y7nJ4pj3X44Il6JiD+OiHf1pboeHZ1qkenFWUma\nT7/2Ffi/wPbMbEbEQ8CngXu7G0XEXmAvwPbt2/v01u35enDZpSTNp5eR/UlgW8fzrcWxSzLzzcxs\nFo8PAMsjYlP3N8rM/Zk5kZkT4+PjN1D2leaWXe7c5Mhekq6ml7B/Ebg3InZGxBjwCPB8Z4OIeEdE\nRPH4vuL7nup3sfOpNZpsWb+KVWMjN+stJWmoLDqNk5kzEfEE8AIwAjyTmYci4vHi/D7gZ4BfiogZ\n4CzwSGbmEtZ9hXqjxe7NTuFI0nx6mrMvpmYOdB3b1/H4KeCp/pbWm/YGaE3+3j3bFm8sSRU19J+g\n/fab52lNX3RkL0kLGPqwn7s4u9uLs5I0r6EP+0v3nXVkL0nzGvqwrzVarBkbYfNtKwZdiiTdskoQ\n9k12b15LsfJTknQVQx/29UaLXc7XS9KChjrsz0zPuAGaJPVgqMP+6FSxJ44XZyVpQUMd9nP3nXW3\nS0la2FCHfb3RJAJ23GHYS9JChjrsa40WWzesYuVyN0CTpIUMddjXG00vzkpSD4Y27Gdns1h2adhL\n0mKGNuxfe/McZy9cZPdm5+slaTFDG/aX9sRxGkeSFjW0YV+bbIe9yy4laXFDG/b1qRa3rRxlfK0b\noEnSYoY27GuNJrvG3QBNknoxtGFfb7TY7RSOJPVkKMO+eX6G194458VZSerRUIb90WJPHEf2ktSb\noQz7+pTLLiXpWgxl2NcmmywL2H7H6kGXIklDYTjDfqrF9o2rWTHqBmiS1Iuewj4iHoiIwxFxJCKe\nXKDd34iImYj4mf6V+Ha1yfayS0lSbxYN+4gYAZ4GHgT2AI9GxJ552v0G8Ll+F9lpdjY5OuWyS0m6\nFr2M7O8DjmRmPTOngeeAh6/S7kPAJ4HJPtb3NidfP8v5mVlH9pJ0DXoJ+y3A8Y7nJ4pjl0TEFuCn\ngY/1r7Srq8/dd9awl6Se9esC7UeBX83M2YUaRcTeiDgYEQcbjcZ1vdGasRHev+dOp3Ek6RqM9tDm\nJLCt4/nW4linCeC5Yp+aTcBDETGTmZ/ubJSZ+4H9ABMTE3k9BU/s2MjEjo3X81JJqqxewv5F4N6I\n2Ek75B8BfrazQWbunHscEc8Cn+0OeknS4Cwa9pk5ExFPAC8AI8AzmXkoIh4vzu9b4holSTeol5E9\nmXkAONB17Kohn5k/f+NlSZL6aSg/QStJujaGvSRVgGEvSRVg2EtSBRj2klQBkXldn2268TeOaADf\nus6XbwKm+ljOMLDP1WCfq+FG+nxPZo5f64sGFvY3IiIOZubEoOu4mexzNdjnahhEn53GkaQKMOwl\nqQKGNez3D7qAAbDP1WCfq+Gm93ko5+wlSddmWEf2kqRrMHRh3+vNz28VEbEtIv5XRHw1Ig5FxK8U\nxzdGxH+PiG8Uv27oeM2Hi/4djoif7Dj+AxHxleLcb0VxA4GIWBERv18c/2JE7Oh4zc8V7/GNiPi5\nm9jvkYj4i4j4bBX6W7z3+oj4o4j4ekR8LSLeW+Z+R8Q/Kv5OvxoRn4iIlWXsb0Q8ExGTEfFqx7GB\n9jMidhZtjxSvHVu0I5k5NF+0t1iuAbuAMeDLwJ5B17VIzXcB7yke3wb8Je0bt/8m8GRx/EngN4rH\ne4p+rQB2Fv0dKc59CfghIIA/Bh4sjv8ysK94/Ajw+8XjjUC9+HVD8XjDTer3Pwb+M+17G1D2/hbv\n/3HgF4vHY8D6svab9q1JjwKriud/APx8GfsL/CjwHuDVjmMD7Wfx+/1I8Xgf8EuL9uNm/UPo02/6\ne4EXOp5/GPjwoOu6xj58Bng/cBi4qzh2F3D4an2ifR+B9xZtvt5x/FHgtzvbFI9HaX9YIzrbFOd+\nG3j0JvRxK/AnwI9xOexL29/ivdbRDr/oOl7KfnP53tQbi1o+C/xEifu7gyvDfmD9LM5NAaPF8Sty\ncb6vYZvGWfTm57ey4sez7we+CNyZma8Vp/4fcGfxeL4+bikedx+/4jWZOQO8AdyxwPdaah8F/inQ\neU/iMvcX2qO4BvC7xfTV70TEGkra78w8Cfwb4BjwGvBGZn6Okvb3KgbZzzuA14u23d9rXsMW9kMr\nItYCnwT+YWa+2Xku2/89l2JZVET8FDCZmS/N16ZM/e0wSvtH/Y9l5vcDLdo/3l9Spn4Xc9QP0/5P\n7m5gTUT8/c42ZervQoaln8MW9r3c/PyWExHLaQf9f8rMTxWHvx0RdxXn7wImi+Pz9fFk8bj7+BWv\niYhR2lMKpxb4XkvpbwJ/JyK+CTwH/FhE/EfK2985J4ATmfnF4vkf0Q7/svb7bwFHM7ORmReATwE/\nTHn7222Q/TwFrC/adn+v+S3lPNcSzJuN0r5IsZPLF2jfNei6Fqk5gN8DPtp1/CNceYHnN4vH7+LK\nCzx15r/A81Bx/INceYHnD4rHG2nPI28ovo4CG29i3+/n8px9Ffr7Z8B3F4//RdHnUvYb+EHgELC6\nqPPjwIdK3N8dXDlnP9B+An/IlRdof3nRPtysfwh9/E1/iPaKlhrwa4Oup4d6f4T2j3ivAC8XXw/R\nnnf7E+AbwP/o/MsK/FrRv8MUV+yL4xPAq8W5p7j8obiVxR/+keIv1K6O1/yD4vgR4LGb3Pf7uRz2\nVejvu4GDxZ/1p4t/oKXtN/Avga8Xtf4H2gFXuv4Cn6B9XeIC7Z/gfmHQ/aS9IvFLxfE/BFYs1g8/\nQStJFTBsc/aSpOtg2EtSBRj2klQBhr0kVYBhL0kVYNhLUgUY9pJUAYa9JFXA/wfDv82JZ6g9swAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdac75bad50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([beta_by_frame(i) for i in range(1000000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Training</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwcAAAE/CAYAAAAT7r+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXmcHGWd/z/f6u6ZSSY3OQg5CCBXQA4JyCUighwquLoq\nuuvBrj88d9XFA9cLXd1Fd1VEWFl2VbyVQxTlUIjcCJIAAcKVcCYh9z1XT3fX8/uj6ql+qvqp6qru\nnpmemc/79ZpXd1fV89TT3cnM9/N8L1FKgRBCCCGEEEKckV4AIYQQQgghpD2gOCCEEEIIIYQAoDgg\nhBBCCCGE+FAcEEIIIYQQQgBQHBBCCCGEEEJ8KA4IIYQQQgghACgORj0icqCIPCIiu0Xkn0d6PaR9\nEZFFIqJEJD/SayGEkHZGRF4QkVNHeh2EjAQUB6OfzwC4XSk1WSl16UgvJoqIXCkiT4uIKyLvt5z/\npIhsEJFdIvJDEek0zs0QketFpFdEXhSRd0fGvl5EnhKRPhG5XUT2Ns6JiHxDRLb6P98QETHOL/LH\n9PlznBqZ+93+PXtF5LciMqOlH8woQEQ+JiLLRKQoIldFzh0rIreKyDYR2Swi14jI3DrznSsiT/qf\n6bMi8pohfQOEEEIIyQzFwehnbwAr406KSG4Y12JjBYCPAHgoekJETgdwIYDXw3sf+wL4inHJ5QAG\nAcwB8HcAvi8ih/hjZwL4DYAvApgBYBmAXxtjzwfwFgCHAzgMwJsBfNA4/0sADwPYA8DnAVwrIrP8\nuQ8B8D8A3uPfuw/Afzfy5kdql75F930ZwNcA/NBybjqAKwEsgvfd7Qbwo4T1nAbgGwDOAzAZwEkA\nnmvBGgkhhBDSQigORjEi8mcArwNwmYj0iMgBInKViHxfRG4SkV4ArxORN4rIw/7u/BoRuciYQ4ea\nnOef2y4iHxKRo0XkURHZISKXRe77D/4O8HYR+aO5Yx9FKXW5UmopgAHL6fcB+IFSaqVSajuArwJ4\nv3+PbgBvA/BFpVSPUuoeAL+DZ7ADwFsBrFRKXaOUGgBwEYDDReQgY+5vKaXWKqXWAfgvY+4DALwK\nwJeVUv1KqesAPOrfD/CEyO+VUncppXrgCZC3isjkpO8j8nn+o4i8BODP/vFjReQ+//NcISIn+8df\nJyKPGeNvFZEHjdd3i8hb/OcX+jvuu0XkCRH5G+O694vIvSLyHRHZCuAiEcmJyH+JyBYReQ7AG+ut\n30Qp9Rul1G8BbLWcu9n/7HcppfoAXAbghITpvgLgq0qp+5VSrlJqnf+9EEJI2yIinSJyiYi87P9c\noj3cIjJTRP7g/17f5v++dvxznxWRdf7v66dF5PUj+04ISQ/FwShGKXUKgLsBfEwpNUkp9Yx/6t0A\nvg5vh/YeAL0A3gtgGjwD8cPa4DR4NYD9AbwTwCXwdtNPBXAIgHeIyGsBQETOAfCv8IzzWf79f9ng\nWzgEnmdBswLAHBHZA8ABAMrGe9LnD7GNVUr1Algdd94y9jml1O6Ucz8LoOivKS2vBXAwgNNFZB6A\nG+Htws8A8CkA1/meivsB7O//kSnA83LsJSKTRWQCgCXwPmMAeBbAawBMhWds/ywSyvNqeLvxc+B9\n//8PwJsAHOnP87fmAn2x8YcM7ymJkxDjwfK9V0sAzBKR1SKyVkQu898fIYS0M58HcCyAI+B5oo8B\n8AX/3AUA1sL7WzgH3t9GJSIHAvgYgKOVUpMBnA7gheFdNiGNQ3EwNvmdUupef4d2QCl1h1LqMf/1\no/CM+ddGxvybf+2f4ImJXyqlNvm7u3fDMzAB4EMA/kMp9aRSqgzg3wEckeQ9SGASgJ3G613+42T/\n3K7I9bv8c7ax9c7vAjBJRKSBsdHzabhIKdWrlOoH8PcAblJK3eR/B7fCC4M6yz//IDzj+ih4ouRe\neLvwxwJYpZTaCgD+Tv3L/hy/BrAK3h8qzctKqe8ppcr+vO8AcIlSao1SahuA/zAXqJS6WCn1pgzv\nyYqIHAbgSwA+HXPJHAAFeOLkNfD+yB6J6h9YQghpV/4Ontdzk1JqM7yNGe3BLgGYC2BvpVRJKXW3\nUkoBqADoBLBYRApKqRf8TSZCRgUUB2OTNeYLEXm1eMm3m0VkJzwDf2ZkzEbjeb/l9ST/+d4Avuu7\nUXcA2AZAAMxrYJ09AKYYr6f6j7st5/R5vduf9fxUAD3+L+5m506D+R3sDeDt+jPzP7cT4f1RAYA7\nAZwMTyDcCeAOeOLttf5rAICIvFe8ylR6jkMR/h5D3zuAvSLHXsyw/lSIyCsA3Azg40qpu2Mu6/cf\nv6eUWq+U2gLg2wDOavV6CCGkxeyF8O/OF/1jAPCf8DzWfxKR50TkQgBQSq0G8Al44a6bRORXIrIX\nCBklUByMTVTk9S8A3ABggVJqKoAr4Bn0jbAGwAeVUtOMnwlKqfsamGslPDet5nAAG/2d8mcA5EVk\n/8j5lbaxfo7CfnHnLWP3jeQQJM29H4AOf01pMb+DNQB+GvnMupVSF/vno+LgTkTEge+Z+V94ruo9\nlFLTADyO8PcY/d7XA1hgvF6YYf118dd0Gzyv00/jrvPzSdZG1hddKyGEtCMvw9vg0Sz0j0EptVsp\ndYFSal8AZwP4F51boJT6hVLqRH+sgleQgZBRAcXB+GAygG1KqQEROQZeTkKjXAHgc1KtGjRVRN4e\nd7GIdIhIFzwjtiAiXTphC8BPAPyjiCwWkenwEn+vAoIcgt8A+KqIdIvIifB++Woj9HoAh4rI2/z5\nvwxghVLqKWPufxGReX7M/wXG3M8AeATAl/31vBXAKwFc54/9OYA3i8hrfNHxbwB+E8lRyMLP/PlO\n95OEu0TkZBGZ75+/D8CB8EKE/qqUWgnvD8qrAdzlX9MN7w/MZv9zPQ+e5yCJqwH8s4jM9z/fC7Ms\nWkTy/mebA6DXnffPzYOXbH2ZUuqKFNP9CMA/ichsfy2fBNCqfAdCCBkqfgngCyIyS7wqeV+C9zsd\nIvImEXmFH666E144kSte/6FT/MTlAXjeU3eE1k9IZigOxgcfgWdk74b3i+3qRidSSl0PbwfkVyKy\nC97u9ZkJQ/4E7xfj8fBKX/bD2yGHUuoWAN8EcDs8V+3z8Ix8c90TAGyC5/34sG84w4/9fBu8xNvt\n8Azrc42x/wPg9wAe83/+4B/TnAsvSXY7vFj8v/XnhH+PD8ETCZvgGeYf0QNF5AoRSWMQw59vDQCd\nyL0Znifh0/D///lC6CF41ZcG/WF/AfCiUmqTf80TAL7lH98IT8zcW+fW/wvgj/DyGB6CJ7YCRORf\nReTmhPFfgPd9XQgvb6If1TyBD8ArPXuReJWyekSkJ2Huf4OXW/EMgCfhlZH9ep31E0LISPM1eDli\nj8L7W/KQfwzwinjcBi8U9S8A/lspdTu8fIOLAWwBsAHAbACfG95lE9I44oVgE0IIIYQQQsY79BwQ\nQgghhBBCAFAcEEIIIYQQQnwoDgghhBBCCCEAKA4IIYQQQgghPhQHhBBCCCGEEABAfqQXYDJz5ky1\naNGikV4GIYS0JcuXL9+ilJo10usYSfh3ghBC7LTqb0RbiYNFixZh2bJlI70MQghpS0TkxZFew0jD\nvxOEEGKnVX8jGFZECCGEEEIIAUBxQAghhBBCCPGhOCCEEEIIIYQAoDgghBBCCCGE+FAcEEIIIYQQ\nQgBQHBBCCCGEEEJ8KA4IIYQQQgghACgOCCGEEEIIIT4UB4QQQgghhBAAFAeEEILH1+3Elp5iQ2Mr\nrsI9q7a0eEWk1dz/3Fb87pF1I70MQghpeygOCCHjnvOuehDfv+PZhsZe9ufV+PsfPID7VlMgtDPX\nLl+Lb97y9EgvgxBC2h6KA0LIuKdnoIztvYMNjX1m424AwLa+xsYTQggh7UTT4kBEFojI7SLyhIis\nFJGP+8dniMitIrLKf5ze/HIJIaT1DFZc9BTLDY8FgLzDvRZCCCGjn1b8NSsDuEAptRjAsQA+KiKL\nAVwIYKlSan8AS/3XhBDSVlRchYqrGhYHJV8cdOSllcsihBBCRoSmxYFSar1S6iH/+W4ATwKYB+Ac\nAD/2L/sxgLc0ey9CCGk12rjvbVIcFHL0HBBCCBn9tPSvmYgsAnAkgAcAzFFKrfdPbQAwp5X3ImS8\n8aeVG7B+Z/9IL8PK0ic3Yu32vobHK6Vw7fK16BtszEAHgIFSBdcsWwOlVOI1Vy9bg3U7+nHjo+tx\n9bI1KJY94z6t5+C65WuxtaeIq/17lSre/bb0FHHrExuhlMJ1y9di10ApNO7xdTux/MXtDb47Qggh\nZHjIt2oiEZkE4DoAn1BK7RKputiVUkpErH+xReR8AOcDwMKFC1u1HELGHOf/dDlmdHfgoS+eNtJL\nqeEff7wMkzrzePwrpzc0/v7ntuFT16zA8he34T/eelhDc3ztxifws/tfwtypE3Di/jOt11x881O4\n6r4XQsf0b6reYqXuPZ7esBsXXLMieD2lqxB4Dj75a+/4H/7pRFxwzQqc/cxeuPRdRwbXvul79wAA\nXrj4jWnfEiGEEDLstMRzICIFeMLg50qp3/iHN4rIXP/8XACbbGOVUlcqpZYopZbMmjWrFcshZMyh\nd8O3NVhRZzhoNGbfHLtxV2O9BgDg5R0DAIBiOd7It3le9GeaJqwoGjm0pacYiIPqOrx7rGnCk0II\nIYSMFK2oViQAfgDgSaXUt41TNwB4n//8fQB+1+y9CBmvJETKjDhJYTxpkchjI2gjPZ8Q+x+x4wEA\ng35YUe9gue57iZ4eKFVQroQPvrC1N8VqCSGEkPakFZ6DEwC8B8ApIvKI/3MWgIsBnCYiqwCc6r8m\nhDSA28bqoOK2x9qCxGAnXmJU3Fp1oEuRugroLyWHFpUj77VYdoPxmuc2UxwQQggZvTSdc6CUugfx\nG36vb3Z+QghQMcSBUgpmTs9IEzWYm6GZmXRicCEfv+dhW6v2HABeeNPEjvhfi1EhVCxVQuOBqjho\ntPoRIYQQMpKw9h4hLaRUcfGdW59BT7GMwbKLb9/6DHZHqtZkpVzx5tH0FMvYvLuIy/68KjYM5pE1\nO/CZa1fg8XU7ccltz+D5Lb343tJVeHzdTly9bA0AYOXLO3HNsjWouAqX3PYMdvaH11muuN7xPu/4\no2t34NPXeHMCwPbeQXxv6aqamPtmcV2FS5euwo5Ix2F9fFvvIO54ehNufyqcxlT21yEAdvaVcMlt\nz9QY8zYPTNEw7r9x89PY0mPPe/jxfS/gS797vGZs9P0/t6UHAPDMxh48/BKrExFCCBldtKxaESEE\neHL9Lnx36Sos3msKNu8u4tKlq1BxXXz69IManvOmxzfgf+58Lni9o6+ET12zAg88vw2vP3gODp47\npWbMbx5ai6uXrcVfn9+GF7b24ZLbVgEAvuWLjHcsWYA3XupVz5nR3YFLbluFF7f24TvvPCKY48bH\n1uOS21Zhe+8gvnLOofjNQ+twzfK1mNxVwKHzpuLzv30MNz22AfvNntTwe7Nx77Nb8O1bn8HTG3bj\n8r97VXD8/ue34tu3PoMnXt6FW1ZuABCu/DPoew4qrsLXb3oCVy9bi0P2morTFlerKEfzA7xxVeP+\nuofWordYxhXvOarmui/fsLLmmC3nYEtPVdRcs3wtjlwYbg7vugpOQugTIYQQMpLQc0BIC9E71QOl\nSlAZpyOXa2rOYiQO3lUKL23zKuF0xoTQ6B3y3QP1Q1u0oRqthLRhp1f9Ryf4RpuF9filPwfqxOmn\nQZvXgqoB3xvT82B7n71ik/YcVFyFvkFvTdHKRVbPQSm88x/NIUiiv1SxXn/U3tOx19SumpCjrPMT\nQgghww3FASEtREexFEtuYGzvMamjqTmj5mzZVdjqzx2XDKxtYJtxGmVCwRMv0WTc6Pr1vXoiRnsr\nwopsicJRO14LoTjjWucTVJSC4+dkRD8f2+cVna+rUPtrMe5ztoUVAYAjQEfesZ5rhZgihBBChgqK\nA0JaiN6ZLpYrQXjJxI7mPAc193BVYPTHGcqBSIk5bxq72uiOGq16/VMnFABUjW/tOdCRMYNGWE25\nQaGgk4kVEJQ3iJrj9QSPPl5xVbC2qMCwioOId6ErX/t9xd3TFlYEAI4ICjnHOm6gRM8BIYSQ9oXi\ngJAW4gZhRS62+omtTVfziQw357MZpuaguF1902jV0/UPho3krb3F0PlyJKxIR81XjHuk6TJso2x4\nDvS80WTrQBAZazevKQUlSaueg2gYUb1qRQDQWagVB3G7/QMl1zqnI4KOvF0cJDVpI4QQQkYaigNC\nWsTFNz+Fh9fsAOAZk/VCf0y+ectTeMQfa7JmWx8+/9vHQsfM+cqWcBwA0Ifj2iP8/IEXq9f6F8WF\nFWnjXxvBD76wHb95aK2xhupNbn96E77rJz8/8NxWfMeosgQA1y5fi8/95lF89tpHceF1jwbGc6lc\nnSOuTKv2kvQZIkZXGlr2wjZs2u2Jmc9c+xjW+/kSrgIuXboKf3l2q/deLN9FMSoOjDyOTbsG8Olr\nVmBHv73i1J3PbLYezzm+58AaVkTPASGEkPaF1YoIaQFKKVxx57PB64FyJdiJr+c5UErhv+94Fv99\nx7Oh6jsA8IlfPxKE3GjM+aLngjnrdAz42o1PBs+1wRz1HGzZHfZ8mF6Kf7l6BU45aDaAcGjTJ379\nCADg46fuj3deeT8A4JOnHRCc/9Q1K0L3OPXgOTh18RyU3GoZ0uA9RN6CFhJmidOeYhldhRz+9oq/\nVNfdUwzKkVbcahnYFy5+ozUhObq732V4Di5ZugrXLF+LmZM7a8Yl4TjxngPmHBBCCGln6DkgpAVE\n7X+zAk69OPwkz4JtD92cLy5sKEskU5znYIvvOQjEQWRSvTZz119jhvskJUVrz0fZzDmIoRRUMaqu\nU4c45WNKg0a9Atawoshn2JGrztXt54ts9D0RaXEE6MjZE5KjayKEEELaCYoDQlpAdEd6wIgrrxdW\nlORZsNXDHwgJj+RqRWnQIUimOBgwOv/q9UcrCunoH5sBPBjKQ4gvp6oNfnOOuA4Ag5XaHfcef26d\nNB13XmNPSA6v37xEz7thVzZxkNM5B+O4WpGILBCR20XkCRFZKSIft1wjInKpiKwWkUdF5FW2uQgh\nhAwfFAeEtIAacWAa8M2IA4ulbCa0xnkO6oUVmVT8tZtvYfPuapfgUiTnoIq3OJuxa77/qIEeurfF\nKyFBtaLw/WweCp0AHScOdHfn6P1MouKgZIggHWK0MaM4EBEUcmJd83gRBwDKAC5QSi0GcCyAj4rI\n4sg1ZwLY3/85H8D3h3eJhBBColAcENICojv1xbIbGLl1w4piKw4hqLoTnVsTJyyyeQ5qLzaN4Yol\n58AkGo7krdEI/YlpZgZUPQylsuk58N5zzWdq+Ry1V2LqxBhxEEkktr2HaJiP+X3oc5sMsZSGnAN0\n5HMhz0GHLhk7TsKKlFLrlVIP+c93A3gSwLzIZecA+InyuB/ANBGZO8xLJYQQYkBxQEgCX/jtY/je\n0lV1r4sasgOlSnDMZsBf/eAafOwXDwEI71T/+sGXguOAV/UmSpLn4D0/eAB/ePTlmjKgSdjWt3FX\nseZ8dNddL80qDgzPwRmX3B0bWqSPl/y5//zUJvx+xcsAgPue3RqqilSyGNXaK9HdYa+tsCPiObCV\nES2WK6HP2fw89C5/mk7TJjlH0BHpc9Dpd5qOdrweD4jIIgBHAnggcmoegDXG67WoFRCEEEKGEYoD\nQhL42f0v4VuRcpw2asOKKsExWyjLZ657FH94dH3N+c9e91hwHLCX9TQN72i1ortXbcHHfvFwpoRk\nm2fD3O3X50uui8VzpwAA5k2bEHhGipbSnNHQGVuZVqAqDsw1/HpZ1Vb8l6ur1Y30LvyZh+6JNyye\nA6AqTOJ6B0Q9B7bqTrsHyphYyOGq84721mKItUZDgEQEHXkJeQ4mdXkCZv70iQ3NOVoRkUkArgPw\nCaXUrgbnOF9ElonIss2b7eVjCSGEtAaKA0JaQFQcFEuukWwbb6krpRJzDnKWnAPTYI0LWcrSdq1k\nub8ZamR6DuZM6cQ7lswPCZq4nAPT6VHI2X/V9BTTlXsFqp6D773rSPz7W18JoLoLXyy7OHTelJox\nteLAXj2ouzOPkw+cjdmTO0PvrdHKQjmRmmpFk7vyOPPQPXHcfns0NOdoREQK8ITBz5VSv7Fcsg7A\nAuP1fP9YCKXUlUqpJUqpJbNmzRqaxRJCCAFAcUBIS4iatsVyJbbKj0l/qZKYk1Av58Bm2AO1YiUJ\n2/oqxngz5yDnOCjkHJRdN8gNiMs5yBuCoC8m7yAIK6qTlwF4ngMRL2RHJwrrxOeBUgUTLaFFacQB\nAHR3evPlHQnlJTTqOQiaoEU6Ucf0dxuTiOf2+gGAJ5VS34657AYA7/WrFh0LYKdSan3MtYQQQoYB\nNkEjpAWoiM05UHKrNfwTdsV7imXreddVcByxhhWZBqcpLEJ5BhlcBzbPhrkkfb7susgbRq9eWpzn\nIO8IdLuy3mLFmgfRk0UclF105ByISNDFWN97oOQGPQlMTHHguio23GpSp/erMJeTSM5BY54DEdQ0\nQVNKxXZ/HqOcAOA9AB4TkUf8Y/8KYCEAKKWuAHATgLMArAbQB+C8EVgnIYQQA4oDQlqArc+B3oGO\nq/IDeEaz7XzJddHp5GCLxhmISUg2Dd8sngPz/tqA1WFFHXkn8CyUXYV8TrzddVcF4qA/JufATPLt\nLZatIqQnyDmov97BiicOAC9MKedI4EUZKFUwsbP215me35Fw4neUbn9swXGsCclZyYn4HhYVCD2l\n4ns4jEWUUvegzltWnmL86PCsiBBCSBooDghpgJe29uGk/7wd13/keFx881M1MfUDpUqos/Anf/0I\nbnxsPT568ivwnduqCc69xbI11ORfrl6BPz6+wepVuPz2Z4PnpYrCHU9vwvt/9CD++vnXB8ezlDI1\nE3Bd5eU56FCizpwTEjl5R5A3jun3GmWgXAnlLfQUy1bvQG+xjLdcfm9swjIALLrwRlx13tGe5yBf\n/Zy78g4GShV89BcPYdPuotVzYL6vA79wS+x5LQ5yjuD3K17GPntMxLSJHfjTExtjxyTh+E3QAE/U\ndDk5KNjDxAghhJB2guKAkAa4c5VXMeXa5WvxwPPbas73DZodkl3csOJlVFwVEgaAZzRPtBi1j7y0\nI1WSbrmicNV9LwAAHnpxe3Dc9BwcPn8qliyagR/c83zsHNW1KuQcCcZ3FpxQQrKXcyAouW4gIGzi\noFhyMVB2se+sbjy3uRe9xXJNszHAE05JwkDz3aWrsP/sSSER1lnIYaBcwY1+dSdbzkFaJhniAAAu\n/fPqhucCvM7W2stRqrjoKuTgKjWucg4IIYSMTpiQTEgTxJnvZl1/L5HXbhXGhdvEleaMUnZdzJjY\nAQDY0jMYHDdn3GdmN045aHbsHGZ1noohBACgI+cYXYxdFHJeuIxS1dwHW0Jy72AZFVfhLUfMQ0fO\nQc+g3XOQNjdi8+5ijOegOqdOKm6EICHZVh7KpyOm4pKJNv69Jmi+58D/nJSi54AQQkj7Q3FASAPU\nM/HMTf+y64Xj2Ogplq19ENImwg5WXEzzxcFWUxwYU3qJzfFzmLH4OsSoEngOcoFRr0WONqD1GvsH\na8XBTr/5WFfBwaSuPHqLZWtZ0LS5EZt3F1GqKBQM472rkGu41GiUbku+Qu019cVHV967xgwr0uLP\nVWpc5RwQQggZnVAcEDLEVFyFXIx13lushGL+NWkTYcsVhakTCgCAbb3VrsZmZaCcSFB2NG4Oc61A\ntc9B2HOgUMg5KDjer41+o8dAlB39Whzk0N2ZQ8+A3XPgqrDBH0ex7KJYdtGRrxroHX7OgWZXf7Yu\nxiaT/JCkpMToNAKis+B9No6fkAyEPQfjrFoRIYSQUQjFASFDTKniIhdjAPcWy1aDNE2+gZ5b7/Jv\n6bWHFeUcQYzjouZe1bAi73VHPppzYHoO4gXMDt9z0Jl30N2Rx7a+EnYP1BrvCumTp7f0FNER8RyY\na4j2NMiC7l6c9LlPSiMO8lVxYCYkA7oSVMNLJIQQQoYFJiQT4vOPVz2IpU9twgsXvzH1mDSGbZLn\n4Os3PZn6XjZ+8pcXcdCekwEAm3YNBMfNcJ24fgkas1dCxVX4xK8exm8feRmAFgfe+VKl2ucAAJ7a\nsDt2zh19nlDpKuQwbWIBdz2zGXc9s7nmOlelDy16ZM0OHLFgWvC6q+CEvBZ7TZuQah4b3b7nwBbi\npTly4fTE9wwAnb5nI+cgEDLac+AqJIo0QgghpB2g54AQn6VPbUp9bZYd4LIbn5AcZeakjvQT+2iD\n1cxTMO3tnIQ9BwfMmYS9pnaF1mc+18JAJNwxuOL3OUgTBrRuRz8Az1j+6jmH4rUHzKq55rD5UwGl\noAC8+fC9cN2Hjw8lHNv48Mn7Bc878zkUSxV05B0cNn8qPnna/nXXFUeXXzEqGuJ13gmL8ODnT8XP\nP/BqXHT24rrzBJ4DR4LkYy1+FBQTkgkhhLQ9FAeEDDFl100tDt502F4N3yfcEM3IOYgkJJ/4ilk4\n85VzrePMnfOceCFEFVdBKeWLHAd5x/5rw6zm8+LWPgBeDP4BcybjVQunh649YsE07NHdAVd5Qmaf\nmd04au/pONLwDNhYPHdK8Lyr4KC/VMFg2cXrDpwd7NrXo6tQu/4u36iPhngdPn8aZk3uxAmvmJlq\n/q5CNSFZe2v0V+GqbKKSEEIIGQkoDghpinS9CNKGzkzuajzSz1oqFJ5BaoYV5XMSEiu2UqaAt/ud\ncxyUXBUczxs5B9F7dBq7/jpZWVfv6YwY5I54BrT+XPRykkqJAgitu6uQC/IMtFFuEtcUbVJnoeaY\nHh/NOchqzOvPwPTWKP/fCBOSCSGEjAYoDghpgrQ5BzF2ew1pKuLEYfZLcGuqFVXJGSEv0XHliOeg\n4Agqrhscz+fEWu8/TjToXfqufFQceN4MLTr0CnMxXgnzPprOvGOIg9pxexqhUyY2AaaN+mjOQVqP\nj6YjSEiGEVbknVMsZUoIIWQUQHFASEa29Q4GRuTm3cU6VwMlN73noDlxEJNzEBEDORGY9n00IVnj\niDe2XFGtcAbGAAAgAElEQVRVceAI8lZx4FiP6x356M6+DruJeg7qpTM4Ec+BzrOweQ7ixIGt6pAe\nH/W+xCWSx6GTtR1HgmYY1ZwDNkEjhBDS/rBaESEZcF2FV/3brcHrNEnMFddNrIJjsveMiQ2vLS7n\nwImKA0dC4qESSkgOG8f5nKDsKlR870LOcawegpzjeRmi6B35mrAix7OdA89B0Fk4vedAN38z72My\nfaI9uTtJHES/p6QwoAPnTMbTG8PVi7Q48MKKojkHLGVKCCGk/aHngJAMlCwNy5LozDtezkEKcfCr\n84/FfrMnpZ772g8dF16bER4UrVZkGqV5R0I78GYokfn2FDyPQMVVgWgo5CRogmaSi/EoBJ6DfK3n\nwMs58F5rIzyuk7R5H80HT9q35j6auz/zutiQIJt3RosL/VkEnozIHA998TQs+8KpuO/CU3DdR46v\nmacj713vGL0ldEM6peg5IIQQ0v7Qc0BIBpI66NroKuS8nfcUYUUHzJmc2sMAAPOnh70Mg0bNf9OL\n4ESqFeVyAhj9y0xvgflcKc9YL1WqOQc5x17K1BFYjyeHFVk8BxkSkrs78+juyKF3sFKTc7BgxsRY\nQ3xSZ20IUtRzkM85GCy7iOqdGd3JpWZ17oRjdKV2Dc8BIYQQ0u7Qc0BIDMpizGUXB46fkFx/XFyy\nb9LcJqYgMJuD5QxDVb82PQemx8Fcp4IKSplqcVBwHGuoTc5ojmaeDsKKImE/4ifsVoKdegnWlkR0\nJ3/qhII/f63BHzfVhA5bWFE4IVl7MDJXF/Iv9xKSvefBvyN6DgghhIwCKA4IicG20TuYtuyQT2c+\nh7Lrpto1zjtStwlY6PqcEwrDMcODQuLA8WL8q6/DOQhmQrI5h1JeDkA450BCHgqNI9VqRWZMv96R\n77R4DhCqVuS/pzphRdEeC1P9vAJbtaI4Q3yipcRpVFzodWRNSNbkQgnJ+pE5B4QQQtofigMypimW\nK3hy/a6GxtpCgaIJu/XoKjhYs60/tDsfR1zIThxx5UMB731rHCfsOcg7kWpFhiBYt70/eK78azfv\nLmJb36A3NiehuYN7iASG+5Suah8BvdMfNd61QKmoiOegjjiInp7mew7M7tBx12omWCobRT0bOn+i\n0Z1+nVMBGH0OEtZECCGEtAsUB2RM8/sV6/Hm792D3QOlzGNtu/1Zw4rmTLGX07RR8MuBprVH8449\nORgIG8tmQy4AyOWciOeg+p4uuGZF9UIF7DHJ25m/4o5nvbGOYNHM7pr7mcImqRqQRuAZym4k56Be\nE7RomM9bjvQ6Ss+1lC2NM+wnWDwHOszqb46cB6Dq0ahTPKl2fcG9q/cPVyuiOiCEENLeUByQMc3u\ngRLKrgqF2aTFFgkU14U4js+ecVDqa7WBWi+0RpNL8BwMDFZC14USkiXcITnJG/JPp+wPANjuew5y\nIthv1iQ8etEb8MRXT8cpB80GUM0hAIBJCU3GNH5UUeA50EZz1p36dx69ECu+/AbsO6u2ylPcVLaw\nIs1//u1h2HuPiYE4aCasSA91jWpF1AaEEELaHYoDMqbRRl4jlWJsScTlDNWEAM8oTmvsa3IpRYJI\nNQk4Sn+pErrO3LH2worsCckmCgo5RzB7cif6fLGhBcyUrgImduQDb4GZx2ALDbI1KTMTktPmHNjQ\nSclR4nbpbWFFmnzOQXdHPhBMTsb16HuKmKVMq4/CHsmEEELaHIoDMqYJNEEDVSRtgsKWjJtEdNc+\n1ZiU8fcAYsWBKWJy1dzYYF5zhz6ukpJ++12FHPoGy6G1acwqQzoEx7ZqmziAEVZU7SvQul9JsTkH\nCZ4DAKESq1k9Gboykfe961KmOudAMeeAEEJI20NxQMY0Omwl44Z/7Ji0ngO9O513nMy7xdogTbOL\nXi9GH6gVA/lc1HNgFzz6nXYVnMBzEBUsemfdDCuyGdS2DsahhGTtLcmQkF2P2JyDBM+BHhdthpYV\nr3ysRzXngKVMCSGEtD9sgkbGNOaubeaxtrCilDkHe0zqwNrt/Y2Vrwx20VOIgxTXRJugOSIhD0C9\nHgyd+Rw2FAeCuUxML0c1pKZ2jloPh2c863SHaiLv0IuDeuViRaoGfZrvIDy2Vixd//A6VFzFUqaE\nEEJGBfQckDGN62b3HKjA21A7KE1JUgD459d7ibzTuztiDcLY4/5j3hIyFI2vj9b9t5GTsOcg54Sb\noMV5Q/Tn0FVwghyG2rAi/SjIGc8PnjsFh82fmrAqFfIcSAZvSRKHzZ+K/WdP8ue0X7N47pTQ631n\nhasvmbkKacXKrMmd2HuPasdqM5zshhUv4wM/WeYnJFMdEEIIaW/oOSBjGm332rodJ43JSXN9Dt6x\nZAHesWQBgKqBae5I6+P6HmZ/gyRD+c5Pn4xpfuMvACj565nUmUdPsWxdi81zYM5cdl0UclIjfPRn\n11XIBeeiS9Iiw6zrLwLc/PHX1KzjhYvfiP+7+zl87cYng+sqkVKmWXfqo9zwsROra7MY4vdeeApm\nT+nCk189Awd/6RYAwJ8vODl0jTkqrTj47UdPwLxpE/DPv3w4GGcbS2lACCGk3aHngIxptPGZpViR\nUvFjspYyBaoGYTTW3dyFNw1+bR/bxEE0rEcnSMdV7NH3kYjnwKRcUeiISWwGwvkC0fubFYrMSj1x\nmAazbYe+Wc9B+F61x3KGgEkzLq1YiXpUROz3YM4BIYSQdofigIxpkgz9ONwggbTxsCITbQRHxYFp\nJ84wxIG+PmdJzo0al7p/w5QkceBEmqA54XuXKioxDr/TWHfUWNZGsdf0C8HzOMxz5hqCnIMWigOb\nSNFRWElGf1i0pLtXNLorrmwptQEhhJB2h+KAjGkqDSQk62utfQ4aEgfeY7ScpxPyHFSN+yDnwJJP\nEN2hLvq5ANMSxIHjSMhQjRrNXlhR/K+Crnx13VFxEoQVGRWRkuzfqvEfFixD4TmwhSolVVSqXmM8\nz+g5CIVvWT5SljIlhBDS7lAckDFN1QuQfozZtCpK2pwDE20PdhbC/91MQ3FGt+k58B5tu9tRm7aY\nMqwo5DmITLKjr5ToOegy1l1TylSv1ei6nGR4m8IktLPeopyD8Nq8ucx8DtPTkWaNaTskR4Wc8hOu\nk+YmhBBC2hGKAzKmcYOcgywJyc1XKzLRu8/mDrx5HAA+cvIrjDPxu+hRg3OwkkIcOAht5ztS25ht\n7fb+2PGdxrqjxrIpCPSpdPH8yuo5aK048B4LhuFeTZpOEDChOdKtx9YAzjaS2oAQQki7Q3FAxjRu\nA03QtCawhRXZEpK/e+4RifNpAzPamVcf/9F5R+OVRtnPqOfANJijxrNe69SJ8eJAIpVz6lU//eSp\nB4Rem56D6NggTMdJZ3jHJSRXQ6mGwHOQj19/0ri01wNxXh7LMdYrIoQQ0ua0RByIyA9FZJOIPG4c\nmyEit4rIKv9xeivuRUgWqrZ8I56D2nO2Jmj1eg1oc7ArElakbceoQZxkKMfZznWrFdW8jjdSuzvD\nIqYrISHZjOHXp5LM31xIHBjzBInCrduv0Ma5WYkpjWciWvY1DbbrbLdizgEhhJB2p1V/ia8CcEbk\n2IUAliql9gew1H9NyLDSiOcga7WivKWqkIm2G6PViuJCaQLRYEkSjtuVr1+tyNwNl8QE7e7OcPuT\nUM5BTVhR7T2Scw6qz0NehIRQqkYJSsIa308aYz/aMC4NtnVbPQcUB4QQQtqclogDpdRdALZFDp8D\n4Mf+8x8DeEsr7kVIFtwGSpkiQRzYEpLrGbTaSOyMEQfRSkHaUM4Sfz+lK76foa0JWhITI+FPZs5B\n1OA1BUG1z0H83NV7h30X+nBLS5laPsc04qARz4Et3MvuOaA6IIQQ0t4MZc7BHKXUev/5BgBzhvBe\nZBxzz6ot+OjPH8KT63fVnEtKLo5DX3v57c/WnLN5DuoZtPpsredAP8Z4DjIYykmlSAW1tfsTw4o6\nEjwH0bCiICG56kVILBPqmM9r8w+GwnOQlLNho5E+B7YGcMwvIIQQMhqJ325sIUopJSJW60xEzgdw\nPgAsXLhwOJZDxhjXLl+DGx9bj4P2nIyD504JndMpAtmaoCkMll38fsXLNedsCcn1doP1+WjOgROz\n056lcs8Vf/8q3L1qS52mXuF72K79/FkH48n1u7Bh1wAW7jExdM4scxoNKzIN8OBzSOU5UJGqQN5j\nksjJihYf4WpF1fPnHr0AJx0wq2ZcKD+jzndw1XlH4/qH11nPsUMyIYSQ0chQioONIjJXKbVeROYC\n2GS7SCl1JYArAWDJkiXZ60QS4lOxKADVgOdAwS4CAHsTtHq18ONyDrThGTc6zS76GYfOxRmHzsXS\nJzcmXhfKObCs933HLwpEwLod4bKmZpJwNF+42jegGlaU3GCs1lsAVHfZOw0h8vW/ORSfv/5xNIqt\nX4R5z4vfdljMGs05kr+Dkw+cjZMPnF1zXCm7R4nagBBCSLszlGFFNwB4n//8fQB+N4T3IiToaWBi\nK0dadx7fc2CjZMk5qGfDx4UVxRmKSQnJcZgGcCGSIC2WbsTRe5tCJCp2CglhOdoAFql+DkkfR1gc\nmMe9x1BlpCYt6UZ7J4SaoGUdG/M8uiZCCCGkXWlVKdNfAvgLgANFZK2I/COAiwGcJiKrAJzqvyak\n5Wj73+Y50OeyJiRn8RzU212ul5Bce733qA32NA3czHKqtupHZvy7zeB1EgRALkE42DwSSfZ09ZxY\nd+jDPRWaFQfeY1YDP6mbdFoU0pc3JYQQQtqJloQVKaXeFXPq9a2Yn5AkdMiQzZ5vLCE5SRzUHq9n\nfGobsSsmIbnm+gaqFYU8B46DAYTXGc45SJ4rGs5k5gFEDXZbsm9yKVN7wq4+nNSNOSt6HdmTnO3e\njYxD7d8vPQeEEELaHHZIJqMelVB6NChlmmE+VymrONjZX8KP//JizfF6tmdcQrIWC7XlQb3HLEat\nWcvf1nfBvIWIJAqPXC5BACR4DowqpbHYQonMeczPqF7/iHqkyYGwEfIcZBQW+UAgwfo50HNACCGk\n3RmWakWEDCVVz0F8zkHWhORBizh4bnOP9fp64S/6dDTn4NJ3HYlrlq3BYfOmho5ro1YnAisA13zo\nODy2dmfsPUwj1parEGrsJYLTD9kTJ7xiD9y7emvtXBFj2hQp0fdqM/ATPQfBMxXxIniY3pVmKxdJ\nzZN01EveTuJzZx6MiR15vOmwvbBroGRZE9UBIYSQ9oaeAzLqSQodUg3kHLiuQqkc74U4/6R9Q8fr\nGZDVePqwOJg/fQI+/8bFseLCNMqPXjQD/3DiPrH3MK+Nehyim9iOCAo5B185+xDrXNHdclNsJOUj\npElIFpunAdUqSGa1IvN5I6RZj40470Yapnd34KKzD0FH3mHOASGEkFEJxQEZ9WiHQVK1ojRJvRql\ngMFKxTKX9xg1vhutVlQ3ITlDWE3YcxAZJ5HdcMc4YSH6/mwCoLpWW0Jy/LrN7yEUkqRLmRqfUVRM\nZaXRhGYzsbpesnkSTDkghBAyGqE4IKMeLQrs1Yqy5xwoKAxaPAdlv4xpNNylvufAe+yM5BzEljL1\nHzPlHBjVivLRZgSRe+XMuHgLNZ4Dp1YABNdGchmi94onHGBTTdqurj2ao5GVhg17f1izZUdt45sR\nG4QQQshwQHFARj2pqhVl6HfgKnvOgW5x0JHPKg688+k9B+GcgzTkEsOKJNMOf9SANT0RiWVOHfv4\nOMydfT2mI2eGFTXpOWjQDg8+n2ZjgCzDKQ0IIYS0OxQHZNSTFFYU9DnINJ9CKdIE7dRv34kbVqwD\nUNtkrJ4Nr6+OGrtxpToDz0GGsKJQE7OIURsnZuqJGj0u1EMhMsYmOhrZHA/yA4zBzXoOGt3516Oa\n1Qa28eOpCZqI/FBENomItc21iJwsIjtF5BH/50vDvUZCCCG1sFoRGfUEngNLWFHaakVmLLyyeA5W\nb+rB6k1etaJoWJFpMH/r7Ydj0cyJofOm0fzVcw7Bl363MnhtQx9vuM+Bv763vWo+Dp03Ba9aOM16\nbZKd+p13Ho7D50+rmTupz0G1dGiaFStrzoGJKaa+e+4ROGSvKWkmrq61Yc+B99iqPgsm40gbAMBV\nAC4D8JOEa+5WSr1peJZDCCEkDfQckFGPSvQcpHMdmENVTJ8DTVQcmLvdrz1wFo7ae0bkfHWdbz9q\ngXWcSSPNu2x9DmZ0F3DeCfvE9lFIMlT/5sj52HfWJAC1nhLrXJGOx2moVxXITEg+54h5eMXsyRnn\nb9Bz0KKwItvtx5PnQCl1F4BtI70OQggh2aA4IKOeJM9BSm0Q8iwoAMVyvDjoiHoOEhJ2gaqxqaAy\n7RzredMUWop2SDbvG0UbvWkN1SQPhmO8t+B5qhiuiJiwioMW9TnIOm5IE5KbmnIscryIPCoiN4uI\nvbYuIYSQYYVhRWTU04omaOb5uA7JmkLevhMPxJSv9B+VymYcZmkCFsoLqBM2lDbnIM06cpZch7T5\nHWYokW0tzSckN5YDodeVtTtyzTy2hGSqA5OHACxUSvWIyFkAfgtgf9uFInI+gPMBYOHChcO3QkII\nGYfQc0BGPUHSscUqDUqZ1gsrcsPPBxM8B0mlTK2Nr5zqWtIY5I0YtbY+B3HdeHMZ50/jOcgyn+16\n29CkcKY0ZCj2FB4n4cdGsX3+lAZVlFK7lFI9/vObABREZGbMtVcqpZYopZbMmjVrWNdJCCHjDYoD\nMupRCZ6DpO7JJ//n7fjAjx+sOa+QLecgVJLT8j9q0R7dAICJHblU4qCRzeW8JSE5zrh1gpKj2eeu\nnStZGCURqnRkuUezu+wNVytqWVhR9XkuYyjXeEBE9hT/SxaRY+D9Pdo6sqsihBDCsCIy6tGawN4E\nzXu0OQ5e2NqHF7b21YxVqp7nID6syGb8Xfy2w3D24XvhFbMnp+rU3Ij9aOtz0KqwonxCWJEtITld\nzoGKDce6+eOvSTlHMo2KC/25NBtWZH6+OUdQcbPlnIx2ROSXAE4GMFNE1gL4MoACACilrgDwtwA+\nLCJlAP0AzlVZWpkTQggZEigOyKgnqdGZ9ibUszmUGVakFAYr8dcnlTK12X6TOvN4wyF7eufTeA6Q\nHBZkI99AWFFqcZBgJOu5FNKFTJmEQ5Kqzw+em61kafz8jY2TjJ9P/DzV5wVHMNjEmkYjSql31Tl/\nGbxSp4QQQtoIhhWRUU+S50ClzTnI5DmIL2XairCRZj0HurNynCGq5097m7Q76NXPIc3mr9QtZdos\nTYcVNfnbUSKeg+gxQgghpB2hOCCjnqScg0qQcxA+Xo7kFFSyVCuqSUiuPm+F7aenyDKXiAQ7/IU6\njQyy7oyn7bego63SBoZIjOegVTQ6YzUhuXVr0qFZlAaEEELaHYoDMuqJJh0rpdA/WPGO+Ta+GVZU\nrrjY3leyzuE9B9Zt74+9X9Y+B5lpcA69jmrya53bpPzfn9Zw10nFqcWBOXYIrOaGm6DpUqYtFAdM\nSCaEEDJaoDggox4tALTn4Pt3PouDv3QLtvUOGsKhev27/+8BHP3120JzmAbti1t7ccvKDbH3i/Y5\nCO+AN/IOwhwxfyoAYEZ3R6Zx+Yg4qJez0EpDVSCpjPE5U7oAAAfPnRzOORiCPXUzWbqRcc12SDap\nlyROCCGEtAtMSCajnmpCsvf69yvWAwBe3tFveASq1v9fn99WM4cZkrS1ZxAA8MlTD8DSpzbi0bU7\nQ9cmhRW1wuD+/BsX421HzU/Me7AR7E6nNERbaacqo/qQSsg5OHzBNFz/keNx2PxpuHrZmupaYhZz\n34WnpG6qFsX8Lv7yuVNqQsviqIZdNXhjCzpJnJ4DQggh7Q49B2TUo+1/LQS6Ct4/64FSxahWlDyH\nGVZU9sccuXAaXjlvas21yWFF2dZuoyPv4LD50zKHxei49rQ5Aq02VPV89T7rIxdOR86RVLkae02b\ngHnTJjS2HuNrmjs1/Tyt6nNgEnSwpjYghBDS5lAckFGPNux1UvGEQg4A0F+qGMKhzhzGJn3Ff+GI\nWI3WfE2fg6FJrM06VTSu3db4rZn5E+eCGGVN04/RDMWOerM5B61cE3MOCCGEjBYoDsioJ9rnQIuD\ngZIbCIakUBdzDqDqOXDEHq9eG1Y0NAZf1nmjOQf1dvBbvu6M00kKz0EzNPr+tEej2SZoJkHOQctm\nJIQQQoYGigPS9qzb0Y9iuRJ7XhvBlSCsSIuDijUh2cbmnmLwXIuMuJ3njnx8zkEryTpttFpRvcZv\nI72J3er+EFEab4Lmj2+lOMhRFhBCCBkdUByQtmagVMEJF/8Zn77m0dhrgrAiPzRIi4P+wYq1lKmN\nt1/xl+B5yHNgCyuKGI1t6zlo8fxJmJ6ZtKVMQzkHLVuJOWejnoPWJyTrxnSNJlcTQgghwwXFAWlr\nBn2L//anNsVeo70CesdfJyT3FMtG74P4e9Q0RNPiwLGbl1GjupU7zCaN5hzkUuYctHrZWacLhxUN\nRc5B7X3SDfQeCs22SDbQwq3ed0IIIYSMNBQHZFSQZFJFE5K156C3WK5WK0qYoVQJn6t6Duy1+4cr\nqTTrbXRFnKCkaB07tKXJ00afg3r5HZpwInfLltI0el2tzDnQc7lp66kSQgghIwTFAWlr0phnKuI5\n0GN6BsuGVyF+fLSfQMUIK7IxVDkGtffJdqNqzsHIhLAEq83YTwBoryo++vttZZ5AIac9By2bkhBC\nCBkSKA7IqCfqOdA7/71mWFHC+MFIWFG5TinTVu4oJ5HZc5DT4sB7PVIhLOlLmdqfjzQ6mCxtv4g0\naMHGsCJCCCHtDsUBaWvSmFLa4Hpxax9+v+JlFH1PQG+xWq1o0+4BPLNxt3V8VBzol46INal1KOLj\nbTTqOQhyIIbZDs36sTht7jnIDUHOQb3EeEIIIWSkoTggbY1KCAfSmKEa//TLh3H/c1sBAH2D1ZyD\nb97yNN7wnbus40s1YUXe65G2V7PePp8xIVnzziUL6l4zfWIBc6d21Rw/eO4UAMCZr9wzOJbWAB7q\nPgcN4y+m0MKwIi3cKin+PRNCCCEjSX6kF0BIEtVqQ/EGZ/Tc7oEyAM8QS2OnDlZc7DmlC9MmFvDU\nht1BWFLOsYcVDRdZPRR6vXoXPs17f/bfz0qVQ7H8C6dZjy+a2Y3VXz8T+ZyD6x9e69035XqdNhUH\nel2trEJVzTmg54AQQkh7Q3FA2pp0YUXh14N+wzRXqcBzkMRg2euk3OlXOaqY1Yoyrba1NFKtyMyT\nSJP8mjZ/IslQzvtJDvrTSm//Dm1YUaN2uH4fuRauycnozSGEEEJGCoYVkbYmjTEVvUbnEFRclWr8\nYMWF6yoUfAPY1gQt2hV5OGgk50CAzCVFW0X2nIPGx2ah0XW1MvE8izeHEEIIGUkoDkhbk8aYitaO\n1wnJrkonDkq+5yBai16MPgedueH/r9JIzoHp7RgpQzR1taI2TUhuuHlaAjk2QSOEEDJKoDggbY1K\nUYo0am/p157noP49bn58A3b0lVDwBUDIc+Bf01kYJZ4DMZugtbchGvIcjNwyatCipZVhRXqqNGFu\nhBBCyEhCcUDamiylTKNU3HQ5B1fd9wKAap8AM+dA05nPWcfOnNQBAHj/8Yswc1JnitWmp5E+B2ZX\n53a3Q8PVitpHHsgQhBWde/RCAMCx++7RsjkJIYSQoYAJyaStSZdzYD+edZc2H8k5yBmug86YnINl\nfhWfi84+BBedfUim+9Ujq72ccxyIkScx/DkH2Wr5m4KgjbRBIApbWa3omH1m4IWL39iy+QghhJCh\ngp4D0takyjmIuaiUWRx4/x3MPgc6gn8kEpKz7qYHOQcjlPwa5DpkvB5os5wD/3GYGmETQgghbQXF\nAWlrqn0O4q+JOxdtblaPnB9WVK4YpUzreA6GkqzGqc450MNGLKwo5X1NQTAUdnijnhNnCHIOCCGE\nkNECxQFpa5rxHJTdbOKgEKkoY1b+GQ2lTLXnoDqu3UuZtne1olaGFRFCCCGjBeYckLYmSRxs2DmA\nPzz6Mipx4qCSzTjOOZZqRUFZy+E3FLPeMfAc6CZo2bRRy0i7Yx/6SIeyz0HGybU4pOeAEELIeITi\ngLQ1SYbmB3+2HCvW7Ig9X8poHeuE5KBakTM0YS/nHr0A07s76l6XVZAct98ecJWqljKNfHZ/f+xC\nTOwYuv/yWTskm29vKDbpj1gwDQtmTMAFbzgg0zgdjtXKakWEEELIaIHigLQ1SXHzu/pLiWOzeg7y\ntpwD3+Bt5SbyxW87LNV1We/5psP2wpsO2wtXP7gGQO1n97W3vDLbhBkJqiSlFQeG5BoKz8zkrgLu\n/swpmcfZxCEhhBAyXmDOAWlrkkqZ1itzWqpkzDnI6WpFtWFFIxET3+g9sxrprSLrap0h9hw0iu6Q\nzbAiQggh4xGKA9LWNGPgljLnHPhhRf5NxUhIHgk7sdFbZu030GrS5xyYYVvtY4hXVFUcEkIIIeMN\nigPS1mgD12Zw1rN9s3oOajskI1AFI2G8Nuo5qOYcDC/ZqxU1PnYocRlWRAghZBxDcUCGlduf3oSr\n7n0+9fVJBm69HeqsOQeFoFqRJyqcEfYcNKpHgmpFI+U5aCAhuZ3EQYXVigghhIxjmJBMhpXzfvQg\nAOD9J+yT6vokA9c89faj5qOQd3Dt8rUY9JufDVo8B0mhNkFYkS8qotVqvnL2IZg2sQAAuOjNi1NV\nHGqGRjeuzzx0Lu58ejM+d9bBrV1QHV530Gy85Yi98JkzDkp1vRlW1E59DvQ/G3oOCCGEjEcoDkhb\nk7Yz8gFzJuP/nbQvbnjk5UAc2EiqflTQ1YpcnXMQ7nPwvuMXBdemFTfN0GgFn65CDpece2SLV1Of\nzny2+5rvrp3EQdDngOKAEELIOGTIw4pE5AwReVpEVovIhUN9PzK20IZavVAVbVvWM+iSPBG6CVo1\n50BGNFF2rNumTighuX1gtSJCCCHjmSEVByKSA3A5gDMBLAbwLhFZPJT3JGOLZM9B9aQ2NPNNiAPt\nOaLbGCgAACAASURBVKhWq5ERjYVvpwo+Q0FIHLTRW61WqxrhhRBCCCEjwFB7Do4BsFop9ZxSahDA\nrwCcM8T3JGOIRHFgPHdSeg6S5ovmHDgysjvaY904DSckt8+bDTwHY911QwghhFgYanEwD8Aa4/Va\n/xghqUhbM18njzYjDrTXoewqP9+g6jkYCTOxneLwh4J2fXs6L4XigBBCyHhkxEuZisj5IrJMRJZt\n3rx5pJdD2gxtqNlsetPQ1zvPzeQc5I0OyVHDfESaoI1x27Rdw6bMsDJCCCFkvDHU4mAdgAXG6/n+\nsQCl1JVKqSVKqSWzZs0a4uWQ0UZS6VE3lHPgPdbLOUjyQ1Q9B24w30iGu4x149QZ8a0JOwwrIoQQ\nMp4Z6j/PDwLYX0T2EZEOAOcCuGGI70nGEEmlR020IV2vNn2y50CCe7ZDDPzIr2BoaVvPgdkhmxBC\nCBlnDGmfA6VUWUQ+BuCPAHIAfqiUWjmU9yRjiyTPgS0hua7nIL4FAvLGVnbVc+A/Js46NLSBPhlS\n2tX4ZlgRIYSQ8cyQN0FTSt0E4Kahvg8ZmyQ5Dmw5B/UMujSlTM15RnJ3ux28F0NJu749hhURQggZ\nz7Rp1C8ZbZxz+b344m8fT329mzJeKPm62j4HGtPQt4+oJWd4DnQDrHYwYGdN7hzpJQwJuTZNOpje\n3QEAmNxVGOGVEEIIIcPPkHsOyPhgxZodWLFmB/7tLYemur7sKnSk2JlN6znQUw1WvLihSZ15bO8r\n1YwxPQci4TkKOUEhJyhVVE040Ujt4l/x96/CK+dPG5F7DzV7z5iIfz3rIMye3DXSSwnx2TMOwv6z\nJ+PUg2c3PdcNHzsBxXJCLBshhBDSZlAckBEhKbzHep3l8nDOgWe8DwxWAABTJhTqioMoOUfQ3ZnH\njr5STWLzSDkQzjh07gjdeehxHMH5J+030suooauQw7tfvbAlcx02RoUdIYSQsUt7+vXJmKeStgxR\nUofkiBcAAAb8XdopMSEhiR2SRdDd4ellp43CiggZjYjID0Vkk4hY4w3F41IRWS0ij4rIq4Z7jYQQ\nQmqhOCAjQjltzkHGUqYDJe05sDvFksSBiGBSpxYH/rE2LbdJyCjgKgBnJJw/E8D+/s/5AL4/DGsi\nhBBSB4oDMiKkTUhWEdeB6S2whRX1a3EQ4zmoH1aUC81HzwEhjaGUugvAtoRLzgHwE+VxP4BpIjJ2\n4+gIIWSUQHFARoRKxEg/70d/xaILb6y5zg1SDhSK5Qr2+dxNuPz21d4xS0KyPja5y+45CCUkR87l\nHGCSLyqi1Y8oEghpOfMArDFer/WPEUIIGUEoDkjTlCvZq7FEPQe3P73Zep3pKdi0qwgA+Nn9L9ac\ni1YTaiTnwBHBpMBzYJ+XEDL8iMj5IrJMRJZt3mz/XUEIIaQ1UByQpilVUiYGGKTNOTCN+c09njiY\nPtGrQ2/rkKyZOqGBhGSnmpBcKwooEghpMesALDBez/eP1aCUulIptUQptWTWrFnDsjhCCBmvUByQ\nphlsoI572mpFbshzMAAA2GOSJw4QCiuKeA5ixIGb4G1wxCtlCgC6PxclASFDxg0A3utXLToWwE6l\n1PqRXhQhhIx32OeANM1gI2FFKfscmJdt9MOK9ui2eA4iMndCRy7zfXOOWa0onJDM6CJCsiEivwRw\nMoCZIrIWwJcBFABAKXUFgJsAnAVgNYA+AOeNzEoJIYSYUByQpmlEHFRcha09RTgiNcnDW3qKmDmp\nE0DVmFcK2Oh7DiZ3FbC1p5iYc5CLseaTHBa6CRpQ9YZQExDSGEqpd9U5rwB8dJiWQwghJCUUB6Rp\nGgkrunvVFnz5hpUAEOpGe9/qLXj3/z2A/33vEpy2eE7IO7B5t+c5uO/ZLfipn5Ss0Tv9e03twss7\nB5CLJiEExKsDMyG5z++0zIRkQgghhIwnmHNAmqbUgOfgoZe2B8+vXb42eL78xe2h86Z3QBvsz27u\nrZkv74uBmz9xEu698BQ4MeIgOdVBobPgiQP9noKwoqRhhBBCCCFjBIoD0jTNJiSb43WIkg4LMlME\niuVK7HxaHEydUMC8aRMSwori+xwoBXT54iD6nuhAIIQQQsh4gOKANE3anAPTCxCXGFz0jXK9819t\nggYMlOLvk89Fcg5i/mW7CUtVADrz3kBdapWagBBCCCHjCYoD0jRpPQdmSE+ckT5Q8rwDgefAyBHQ\n52zkIuWKzFwBc9dfQcV6AUzPgXUwIYQQQsgYh+KANMX23kG8sKU2B8BG2VAElRjPwWPrdgKoNjXT\ngqLiKqzfORA7d96Jr1Zknlu3vT+2EZqCQlc+IjJi70gIIYQQMvZgtSLSFCd+48/oHYzf0TcxvQUq\nxkJ/+KUdAKphReZ163b0x85dG1Ykoee6i/P5P10eHBcBDl8wDSvWePecOamzJnSpmpBMmUAIIYSQ\nsQ/FAWmKtMIAiHgO6nRIzjm1CclJRD0HZrWivOMAsMcxXfPB4zBYcVFxFaZOKGBLTzF0nqKAEEII\nIeMJigMybJiegzrawJpzkEQ+knNghhXF9zwAOvIOOoxQoq68vbMyUw8IIYQQMh5gzgEZNkzPQVy1\nIk1QrShlldSoADC1QtSrkEQ0IZmigBBCCCHjCYoDMmyYScj1woqqCckpPQfRnIOUnoMonTEJyRQJ\nhBBCCBkPUByQuriuwp9WboBbLxbIYM22Pjy2dmfomCkI0hr9ae9YE1bk2KsVmdjyCeg5IIQQQsh4\nhuKA1OXa5Wtx/k+X4xd/fSn1mNd883a8+bJ7QsdC4qBOuJC+Nq6qUZSaUqZmtaKc3cL/8Mn71Ryr\n9RxQHRBCCCFk/EBxQOqyabfXXyCplChQDQWKwxQHcX0OotemrVYUFQCdRmJxwdIu+QMn7oNPnnZA\nzXEn+iZYypQQQggh4wiKA1KXvG9c18sTkDoxOFnCisr+tWkjmQqRsKKuQvV1h0Uc1IiAelAbEEII\nIWQcQHFA6qJDdsqV+rv90TCgwbK9t0G9/IXAc5Ay6yCadGzmDnTka/+Zp7X1qQkIIYQQMp6gOCB1\n0YZ3JUVd0ajN31ssB8/NUKJ6HoFKRs9BNOfAzB2whRWltfrreUMIIYQQQsYSFAejlMGyi6vufR7l\nSspGAE2gw4pKhqW+afcArl2+tubaaOjRj+57AQCweXcRv35wTXD8sXXhSkax86RMOoiGCZmeg4Il\nIdlJafRTGhBCCCFkPMEOyaOUH9zzPL5xy1PIOYL3HLdoSO9V0J4DI6zowz97CMtf3F5zbTSX4NKl\nq/De4/bG+T9dhodf2pH6nnE5BycfOAt3PL257nhTHETLnALJRv9JB8zCgukTvOuk/vWEEEIIIWMF\nioNRyu6BEgBg10C5zpXNo3fly4alvr130Hqt3vEXqW76lysKm3YVM91ThzCZOQz/etZB2NFXSiUO\nzBwEW/JxkuPgJ/9wTKrrCCGEEELGGgwrGqXosJgsjckaRRvoZSPnoDPSLEyjBcQE4/xAqZLZyNbR\nUubbyztOEOKUBVubg/RhRVQHhBBCCBk/UByMUvRmeL1+Aa2gVNHioHovs1SoiRYrEzsMcVBuRBz4\nngPjWD4n6IhpaJaETQhknYWJyYQQQggZDzCsaJSijdVhcBwEoUJmzsGEGM+BFitmzH+xlD1pumzp\nkJxzpCHPgdWwT+s5YM4BIWMCWwEFQgghtdBzMErRu+HRvgJZ6R+s4N9vehL9g5XYa0p+jE/ZdXHd\n8rW4b/WWeHHg1oqDL92wEv2D2QTCC1t78Z1bnwklOBccp6ZkaRpsQ2jsE0IIIYTUQnEwStEGb71O\nw/X4v7ufw5V3PYcf3fd87DXa4C+7ChdcswLv/r8HQsa/7drJXVWn1Io1O7ClJ11C8mHzpwIA7l29\nFd9dugobdlbH5RwJ9S9Iiy2sKG3OASGEEELIeILiYJTiBI3JmptHewXMTsZRdIiP2cOgMybnwLxm\nRneH9ZoPvXY/6/GPvm4/3PCxE7FgxoTgmCl+8jlBZ94uSpKIdk8G0lchGoaUDkIIIYSQtoHiYJSi\njdtmw4rSWMllnZBcMROS7Ua6NuZdV4WSkk06Ynb/c/5acsaaiuVquFPecWJFSRLWlIOUYxWqpVkJ\nIYQQQsY6TEgepeSChOTWbG0nTaMrBw0abopcjLVsNi+LEwdxoUHaG2Lu9JvJzF5CcnbPgTWsKGPu\nArUBIYQQQsYDFAejFKdF1Yq00Zs0jTb4zV38csyNXSMEaWKH/Z9XR0zFIS04zI7GRSPcqZBrrFpR\nAznMhBBCCCHjEoYVjVJ0ydBGPQe/e2Qdvrd0VTVcRils7x3EB3+6DDv6wt2PtRAYMHbxK649R8Fc\nV6znIBIaNKnTExF6N9/c6L/xsfXB81YmJDPngBBCCCGkFoqDUUol6APQ2PiP/+oRfOvWZ0IdgH94\n7/P448qN+PF9L4au1bkGZrnTaCL0p08/MLSuJHEQ9RxM8K/T4UQDJXtZ1bzjxOY62Pj5B16Ni968\n2NrnIGu1IjZBI4QQQsh4gOJglKKrDFVa1AVNGXNFI3e0l6CnWK45BgDvO25vHDhncmg9FVfFVhaK\nJiR3++JAh//0x4iDnCOxnZltnPCKmXj/CfuwzwEhhBBCSEooDkYpZu+BZnCCqkfV/IVosm7JP2GK\nA/O+jiPBrr/p0YhL+o2KA52boHfzB2I6KjuChkqZMqyIEEIIISQdFAejFG2cJ/UnSIM2kl2lgvyF\nqDFdqdT2OTCf50QCIaDnqCgVmwgcNfC7O9OFFWX1HGhsIiVtWJF+l/Q0EEIIIWQ8QHEwSilXasuL\nNoIZS69zC6JlSkuW5ONSJVJi1DfAy4aQiCt3Guc50OKgGCN4HEfQ1ZDnIPOQWqgOCCGEEDIOoDgY\npWjPQalc7XB80jdvx5+f2tjQfArVXX+90/6z+1/El373uDWvobdY3d13HAl24nW1IqXik3ijCcna\nc1BvN98RyZSQbI6LwgRjQgghhJBaKA5GKXqHXnsONu4awEvb+vDF365saD4v50B7Drxj9z27Bb9f\n8bI1r6FvsJp/kBNBIWfxHDjAxW99Zc3Yep6DOHJSW8r0g6/dF9d9+PjEcbZp2fuAEEIIIaQWioNR\nSuA5aDqsyHtUUEa1Iu/gYNnF9r4S+oxEZE3vYNhzoA1+nQPhKoWcIzjj0D1rxkYNfF3yNC4MqXqf\n2vyB0w6eg6P2np44rhkvgWJGMiGEEELGEU2JAxF5u4isFBFXRJZEzn1ORFaLyNMicnpzyyRRdClR\nHZ8fGPkZjVmzz0E0rGjQ9wKs3zlQM663GPUceP+UtFhxlYIYicomUXGgX8dVN9LYwoPqeRvixqX9\nmKoJyXQ1EEIIIWTs06zn4HEAbwVwl3lQRBYDOBfAIQDOAPDfIpI9WJzEosN3tDHe9Aa3AnTesTam\nB8ued2Ddjv6ahmZmWdOcUw0VGjT6L+SkmqhsEg0r0nkE0f4KUWxCIJ04qD2W9eNiigIhhBBCxgNN\niQOl1JNKqactp84B8CulVFEp9TyA1QCOaeZeJIwOK9Jdi/XrRo1ehWoysfYglHwBsnugjGkTCqFx\nfUZYkYgEScbVsCLPKLft2hdydnGQJiE5zbGaayzqgOFChBBCCCG1DFXOwTwAa4zXa/1jpEXo/ICn\nNuzGH1duCMKM3CaMXtfobvzhny3H8he3B+emRMRBqM+BkXPw6WsfxaILb8TO/lKoOZpJ1JsQhBXV\nMfQndNQ6n9J4Dpra9aeGIIQQQsg4oq44EJHbRORxy885rViAiJwvIstEZNnmzZtbMeW4wExEfnTt\njmCXPyvVjsYq8BxUXIWHXtoeum5yVz52jpzhOYgetyUZRw36zkK4CdodnzoZ13zouNA1V77nKMyb\nNgEAcP1HqtWJbGFLUWyioxkRRQghhBAyVom3+HyUUqc2MO86AAuM1/P9Y7b5rwRwJQAsWbKEFltK\nKq7CofOmYGvPIDbuKhpGfrZ5dO6Cq6pCoeKqUB8DAIn9BRxHUMjXigPH8RKSRcLryjuRsKKI52DR\nzG4smtkduuYNh1SrHh25sFqdqF4SM2CvgpT1c2LKASGEEELGA0MVVnQDgHNFpFNE9gGwP4C/DtG9\nxiVlVyHnOJg9pQsbdw00nHNgJjSbuQa9g+HypdE8AZOc1DY2A6rGftQ4z+XCr7sinoMs1Ct/6q2j\n9ljaz0kxrogQQggh44hmS5n+jYisBXAcgBtF5I8AoJRaCeBqAE8AuAXAR5VSlfiZSFbKrou8I5gz\nuRObdhVRbrBqUSAOjD4HPcVSzTxJhnvOqTZBM9FDorv7UYNe5xzUq1YUd+962PocMKyIEEIIIaSW\nZqsVXa+Umq+U6lRKzVFKnW6c+7pSaj+l1IFKqZubX+r4QCmFgVKtjhooVaCUQrniolRxUa4oTxxM\n6cLG3QNGF2NVMw7wqhpt2l3br2Aw5Dnwju3sL9VclxTb74UOxecW1HgOHLvnIE3lobh7JNFUnwP/\nOpYyJYQQQsh4gB2S24xv/ekZHPTFW0ICYddACQd98RZcfvtqvPmye7H/529G2VXI5wSzJ3diR18J\nfX4YkGn03rt6Cw764i148IVtOPuye3DM15fi6Q27Q/fTpUfLrht4Dnb2V0OKdLhQUs5BHNoojwqL\n6OvpEzsAAJM666bA1NBon4OssAkaIdkRkTP8RpirReRCy/mTRWSniDzi/3xpJNZJCCGkCsVBm3H9\nw17e9sZd1V3+bT2DAIBrlq/Fk+t3AajmHOgSozv6anf771rlVX/66/PbsGpTDwDg5R39oWtKRtMy\n/XyX4Tk495gFuPRdR+KzZxwUu2bXtW/Da3EQDSsyXy+94LU4dN4UXP3B43DU3tND193xqZNj7xm9\nR+I1/v1OOmAW3n/8osQ1E0Jag9/48nIAZwJYDOBdfoPMKHcrpY7wf746rIskhBBSA8VBmzFzcicA\nYEtPMThmM2MrrouCI+j2d9t1KJB5rTaAzZ16s7MxUG10Vq4oFH0vghlWNG1iB84+fC/sObUrds1R\nO3u2/x50DkHS7v5+syZBRHDMPjNqQpOiFYtspCllqqc9eO7kwDtBaUDIkHMMgNVKqeeUUoMAfgWv\nQSYhhJA2huKgzZg1yQux2bSrKg7MngaackUh5wgmdXrhPoE4MOKKdB6CaZz3RsSBzjmoKLvnoNNS\nojRKJaIOtGDRxn4jVYjSkqaUqfYuKGV0hE6bc9DowgghaZthHi8ij4rIzSJyyPAsjRBCSBzZg7zJ\nkLJHt7frboYV2RKUy65CIecEhrgOK7J5DnIJngOdc1BxVfB810BVHNiqEEWJVv6ZEClNmqbcaKOk\nER76/q6rAsGStkQpE5IJGVIe+v/t3Xd4XNWZ+PHvmapmWXIT7sYFA6ZjMBBCLzaQsAkpJPAjZMmS\nbJLdkOxC4EfCkqUvIZtAqEtIgGQDAUIJNmAwGOzYxhiwjXuR5SLbkiWrt2ln/7hl7oxmRpI10khX\n7+d59Pjqzp0790jyzHnved9zgEla62al1CXAKxhTXydQSl0PXA8wadKk/r1CIYQYYmTkYIAZXmDU\nEOx3jBy0h41Oe01TfF80ZowcWMFBoz1yYDweisTYXWfUF+ypi9cZbK1qTgg8dtW2AkawYY0cONOK\nupPTn3wXviBgBgf9MHLQncDDOiSm48XJPS05kOBAiB7rcjFMrXWj1rrZ3F4A+JVSo5JPpLV+Qms9\nW2s9e/To0X15zUIIMeRJcDDAWHf7DzQ5gwNj5KAlFB9BCEeNdQ6KkmsOzJ76z19Zx7ubqgH43dId\n9vOeX7WbOXcvsr/fXGXMXhR11BxYdQjG+bpxzUkHFdhpRcb3nl78lU0ozc/4eHcCj6PHFgNw/MTh\n8VmHZJ0DIfraR8AMpdThSqkAcCXGApk2pdRhyhzOU0qdivGZVNvvVyqEEMImaUUDTNTstHZE4oGA\n1Wl3CkdjBHyOtKKkguSFG/b3+HVbOjqnLzk7/j6Pcqyn0PmaLQVJaUW+XkQHb91wVsq0Kkt3goMz\npo9iyU3nMnFEAQ8u2grICslC9DWtdUQp9UPgLcALPKW1Xq+U+p75+GPAV4B/VkpFgDbgSq0lchdC\niFyS4GCAsYp7nUXIqTrHoUgMv9dDUcCqOTCmOz3Uvmw0pjsVK0Ni+o3Pmzo4SP4oD/qNYMCeyrQX\nKTmFQZ8dAKXS3XNPHFEAYK9W0PMVkiWvSIieMlOFFiTte8yx/Vvgt/19XUIIIdKTtKIBxgoOQpHM\nwUF72Bo5sGYr6tyx74lQJEZbitdJHDlI/eeSvGaAN2l9g76sOUi1MnPm441/5d6kEEIIIURnEhwM\nMFZn3Jn3nyqtqC0cxe/14PN6CPo8NLQZIweH2ud1zlCUcD2Ojn9yJ9/qaCenFXmSZinqTlFzf4nP\nVtQ9EkQIIYQQYihxRXCwfm8DL368JyevHYtpVpTH6+f2NbSxo6aly+d9sqsu5YhAd0cOAALmGgRF\nQZ8dTMS0pqqxnboUKyZn4pyhyCkhrSgpOPCbq5ylGzmwYgJfN6ZD7S/xmYu6OZVp0vOEEEIIIdzM\nFcHBW+ur+PcX1pCLOran/r6DK59YweLNxsxAp9/zLuf+cnHG5xxo6uDLjyzjxhfXdnrMyukPOWoO\nUo0cAATMTndyTv4Z977b7eu3JAcH1ijBaVNH2PuuOm1ywjFnH2FMKXj6NGPmwVOmlALx2YmsDni6\n6UZnjSvu8XX2Vny2op4+TwghhBDC/VwRHNiLXOUgBeTjnXUANLV3P+e/NWQcu3x7TafHYl2MHFgd\nckgcObBo3XnFYsvL3z+DYXmpi3sbk4KDc2eOZvOdc5kzdaS978cXzGDLnfPs7z8/YxSb75zL6dOM\nY567/nS23jXPTt2xLiPVKsbb7prHaz88M+W19CV1aLGBEEIIIcSQ4IrgwEpbSdcp7ku1LUauf4m5\neJmlLZR++k1rleLk1YoBrFKDcJqRg7LioL1tpfU4g4NM6TIFAZ/9nGTJP7rCoI+gz5uwTyllByQA\nQZ8n4RivR+H3ehJWJIbUIwc+r6dPC5XT8dgFyRIeCCGEEEIkc0VwYHUycxEcHDSDg+SXrm5qT3G0\nwVpPwFr52CmWIq3IOXJQWhCwt62OujVjEZByqlFLnr/rX3fAa52z61lu8/zelPuTfx+5CALSsdKK\nuv2nIkGEEEIIIYYQVwQHVqFsJJY6N78v1ZnBQTipLmB/Q6bgIH0KktWGcJq0ouGOEQp/io58pgAp\nXWfeqTjfOH9RN4KD5JEFi8dOKxqAwUEPpzKVgmQhhBBCDCWuCA6szmgu04r2N7azaGOV3RGuaupg\n/d4GHly0lVdXV9rHh6Mxnl2xs9N5PiyvZUdNC9aAQeLIQXy7OC8eHART1Bxkki6lyMmqSSgMdGfk\nIPX5PEkzAg2s4MCayrRnfytKSpKFEEIIMQS4YoXkXNYcWH72yjoACgNeWkJRDjZ3sGDtPt5cvx+l\nYO4xhxH0eXl6WQXvbqq2n9cejpLn9/L1J1YAcN6RY4DEgmRnbcLIwnhaUaqRg0wKg17+7aIjuPXl\ndSkfHzc8z+7IF+f3YuTATisyvncGB5cdN7Zb19qVr8+eyCLHz7G7zj5iNHcAlx03rlvHnzndmInp\n8hO6d7wQQgghxGDmiuAglzUHyayc/3BU0xEx0oG0Nu7+B31eGpNmNeoIxxLSfaIpag6qm9o5Z+Zo\n/vDtU3lvc7xDnKk+4MaLZ/KDc6cz5eb5ACy/5TyCPi9XzZnMZ3saeO6j3QnHb7/7EjwKTrlrEQBl\nxXldtjX9yEFSWpH5/VdPnsD9Xz2+y/N2x31fOe6Qnjd9TBEV917a7eOnju7Z8UIIIYQQg5kr0ori\nNQe5Dw6sTn0oGktc5disGygKJt5tb48kzmoUdQQXlqrGDg4zO+t+T/xX5rfTijrfwS8MJO4LOFKK\nUi1K5vUolFLUtnQA3Q0O0hUkG/9axdX2iskDKL1ICCGEEEJ05oqRg1zVHKSaDtPaFYrEUk5Hmpy7\nnrz6sdWGaEwTjWliWlPT3MEYs7Pu7NhnGjlI3ud3TEHq86SPCa3rd06Zmk7a4CBpnYP4iskSHAgh\nhBBCDGSuCA5yVXOQbuViMEYOnCMZVhCQvLZBR1IQEXUEHOFojLrWEFrHO+vOouKAz2h3qoLk5H3O\nkQN/ipGDZKOHdR0cBH2pgwwrCLDa4vVaIwddnlIIIYQQQuSQK4IDr3knvL/TijpSrFNgCUdiRJJm\nHNpV28oTH5QnHPf3bTWs39tgfx9ztOGx97dz5GHFAJQNM9OKEkYOjDv3qWYWSh45SEwr6rqXnq7Y\n2KmrdQ6SF0HzyMiBEEIIIcSA5o7gIEdpRcn1Ak6haIxQVJPn99AejtEeifLjp1fTZo4gjC/Jp7K+\njV/8bUPC85wBzq/f2WpvlxYaU5j6EmoOjHbPPGwYk0YU0B6OUt1k1AwUJtUheBz5/t84ZRKPLt6e\n8rpvmXcka/bUp2+0Q7qC5C+fNJ5nlu/kq7MnAPHF2iQ4EEIIIYQY2FyR6JGr2YoyjhxEjZGDoqDf\nPrapPWw//psrT0j5vFia1bmsdCJ/ipqDiSMK+OCmc/kvxww+me78TxpZwN9vPi/lY989exqPXHVy\n2uc6pXuNCaUFrPrZBUweWQjEi6MlOBBCCCGEGNhcERz4chQcZBo56IgYNQfWomLt4WhCqk+6jnW6\nNlgjBv6E2oHEX1/AUQOQ7q5+/Hy976h3d/Yhq90SGwghhLDsrG2hoS3c9YFCiH7liuDAKniNxNLf\nye8LyTMNOYWjmnA0ZhcGt0eiCUXC6Trv6YIDa8TAOVtRckGw8/uuagb68y5+kRkgOWswhBBCDG1n\n37+YLzy0NNeXIYRI4o7gIGnRrf7SniGtKBSJJgYH4RgFjrUH0hXzRmM65R35eFpR+pED5/fpzm/R\n9N/PyvoZtIbSB1NCCCGGnl0HW3N9CUKIJK4IDuxF0KJ91+F9ckk5S7fWAPC7pTtYurXGXgE5cXud\nagAAHi5JREFUlVAkRiSq7bvmHZEozoGNdNOARmKa/BQde2vEwJkOFPAdelpRf6ZgWbMptWYYaRFC\nCCGEELnnjtmK+qHm4M75GwGouPdS7njdmGHof66Znfb4TmlF4Zi9xsET/+9kgmnu7LeGIgzP93da\nD8EaFSgpCHDpsWPxeVXCSITzGIinFT30jRPZU9fW6XXKhuVx5SkTaWwPc9lx49I3PIVHrzqJ8pqW\nbh9v1Ry0y8iBEEIIIcSA5orgwGfXHPR3WlEXIwcx7QgOorSEInzx+HFcNOuwtKMOLR1RjhpbQGV9\nYofeGjHwehQPX3VSyuemWujsC8en7vh7PIp7rzgu5WNdmXfs2B4dL2lFQoiBQmstq7ULIUQGrkgr\n8iStyNtfMgYHUSOtyLpr3hGJ0dIRsdOMAl5Pytl7WkIRxg7P77TfnyYNycmZVjSQPvwKzDUX2iSt\nSAiRY+v3Nub6EoQQYkBzRXBgTfMZ7cOaA4szdakjYhQRFCal9wS8HkKRGKFojIBXEfR56AhHae6I\n2HfRlVIp6w60JmXNgd/TjeCgGysf54LVnjYZORBC5FhYZk0TQoiMBmZvsoesmoO+SiuKJQQE8Q6u\nNXJgjQZYCoNe+zGf10PQ56ElFKE9HLOLcyH9jEKeFLMVOacwTac7owu5YNVGtIYjXRwphBBiqGto\nDfPM8gp0UjbAxn2NNLbLughC9LWB2ZvsIavj3FcFySHHnSbn9KX2yEEwOTjw2fn1Pq8iz++ltjlk\nPuaYzjTNWgSpFijrzqJlA3XkoMAMiHzdGP0QQoi+1L/Jp+JQ/PSltdz26no+2VWfsH/eb5Zw9ZMf\n5uiqhBg6XNFb6+uag8TgID5yUNcSIt/vTRgNABg9LEhLyLhLHvB6KCnw2zMGORdCCzqmGz196kh7\n2+tRPHvdqVzqKPztTg2BvxujC7kwqijATXNn8tS1p+T6UoQQQ5ysyDvw1bUaN9NCkc4pYGv3NPT3\n5Qgx5LgiOPDZU5n2TS6p8w3KGRxUNXVQVhzs1CmfWFoQHznwKMqK8yg/0AwkjjJYIwfXnXk4v/za\n8fZ+j1J8fsZovjp7Qo+ucyAVITsppfj+OdM5fFRhri9FCDHUydCBEEJk5IqpTL19vAias4CtrjV+\n16mqsZ2y4jySByxGFAbsFCef10NZcR4tZrDgHDmwFirzez2MLgra+63soIGaJiSEEINVf64OL4QQ\ng5Erep9WzUGsr9KKHCMHtc0d9rYVHCRzTinq9yrKiuMd/8KEtCKvfUzA52FkYQAAr5mbn7wCshBC\nCCGEEH3JFb1Pr+rb2YqcIwe1LSF72wgOgp2CEucdf5/HkxBAOAuSralMrULdMeZx9siBBAdCCJFV\n/bwcjhBCDDqu6H16PdmfrWjh+v187fHlxGLanpUI4Ja/fmZvt4djlBXnderEO9cv8Ps8jBkWDw4S\n04qMQMEa+bBGGKxgxy9pRUIIIQawWExzzxsb2d/QnutLEUJkiStqDqw779msOVizp56VOw5S1xoi\nbJ732PHD+awycaaELx4/jhMnlXLCxiomjyzghImlFAS8PPD2FgD8nvRpRXmOtCKAMcOM46x1Dg5l\n5OAP3z6Fw4Z3TnUSQggBfTTAPGR9vKuOx98vZ11lA3/6zmmdHm/piFAQ8A7YCTO60tAa5mBrSCbU\nEEOKK4IDbx/UHFjrGVQ1dtg1Bz+deyRX/y4+x/I5M0czpjiPMcV5nDy5NOH5IwsD1LaE7IJkS8JU\npr54QTLAKEdRMhxaQfI5M8f0+DlCCDFU9NWsdkOVtUhoONL587eyvo3P3fsu3zt7Gl85eQLTxxT1\n9+X12iUPLqGyvo2Key/N9aUI0W9ckbfSFzUH1krIVU3tds1B8p384fn+tM8fYRYX+7yK0cOCKGWk\nPzlTjqzZinxmEDDSDA7qzLoGqTkQQojsqmkOdX3QALd+bwPLttXk+jISpJoFamdtCwCPvb+dC371\nfn9fUlZU1rfl+hKE6Heu6H32Rc2BPXLQ0G6PHCSvZ1DSjeDAoxR+r4eRhUEKk4ZWrXUO/Ob1jzbT\niqwPL6k5EEKI7Fq/d/AvonXpg0v55gBZKXiwpgsJIdJzRe/Tl7TOQWN7GN3LFCNrsbOqxg57heSe\njByMLDKCA2s1zrLiYEJKEcRXSPbZaUXGcw40daR8PSGEEL2zdIDdcRdCiIHGFb1Pj0ehFES15s8r\nd3Hc7Qt56N1tKY+tawkx5eb5PLdyV9rz3f7ael5fuw+A/35nC9999mPAqAE4Znyxfdy0DPmTx4wf\nDkCe2cEfV5JPSUEg4Rh75MAckZhYWgBgFxQnj1QIIYTond0HJU3E8urqSl7+dE9WzpWtkr/tB5r5\ncMdB45xoVpTXMuXm+XaKkhCi77miIBmMuoNoLMb26mYAyg80pzxud10rAM+u2MmVp05Keczq3fUp\n9wd8Hv543Rw27G2kJRTlgqPSF/9+96xpzBgzzD7mZ5ceRUtHNOGY+GxFRgAxcUQBf7xuDsdPNAIL\nWSFZCCFEOku31jC2JI9po7tf6Hvbq+t4e0MVy285nx89txqAL5044ZCvIdtZRTeY12T56ydG8LJ8\ne23WXuPHz6+mtCDAbV84OmvnFMJNXNP79HoUkZimJRQBoDmpI27pzt2N6sbU8zX7vR5KCgKcMX0U\nFx5dljHX0utRCcdMHlnI0eOKE46xC5I98fOcOWMUw/KMdCXJ5RRCiKFnS1UTx/9iYcLaAZ/uquPV\n1ZUJx139uw85/4HMhb7haIwq8zNtys3zeWb5Tvb1Yk2CfQ1tPLV0R9rH1+9tsOv0emJHTQuXPriE\n+rbEgvF1lY1A96agnXLzfL79+5UpH4s4FjN9+dNKnvp7+jYIMdT1KjhQSt2vlNqklFqrlHpZKVXi\neOwWpdQ2pdRmpdTFvb/UzHweRSym7aCgpSOS8ri2cOqgwRKLaarNnP9k2a4BCPoSRw6EEEIMDB+W\n19qdaoBQJMYTH2y3Z6/LllAk1unz6ullFTS0hXl7Y5W970uPLLPv9CebcvP8tOf/+SvrmHP3Ipra\nw4d0fVprXli1267Du/TBpfzn6xvYU9fKnz7cyUozBWjVzjo27W/k0geX8h+vre/x6zz83jbW723s\nlPa1YZ8VHHQvb+m9zQc67ausb2P6rW/wl492d3rsUH8uQrhZb3ulbwPHaK2PA7YAtwAopY4GrgRm\nAXOBR5RS3l6+VkYea+TAfJO1RhCSpQsaLLUtobRToma7Ex8vSJYRAiGEGCjaQlG+/sQK5ty9yE5n\neXpZBXcv2MTTyypSPqexPcwzyyvSTobx5rp9CcGG5YpHlzHrP95K2JepG5wp9749HOWFVbsTruGd\njdUA/NMzqxKO3WOm2HZl8ZYD3PjiWu59YxMAB82ptmMxuPXlddz/1mb72Lm/XgIYoxyZGlJ+oJmv\nPbY8YZ+ni4/BVLMRflRxsFuzFFppxje9tJZ73tho7393UxXH3r6QjyoOorXm6WUVtIVS30B847N9\nXb6OEG7Rq96u1nqh1trqba8ArMTFy4HntNYdWusdwDbg1N68Vld8HkU0pmk2O/+1zSFazQChPRy1\nt/fUGXcllDI+ALZUNRGLaQ62hNhb38a6yvTT3AWzPHJg1RxYKzwLIYTIvTl3v2Nv3/LXtUD8DnNz\nR4SOSJRl2xNnPbr15XXc9up6Pqqo44kPtrOtOl73prXme3/8hDl3L7L3VTe2E4nG+CzFZ06NOXod\njcaobw0l3NQ6+/7Faa/7/rc2c+OLa3lvsxEQPLBwMzXNxrlWlB9MOPbM+95LeY5QJMZLH++xA4ym\nduO1D6QZUU9l0/4mIB6YJPvlws2srIhfz6qKg2w2n5NO8ojNh+W1fPWx5Uz7/wu6vJ5/f2GNvf34\n++X2thX4fbqrjkcWb+c/XlvPUbe9aS/s5vTPf/ok5bnvfH0DDy3a2uU1iMy2VDXx4+dXJ6R/idzJ\nZkHyPwLPm9vjMYIFyx5zX5/xez0Jw7OV9W0cfdtbVNx7Kec/8D6V9W08/M2TEoY7f/KX1byxbj8X\nzyrjrfVV6U6d8BrZNNJcC6GkIP2UqBBfM0EIIUTfa2yPd8aTR5IVijtf38izK3Zy15eO4ao5kwGo\nbzXuqDe0hbl7wSYeWbydJTedi9/r6TS5xL6GNk6/512+OSc+KUZ1UzsjC4MsXL+fhRuMz6Pb/7aB\n2/+2gVHmApldsTrwD7+3nRdW7eGNdfu79bz61pA9m94RP3vDaKeCzfub7Fn55n+2j7ta4/UAX350\nWcZzPrmkPGVe/9/W7GXBZ4nXtbO2lTV7OgdJzgDrzvkbEx5Ll/5r+cuq3eyrb+dHF8ygqjH1sVvN\n89+9YFPC/qqmdh56dxs/ufCIjK/x8HvbeNKsv/iX82ckPHbFo8vYUtXEZ7fHs6q3VjUxobSA/MCh\nJ1LUtYQozvfb6zv11paqJhau388Pz5vR5bHRmOZAU4c9o2I2bKtuxutR/OufP2XT/iauP2sqR40t\n7vqJGURjmj8sq+CqOZPsm7A98ccVOzlt6shur+h93R8+YtGmaletot1lcKCUegc4LMVDt2qtXzWP\nuRWIAH/q6QUopa4HrgeYNCn17EHdMbIoSE1zKGXakLXC4ZKtibmIO2qM4dl0gcGZ00cxa3yxfach\nW/8ZLSdPLmX+v56Z8T/C0p+e22l9BCGEEIeu/EAzU80Zfh5dvJ2XPtnDOz85O+Wxe+rajFQdxz6r\n03rry+vs4KDDXDjTuuNe3xrm2NsXMmlEAY9cdZL93O//6WO7c/y/H8an1D71rvioQjLr7n8mv3xr\nM6+t2QvAxzvrujze6d9fWMuSrQd46tpT7H0PLNzSaXXgE/7z7W5fU3JnHuDlT/fw4+fXdNr/SlKh\nteW2V1PXLiTXWKyrbOg0snHTi8aIz48uSN/pXZyiPgHg9HveBRJ/P5Ynl5QzviSfs44YnZBSdc+C\njdw870h7IpHk30F7OMqF//0BFxw1hie/dQqHorE9zIl3vM0/fu5wbpo7k/ve3ER7OEZJgZ9dta18\n9+ypzBo33O6rPPzeNlpDEW68+Mi057zi0WU0tUf45cItPHb1Scw9ZmzaY//rzU08/kE5H916gb1o\na2+0h6P2ytlHHjYMMALco1JcwrrKBsYOz+ONdfu5as6kjBO2/G3NXu54fQN/WLaDf7twJv9wYvfv\nTV/z1Eo+2HKAPL+HTXfMs/eHozFaQ9GU61st2pR6hGww67LXqbW+INPjSqlrgcuA83U80bESmOg4\nbIK5L9X5nwCeAJg9e/Yhz5RcVhykuqm90yxFznxE5x2gWCyeO5lOJBbj1CkjEoYhs0kpxaxxwzMe\nM8Fc+0AIIUR2nPfA+/zT5w/nlCkjuO9N467xo4u3c9+bmzhhYkmn4298cS3TRhcCxnTYy8vj02o+\nu2Inu2pb7DSZ5A71roOtXPbQUvv75Lvm2fLb91Kv7dMd75iFz1c5Vl1Obkc2pAoMAJZs7d3CdM6f\nLyQGDxf+KvNsTj3hPO+Sm85NeOzxD8p5/INy/vr9MxjruLP+6a462sJR7jFHJ97ZWM2Um+fzzTmT\nuPtLx6Z8nUg0xorygxw3cTjVje1MHzOMy3+71B5deervO2hoC/PSJ4lrVMz/bB9XzZnEXeZ5reDl\nxouPpLK+jW//fiXPXjeHsuL49TU5Rsn+sKzCDg4i0RhrKxsYX5LPwZYQU0cX8q7ZCT7YEuK8Bxbz\no/Nn8J3PT6X8QDN3zt/IV0+ewIVHl7Fkaw3nHmlM4/7sip2U5Pv5wvHjOrXzyJ+/2WnfNU+t5NGr\nTuLiWYfh8Siqm9qpqGnla4/Ha1SOHT+cZdtrKQx6ueb0KZ3OccPzRuH+7oNt3PD8av7hxPH27+79\nG8/hf5aU84svHsOOmhbKDzRz0az4/e8PthgBY3s4xrrKBvIDXqaNLuJHz33Kgs/2U3Hvpby/5QDN\n7RGWl9eQ7xiZ+Ny973La1JHcd8WxNLSFyQ94KQh0/+bun1fuYlxJPmcfMbrbz+krqjcrCSul5gK/\nAs7WWh9w7J8F/C9GncE4YBEwQ2udcaqg2bNn61WrVmU6JK2fvriWxVuqaWyLJMxItOzm8zjjXuMu\nwOdnjLLfhCaNKGBPXWvG6dFOmVLKv100kyufWEHA62HLXfPSHyyEEH1MKfWx1np2rq+ju8zPiN8A\nXuBJrfW9SY8r8/FLgFbgWq116uRu06F+TmSa0UeIXJg4Ij9hdqZNd8zFo5Sd2nWovnHqRC6edRjX\n/v4jAOb/65lc+uDSTsdddHSZncLm9ML3TmfBZ/v4/d8r7H1jh+dRFPSxtbqZt244i4t//QEAd1w+\ni5+nGOG59owpXHBUGVf/zgg4b7x4Jt88dRL5AS+Pv1/OJ7vqeH9LfOSmIOClNakY/AfnTuPh97Zn\nbOvnZ4zic9NHMbNsGCdNLuWhRVvtVK/u+uLx43htzV6evGY233mm83vLBzeey1n3p67RSeX4iSWs\n2V1PYcDLD8+bwYhCP0eNLeappTu44uQJnDl9FG3hKH6vh1/8bT1fOXkiJ0wssd+jtt99ySFnqmTr\nM6K3wcE2IAhYt1FWaK2/Zz52K0YdQgS4QWvd5V97b4KDXy3czIMpVkX+n2tm27M0jCoKUNOcebTA\n6cRJJdxx+TFc9tBSSgv8fHrbRYd0bUIIkQ2DKTgwZ6jbAlyIUXf2EfANrfUGxzGXAP+CERzMAX6j\ntZ6T6bwSHAgh3G7HPZcc0lpX2fqM6FUyu9Z6eobH7gLu6s35e2JMceoCmflr99rbqQKDMcOCaQub\nIlFtF7MUSt6/EEL0xKnANq11OYBS6jmMmew2OI65HHjGTEldoZQqUUqN1VrLvJFCiCHrpU8q+crJ\nh75yeW+5Zg7Nw0cV2tvzjonnj72yem+qw23nzhzTad9U81wXHl1GqTmT0BUn5e6XJIQQg9B4wLnq\nVKpZ67pzDEqp65VSq5RSqw4cSF1EKoQQbuGcfjcXXHM7/IxpI3nnJ2cRjcERZUUcbAnRFo5S3xqm\nKOjD51XUt4aZOKIApWDPwTYKAl4mlOZz7eemMK4kH69HEdOaoM9Da4dRle7xKD75+YWUpKhQF0II\n0feyMXHFtrvmEYlpAl4PLaEI+xva8XgUZcV55Pk8+MzpRkORGOFojHy/F49HEYrEUMqY3KI9HKW2\nJURxnp+GthC769oYFvRR09xBaUGA1lCUgy0hFm85QExrZk8upTjPj1JQWhigKOijoqaFhjbjsygW\n04wsCpLv95If8JDn93KwJUS+30tB0Mfe+ja0NibHKM7zUxT00RqKUhDwsml/ExW1Leypa+WoscUU\nBX2UFefREYnR0BbmYHMHU0cX0dIRIao1IwoDHGjqYNroIupaQygUeX4PO2tbCUdjzCgbRkc4Sk1L\niOH5ftbvbSDf76WkwM+YYXm0haLsa2zHqxSFQWNEfXt1MydNLuWTnXU8saScKSMLOf+oMSzdWoPX\noxg7PJ88v5eATxHwenh/ywFGDwsye8oIfB7FmOI8Sgv8VNa1kR/wopRiWNCHUsaMUFWN7Rw2PN9e\nMfryE8axZGsNU0cVsnl/E03m7ITfPWsqC9btw6MUJ04sIRzV7KlrJRzV5Pk9fLKr3v47OH5iCQ2t\nIc47sozDhgepaw3jVYpwNEbQ72XGmCJiWtMWihLweWhsC7NqZx0FZnHp+r0N7GtoJ+D1UF4TX5Bu\n6qhCmjsiVDd1cPLkUiLRGIVBH/l+b6fZbI6bMJy1SVO3JtcfiKFt851zc/r6vao5yLbe1BwIIYTb\nDbKag9OB27XWF5vf3wKgtb7HcczjwGKt9Z/N7zcD52RKK5LPCSGESC1bnxGuSSsSQggxoHwEzFBK\nHa6UCgBXAq8lHfMacI0ynAY0SL2BEELklmvSioQQQgwcWuuIUuqHwFsYU5k+pbVer5T6nvn4Y8AC\njJmKtmFMZfrtXF2vEEIIgwQHQggh+oTWegFGAODc95hjWwM/6O/rEkIIkZ6kFQkhhBBCCCEACQ6E\nEEIIIYQQJgkOhBBCCCGEEIAEB0IIIYQQQgiTBAdCCCGEEEIIQIIDIYQQQgghhEmCAyGEEEIIIQQA\nyphmemBQSh0Adh7i00cBNVm8nIFO2utu0l53O9T2TtZaj872xQwm8jnRI0OtvTD02iztdb+etDkr\nnxEDKjjoDaXUKq317FxfR3+R9rqbtNfdhlp7B4qh9nMfau2Foddmaa/75aLNklYkhBBCCCGEACQ4\nEEIIIYQQQpjcFBw8kesL6GfSXneT9rrbUGvvQDHUfu5Drb0w9Nos7XW/fm+za2oOhBBCCCGEEL3j\nppEDIYQQQgghRC+4IjhQSs1VSm1WSm1TSt2c6+vJBqXUU0qpaqXUOse+EUqpt5VSW81/Sx2P3WK2\nf7NS6uLcXPWhU0pNVEq9p5TaoJRar5T6kbnflW1WSuUppVYqpdaY7f2Fud+V7QVQSnmVUp8qpV43\nv3dtWwGUUhVKqc+UUquVUqvMfa5u80A2mD4nsvl+qJQ62fw73KaUelAppcz9QaXU8+b+D5VSUxzP\n+Zb5GluVUt/qx3b3+j1ikLW3RCn1olJqk1Jqo1LqdDe3WSn1Y/PveZ1S6s/K+Bx0TXtVlvpt2Wyf\nUupw89ht5nMD3WqM1npQfwFeYDswFQgAa4Cjc31dWWjXWcBJwDrHvv8Cbja3bwbuM7ePNtsdBA43\nfx7eXLehh+0dC5xkbg8DtpjtcmWbAQUUmdt+4EPgNLe212zDT4D/BV43v3dtW812VACjkva5us0D\n9WuwfU5k8/0QWGm+tyjgDWCeuf/7wGPm9pXA8+b2CKDc/LfU3C7tp3b3+j1ikLX3aeA75nYAKHFr\nm4HxwA4g3/z+L8C1bmovWeq3ZbN95s/5SnP7MeCfu9WW/vgP0Md/cKcDbzm+vwW4JdfXlaW2TUn6\nI9sMjDW3xwKbU7UZeAs4PdfX38u2vwpcOBTaDBQAnwBz3NpeYAKwCDiP+Ae/K9vquO4KOgcHrm7z\nQP0a7J8Th/p+aB6zybH/G8DjzmPMbR/GIkvKeYz52OPAN/qhjb1+jxhk7R2O0VlWSftd2WaM4GA3\nRgfWB7wOXOS29tLLfls222c+VgP4zP0J74OZvtyQVmT9wVn2mPvcqExrvc/c3g+Umduu+hmYQ2Un\nYtxNd22bzSH01UA18LbW2s3t/TVwExBz7HNrWy0aeEcp9bFS6npzn9vbPFAN2p9vL98Px5vbyfsT\nnqO1jgANwMgM5+pr2XiPGEztPRw4APzeTKV6UilViEvbrLWuBH4J7AL2AQ1a64W4tL0OuWzfSKDe\nPDb5XBm5ITgYkrQRBupcX0e2KaWKgJeAG7TWjc7H3NZmrXVUa30Cxh2zU5VSxyQ97or2KqUuA6q1\n1h+nO8YtbU1ypvn7nQf8QCl1lvNBl7ZZZNFQeT8cou8RPowUlEe11icCLRhpJzY3tdnMtb8cIyga\nBxQqpa52HuOm9qYymNrnhuCgEpjo+H6Cuc+NqpRSYwHMf6vN/a74GSil/BgfhH/SWv/V3O3qNgNo\nreuB94C5uLO9nwO+qJSqAJ4DzlNK/RF3ttVm3ilDa10NvAycisvbPIANup9vlt4PK83t5P0Jz1FK\n+TDSXGoznKsvZes9YrC0F4y7uHvMEWOAFzGCBbe2+QJgh9b6gNY6DPwVOAP3tteSy/bVAiXmscnn\nysgNwcFHwAyzIjuAUaTxWo6vqa+8BnzL3P4WRh6qtf9Ks5L9cGAGRkHLoGFW4/8O2Ki1/pXjIVe2\nWSk1WilVYm7nY+QTb8KF7dVa36K1nqC1noLx//NdrfXVuLCtFqVUoVJqmLWNkVu7Dhe3eYAbVJ8T\n2Xo/NNMZGpVSp5nnvCbpOda5voLx/1Jj5DVfpJQqNe/2XmTu6zPZeo8YLO0F0FrvB3YrpWaau84H\nNuDeNu8CTlNKFZjXeT6wEfe215Kz9pmPvWcem/z6mfVFQUZ/fwGXYMzmsB24NdfXk6U2/RkjLy+M\ncYfhOoz8sUXAVuAdYITj+FvN9m/GrGwfTF/AmRjDbWuB1ebXJW5tM3Ac8KnZ3nXAbeZ+V7bX0YZz\niBcburatGLPirDG/1lvvS25u80D/GkyfE9l8PwRmm+8x24HfEl/8NA94AdiGEYhOdTznH83924Bv\n93Pbe/UeMZjaC5wArDJ/z69gzDTj2jYDv8C4CbYOeBZjph7XtJcs9duy2T6Mz6KV5v4XgGB32iIr\nJAshhBBCCCEAd6QVCSGEEEIIIbJAggMhhBBCCCEEIMGBEEIIIYQQwiTBgRBCCCGEEAKQ4EAIIYQQ\nQghhkuBACCGEEEIIAUhwIIQQQgghhDBJcCCEEEIIIYQA4P8At5NMkEZfHKQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fda57ee1fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 1000000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = current_model.act(state, epsilon)\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > replay_initial:\n",
    "        beta = beta_by_frame(frame_idx)\n",
    "        loss = compute_td_loss(batch_size, beta)\n",
    "        losses.append(loss.data[0])\n",
    "        \n",
    "    if frame_idx % 10000 == 0:\n",
    "        plot(frame_idx, all_rewards, losses)\n",
    "        \n",
    "    if frame_idx % 1000 == 0:\n",
    "        update_target(current_model, target_model)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}