Reinforcement-Learning-Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "36886af7",
   "metadata": {},
   "source": [
    "# Install libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd72cfbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting box2d-py\n",
      "  Using cached box2d-py-2.3.8.tar.gz (374 kB)\n",
      "Building wheels for collected packages: box2d-py\n",
      "  Building wheel for box2d-py (setup.py): started\n",
      "  Building wheel for box2d-py (setup.py): finished with status 'error'\n",
      "  Running setup.py clean for box2d-py\n",
      "Failed to build box2d-py\n",
      "Installing collected packages: box2d-py\n",
      "    Running setup.py install for box2d-py: started\n",
      "    Running setup.py install for box2d-py: finished with status 'error'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  ERROR: Command errored out with exit status 1:\n",
      "   command: 'C:\\Users\\Karim BOUKADIDA\\anaconda3\\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' bdist_wheel -d 'C:\\Users\\Karim BOUKADIDA\\AppData\\Local\\Temp\\pip-wheel-xu0e9xwo'\n",
      "       cwd: C:\\Users\\Karim BOUKADIDA\\AppData\\Local\\Temp\\pip-install-mk8c9oif\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\n",
      "  Complete output (16 lines):\n",
      "  Using setuptools (version 52.0.0.post20210125).\n",
      "  running bdist_wheel\n",
      "  running build\n",
      "  running build_py\n",
      "  creating build\n",
      "  creating build\\lib.win-amd64-3.8\n",
      "  creating build\\lib.win-amd64-3.8\\Box2D\n",
      "  copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-3.8\\Box2D\n",
      "  copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-3.8\\Box2D\n",
      "  creating build\\lib.win-amd64-3.8\\Box2D\\b2\n",
      "  copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-3.8\\Box2D\\b2\n",
      "  running build_ext\n",
      "  building 'Box2D._Box2D' extension\n",
      "  swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "  swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "  error: command 'swig.exe' failed: No such file or directory\n",
      "  ----------------------------------------\n",
      "  ERROR: Failed building wheel for box2d-py\n",
      "    ERROR: Command errored out with exit status 1:\n",
      "     command: 'C:\\Users\\Karim BOUKADIDA\\anaconda3\\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record 'C:\\Users\\Karim BOUKADIDA\\AppData\\Local\\Temp\\pip-record-y_f_dtwr\\install-record.txt' --single-version-externally-managed --compile --install-headers 'C:\\Users\\Karim BOUKADIDA\\anaconda3\\Include\\box2d-py'\n",
      "         cwd: C:\\Users\\Karim BOUKADIDA\\AppData\\Local\\Temp\\pip-install-mk8c9oif\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\n",
      "    Complete output (16 lines):\n",
      "    Using setuptools (version 52.0.0.post20210125).\n",
      "    running install\n",
      "    running build\n",
      "    running build_py\n",
      "    creating build\n",
      "    creating build\\lib.win-amd64-3.8\n",
      "    creating build\\lib.win-amd64-3.8\\Box2D\n",
      "    copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-3.8\\Box2D\n",
      "    copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-3.8\\Box2D\n",
      "    creating build\\lib.win-amd64-3.8\\Box2D\\b2\n",
      "    copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-3.8\\Box2D\\b2\n",
      "    running build_ext\n",
      "    building 'Box2D._Box2D' extension\n",
      "    swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "    swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "    error: command 'swig.exe' failed: No such file or directory\n",
      "    ----------------------------------------\n",
      "ERROR: Command errored out with exit status 1: 'C:\\Users\\Karim BOUKADIDA\\anaconda3\\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"'; __file__='\"'\"'C:\\\\Users\\\\Karim BOUKADIDA\\\\AppData\\\\Local\\\\Temp\\\\pip-install-mk8c9oif\\\\box2d-py_ae3a6f8973bc474daab159a07c23116d\\\\setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' install --record 'C:\\Users\\Karim BOUKADIDA\\AppData\\Local\\Temp\\pip-record-y_f_dtwr\\install-record.txt' --single-version-externally-managed --compile --install-headers 'C:\\Users\\Karim BOUKADIDA\\anaconda3\\Include\\box2d-py' Check the logs for full command output.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gym[Box_2D] in c:\\users\\karim boukadida\\anaconda3\\lib\\site-packages (0.21.0)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\karim boukadida\\anaconda3\\lib\\site-packages (from gym[Box_2D]) (1.6.0)\n",
      "Requirement already satisfied: numpy>=1.18.0 in c:\\users\\karim boukadida\\anaconda3\\lib\\site-packages (from gym[Box_2D]) (1.20.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: gym 0.21.0 does not provide the extra 'box_2d'\n"
     ]
    }
   ],
   "source": [
    "!pip3 install box2d-py\n",
    "!pip3 install gym[Box_2D]\n",
    "import gym\n",
    "env = gym.make(\"CartPole-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6f927656",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyglet in c:\\users\\karim boukadida\\anaconda3\\lib\\site-packages (1.5.21)\n"
     ]
    }
   ],
   "source": [
    "!pip install pyglet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a9a8738",
   "metadata": {},
   "source": [
    "# Model and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4747d6ee",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Karim BOUKADIDA\\anaconda3\\lib\\site-packages\\keras\\optimizer_v2\\adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0/100, score: 20.0\n",
      "episode: 1/100, score: 16.0\n",
      "episode: 2/100, score: 15.0\n",
      "episode: 3/100, score: 53.0\n",
      "episode: 4/100, score: 19.0\n",
      "episode: 5/100, score: 23.0\n",
      "episode: 6/100, score: 20.0\n",
      "episode: 7/100, score: 9.0\n",
      "episode: 8/100, score: 14.0\n",
      "episode: 9/100, score: 13.0\n",
      "episode: 10/100, score: 12.0\n",
      "episode: 11/100, score: 15.0\n",
      "episode: 12/100, score: 12.0\n",
      "episode: 13/100, score: 14.0\n",
      "episode: 14/100, score: 15.0\n",
      "episode: 15/100, score: 10.0\n",
      "episode: 16/100, score: 9.0\n",
      "episode: 17/100, score: 16.0\n",
      "episode: 18/100, score: 17.0\n",
      "episode: 19/100, score: 11.0\n",
      "episode: 20/100, score: 9.0\n",
      "episode: 21/100, score: 16.0\n",
      "episode: 22/100, score: 14.0\n",
      "episode: 23/100, score: 15.0\n",
      "episode: 24/100, score: 16.0\n",
      "episode: 25/100, score: 13.0\n",
      "episode: 26/100, score: 22.0\n",
      "episode: 27/100, score: 30.0\n",
      "episode: 28/100, score: 60.0\n",
      "episode: 29/100, score: 19.0\n",
      "episode: 30/100, score: 60.0\n",
      "episode: 31/100, score: 25.0\n",
      "episode: 32/100, score: 46.0\n",
      "episode: 33/100, score: 67.0\n",
      "episode: 34/100, score: 81.0\n",
      "episode: 35/100, score: 146.0\n",
      "episode: 36/100, score: 261.0\n",
      "episode: 37/100, score: 166.0\n",
      "episode: 38/100, score: 237.0\n",
      "episode: 39/100, score: 269.0\n",
      "episode: 40/100, score: 320.0\n",
      "episode: 41/100, score: 196.0\n",
      "episode: 42/100, score: 196.0\n",
      "episode: 43/100, score: 258.0\n",
      "episode: 44/100, score: 181.0\n",
      "episode: 45/100, score: 211.0\n",
      "episode: 46/100, score: 268.0\n",
      "episode: 47/100, score: 218.0\n",
      "episode: 48/100, score: 458.0\n",
      "episode: 49/100, score: 233.0\n",
      "episode: 50/100, score: 282.0\n",
      "episode: 51/100, score: 279.0\n",
      "episode: 52/100, score: 217.0\n",
      "episode: 53/100, score: 275.0\n",
      "episode: 54/100, score: 425.0\n",
      "episode: 55/100, score: 262.0\n",
      "episode: 56/100, score: 231.0\n",
      "episode: 57/100, score: 315.0\n",
      "episode: 58/100, score: 325.0\n",
      "episode: 59/100, score: 260.0\n",
      "episode: 60/100, score: 238.0\n",
      "episode: 61/100, score: 261.0\n",
      "episode: 62/100, score: 353.0\n",
      "episode: 63/100, score: 213.0\n",
      "episode: 64/100, score: 234.0\n",
      "episode: 65/100, score: 306.0\n",
      "episode: 66/100, score: 249.0\n",
      "episode: 67/100, score: 263.0\n",
      "episode: 68/100, score: 257.0\n",
      "episode: 69/100, score: 221.0\n",
      "episode: 70/100, score: 251.0\n",
      "episode: 71/100, score: 238.0\n",
      "episode: 72/100, score: 272.0\n",
      "episode: 73/100, score: 235.0\n",
      "episode: 74/100, score: 215.0\n",
      "episode: 75/100, score: 213.0\n",
      "episode: 76/100, score: 231.0\n",
      "episode: 77/100, score: 256.0\n",
      "episode: 78/100, score: 238.0\n",
      "episode: 79/100, score: 217.0\n",
      "episode: 80/100, score: 224.0\n",
      "episode: 81/100, score: 28.0\n",
      "episode: 82/100, score: 157.0\n",
      "episode: 83/100, score: 21.0\n",
      "episode: 84/100, score: 160.0\n",
      "episode: 85/100, score: 197.0\n",
      "episode: 86/100, score: 220.0\n",
      "episode: 87/100, score: 261.0\n",
      "episode: 88/100, score: 219.0\n",
      "episode: 89/100, score: 238.0\n",
      "episode: 90/100, score: 352.0\n",
      "episode: 91/100, score: 210.0\n",
      "episode: 92/100, score: 335.0\n",
      "episode: 93/100, score: 267.0\n",
      "episode: 94/100, score: 182.0\n",
      "episode: 95/100, score: 177.0\n",
      "episode: 96/100, score: 218.0\n",
      "episode: 97/100, score: 233.0\n",
      "episode: 98/100, score: 183.0\n",
      "episode: 99/100, score: 208.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1zElEQVR4nO3deXzcZ33g8c8zmkOjc3Rbl+8jsRPbSRw7CVcOQgKBhBZoTTlcSAvs0gLbbimUZem2S0u73d2ypWmbF6S5CCEkgYQ0FEIIR5rg207iOLZkW5YlW/fMSJoZaa5n/5j5jUbSjObQHBrp+369/BrpN4een21955nv7/t8H6W1RgghxPJiKvYAhBBC5J4EdyGEWIYkuAshxDIkwV0IIZYhCe5CCLEMmYs9AIDGxka9du3aYg9DCCFKyuHDh0e01k2J7lsSwX3t2rUcOnSo2MMQQoiSopQ6n+w+ScsIIcQyJMFdCCGWIQnuQgixDElwF0KIZUiCuxBCLEMS3IUQYhmS4C6EEMuQBHch4miteeJwH5PTwWIPRYhFkeAuRJyeUS9//L3jPPvKpWIPRYhFkeAuRJzRyWkAhqO3QpQqCe5CxHF6A5Fbj7/IIxFicSS4CxHH6Y0E9TEJ7qLESXAXIo7LCO5eCe6itElwFyKOkZaRmbsodRLchYjjkrSMWCYkuAsRx+mRmbtYHiS4CxHHuKDq9YeYCoSKPBohsifBXYg4rmjOHWT2LkqbBHch4ji9fuoqLIAEd1HaJLgLEaW1xuUNsKGpCpDgLkqbBHchorz+EP5QmPVNlYAEd1HaJLgLEWVcTJWZu1gOJLgLEWVcTF3TUIFJLT64P/vqJX7znv9Aa52L4QmREQnuQkQZM/f6Sht1FdZFtyA4ct7JkV6X9IYXRSHBXYgoo/VAXYWF+korY5OLC+4uX+T14ssrhSgUCe5CRBmtBxwVVuoqFz9zd0eDu1OakIkikOAuRJTResBRYaGh0rronLtbZu6iiCS4CxHl9PqptpmxlJmoq7QuesMOt1dm7qJ4JLgLEeXy+nFURlanNlRacXr9hMPZV7oYM3fjVohCkuAuRJTTG6CuwgpAXYWVsF5cYI7l3D0S3EXhSXAXIsrl9eOIBveGqsjtaJapmelgCF+0q6SkZUQxpB3clVJlSqmjSqlnot/XK6WeU0p1RW/r4h77RaVUt1LqlFLqtnwMXIhci8zcI2mZ+kpr9Fh2gTl+xi9pGVEMmczcPwucjPv+C8DzWutNwPPR71FKbQX2AtuA24F7lFJluRmuEPkT6Qg5k5YBGM2y1t0dVyEjM3dRDGkFd6VUB3AH8M24w3cBD0S/fgB4b9zxR7XW01rrc0A3sDsnoxUiT4KhMBNTQRzRmbuRllnszN1sUrHFUUIUUroz978HPg+E4461aK0vAURvm6PH24ELcY/rix6bRSn1CaXUIaXUoeHh4UzHLUROGatJ587cs611N4J7R50dt8zcRRGkDO5KqXcDQ1rrw2m+pkpwbF49mdb6Xq31Lq31rqampjRfWoj8mFmdGpm5l1vKqLSWZR3cZ5qQVcrMXRRFOjP3NwF3KqV6gEeBm5VSDwODSqlWgOjtUPTxfUBn3PM7gIs5G7EQeTDTV8YaO1a3iFWqxsx9bUMF41MBQouolxciGymDu9b6i1rrDq31WiIXSn+mtf4w8DSwL/qwfcBT0a+fBvYqpWxKqXXAJuBAzkcuRA4Zq1Hjg/tiWhAYwb2zvgKtYVwqZkSBmRfx3K8Bjyml7gZ6gQ8AaK1PKKUeA14HgsCntdayjbxY0ow0ipGWgcjMPetqGV+A6nIzjVU2IFqJU2lN8Swhciej4K61/jnw8+jXo8AtSR73VeCrixybEAVjVMXEB+D6Sitdg5NZvZ7bF6DWbqE2+mbhkpm7KDBZoSoEkZy7pUxRaZ1ZklFfsbi0TK3dEkvzuKRiRhSYBHchmGk9oNRMsVd9lRVfIITPn3lWMfJ6ltiKV2n7KwpNgrsQGKtTLbOO1Ru17lnMuo2Zu8NuLIaS4C4KS4K7EESCr6Ni9gXPWH+ZLFIzRnCvLjdjUpKWEYUnwV0IIsF33sy9MrvOkFrraHC3YjIpHBVWScuIgpPgLgSze7kbjOA+5pnO6LV8gRCBkKbWHnmzcNgt0jxMFJwEd7Hiaa1n9XI3zAT3zGbdc2vmHRUWmbmLgpPgLlY8jz8y056blqkpt1BmUhnP3I3VqbGZe4UVl09m7qKwJLiLFS9R6wEAk0lRV2HJeOY+P7hbZKs9UXAS3MWKl6j1gKG+0prxzN14PSO411VYZTcmUXAS3MWKl6j1gKGuwprxrHt87szdbmFyOog/GF7oaULklAR3seLFgnuCmXtDlZXRbHPuxgXV6JuG5N1FIUlwFyveTFomycw9w0oXl89PmUlRbTNHXyMS5N1SMSMKSIK7WPGMmbvDnmDmXmnF6fVntNmG2xegptwc61OTzxYEbl+AwfGpnL+uKH0S3MWK5/JGeq+by+b/OtRVWtGajC6Iun3BWZ8CjAu1+VjI9JfPvM6++2QvnHx44KUe/uz7rxIu0V20JLiLFS/SNCzxRhrZrFJ1ef3UxH0KcOQxLdM1NMnZYU/JBqCl7N9eucQj+3v5u5+cKvZQsiLBXax4kdYD81MykN0q1fFo0zCD8caRj5l7v9OHPxRmJMOLviK1gfEprGUm7vn5Gb5/tK/Yw8mYBHex4iVqPWDIaubuC8zK31dYy7CWmXK+G9NUIMTIZGRcF12Sd88lrTUD41N8+Lo17FlXz58+8SpHep3FHlZGJLiLFS9RL3dDNjN395yZu1KK2gpLztv+9rt8M187fQs8UmTK6Q3gD4bpqLPzzx++hlU15XziwcNcdJXO37MEd7HiuTzze7kbjJRKujP3cFjPS8tEXif3LQj64gJ6KQWdUjDgjnwSaq0tp67Syjf37WIqEOL3HzyE1x8s8ujSI8FdrGiBUJiJ6WDSC6rlljIqrWVpz9wnpoOE9fxWBg577puHGbN1pWbP4sXiGeWlLbXlAGxuqeYfPngVr18a548fO14SF7AluIsVzVjAVFeZOC0Dkb1U0525G60HaubM3PPR9rff5cVsUmxoqpLgnmMD0eC+qqY8duymy5r5s3dezo9eG+Dvn+8q1tDSJsFdrGhGHjxZWgYie6mOpRmYjXr4uQui6vKwG1Of00ero5zOOrukZXJswD2FUtBUbZt1/Pfeso737GjjH1/oZiqQ+cbphSTBXaxoxqrRZBdUIbPOkHM7QhocFbnfjanf6aPdYafNIcE91wbHp2issmGZs7BNKcVbNjYSCmuGJ5Z2+akEd7GizTQNW2DmXmlL+2Lo3KZhBkeFlelgGJ8/d7O9PqePdkcF7XV2nN5AyVzoKwWX3FOzUjLxjNn88KQEdyGWrJm0zEIzd0vanSFn0jKz3yyMTwa5uqjqD4YZnJiio85Ou8MOSMVMLg2OT9GSKrjLzF2IpctIy9Qn6OVuqK+0MRVIb9ZtBO9EaRlgwU8AU4EQd37jRV7sGkn5cy65fWgN7XWRtAzMLo0UizMwPkVrbeLg3lgVCe4jMnMXYulyev1YzSbslrKkj6mPVtKkM3t3+wJYy0yUW2b/ahkXbBdayNQ9NMkrfW5+fmoo5c8xyiBnz9xllWouTAVCuLwBViUJ7g1VkX9LmbkLkUN//exJDvaM5ez1XJ5IXxmjPW8i9ZWRmdqYJ3VKxe0NUJvg9RyxtEzymfu5EQ8A3cOTKX9OXzQF0+GooLnaRplJSVomR2I17knSMpYyE3UVliUf3M3FHoAQ6Rrz+PmXX57F4w9y7dr6nLzmQh0hDcbMPa3gnmB1KqTXPOzscCS4dw2mEdydPpSCVbXlmMtMrKopl1r3HDFWpya7oAqRvLukZYTIkdODEwBcymH6weUNLHgxFTKcuc9pGmYwAv5Cte7nRiJBvd/lS1n50u/0saqmHKs58ivc7rBLcM+R2AKmWlvSxzRV25b8zF2CuygZXdHgftGdu+Ce1sw91l8mdXB3eRPP3MstZdgtZQvm3M+OeCgzRdI5Z4Y8C/6cPqc3lmsHaHOUS1omR4yZe7K0DEQuqo5MLu09cSW4i5Jxypi5u3MXxJze5E3DDDV2M2Umtai0DESbhyWZuWutOTfsYc+6SLqpa2hiwZ/T7/LRUTcT3Nvr7Ay4pzLaDlAkNjA+RZXNTHV58k90TVWRmbvWS/fvW4K7KBmno7loV44W7GitcS3Q7teglIpulJ06uI/7AvMWMBkcC7QgGJ6cZmI6yE1bmjGbFN1DyfPuwVCYAfcU7XXxM3c7wbBmaEIqZhYrUuOePCUD0FhtwxcI4cnhorRck+AuSoLWmtODE1TbIjUAuSj7m5wOEgzrlGkZiGyUPZriY3gw2mEy2czdsUBP93PRi6mbWqpY21hJ1wLBfXBimmBY0+6oiB1rk4VMOTPgnkpaBmloMmrdl3DePWVwV0qVK6UOKKWOK6VOKKX+R/R4vVLqOaVUV/S2Lu45X1RKdSulTimlbsvnCYiVYXhyGpc3wJs3NQK5Sc0Ys+hUF1Qh0jUy1cx9fCryaSJ5Wib57N8og9zQVMWm5irOLBDc42vcDR2ykClnBsenF8y3Q2m0IEhn5j4N3Ky13gHsBG5XSl0HfAF4Xmu9CXg++j1Kqa3AXmAbcDtwj1Iq+QoRIdJweiAS7G7c0gTkpmImnb4yhoZKG6Mpcu6x1gNJ3ixqKyyxx8x1dsSD1WyizWFnY3MVPaMepoOJP/L3Ob0As9IyrbKQKSfCYc3gePK+MobG5TBz1xHGNMIS/aOBu4AHoscfAN4b/fou4FGt9bTW+hzQDezO5aDFymOUQb5lUyS4X8zBzN2ZRi93Q12lBWeK4G6kXBa6oOryBhJehDs77GFtQwVlJsXG5irCGnpGvAlfx5i5x1fLVNnM1NotkpZZpBFPJOWVMi2zTGbuKKXKlFLHgCHgOa31fqBFa30JIHrbHH14O3Ah7ul90WNzX/MTSqlDSqlDw8PDizgFsRKcHpygvtJKm8NOY5UtJzP3dHq5G+orbbh8gQWrUWIdIe3Jt+wLhjWT0/MvBp8dmWRdYyUAG5urgOQVM/0uH41VNsrntEyQWvfFG3RHgnWqtEx9pRWTWtotCNIK7lrrkNZ6J9AB7FZKXbHAwxOt4573G6G1vldrvUtrvaupqSmtwYqV6/TgBJtbIkGvzVGem5m7J/20TH2FBa0X7g0zE9yTpGWSLGQKhsL0jnpZ3xQ5vw1NVShF0oqZPqdvVkrGIH3dF89YwJSsaZihzKSor1zaq1QzqpbRWruAnxPJpQ8qpVoBordGt6M+oDPuaR3AxcUOVKxcWmu6BifZ3FINRH7xLuVgIZPTG0Cp5ME4Xn1V6lWqqYJ7shYEfU4fwbCOzdzLLWV01lUkrZiZW+NuaHeUx1I2IjuJttdLZqmvUk2nWqZJKeWIfm0H3g68ATwN7Is+bB/wVPTrp4G9SimbUmodsAk4kONxixXkknuKielgLLgbM9TFLiBxef3UlFtiq0IXks4qVXeSXZgMseZhc2buZ6NtB9ZHgzuQtGImHNb0O32x6ph47XV2JqaDjE/ldju/lWTQPUWZSdFQtXCdO0BjlZXhJbxKNZ3GYa3AA9GKFxPwmNb6GaXUy8BjSqm7gV7gAwBa6xNKqceA14Eg8Gmt9dKt9BdLnrEyNRbca+14/SHGfcGkC4bS4fQGUi5gMhj93hcK7i5fgAprWazfy1yOJDN3o2GYkZaBSN79V10jBENhzHFbvY1MTuMPhRPO3ONr3WtWZf/3spJdck/Fumym0lRti/3bLUUpg7vW+hXgqgTHR4Fbkjznq8BXFz06IZjpKWPk3FsdkY/MF92+RQZ3f1oXUyEuuKfIuS+U4jHeSOaWQ54d8VBrt8x6o9nYXIU/FOaC0xdL1wBcMCplFgju/U4fl62qSXVKIoGFdmCay0jLaK0XbBldLLJCVSx5pwYmaa62xQJxa20kiC12IZMrg5m7US45tsDH8FTB3bhv7m5M54Y9rG+qnBUgYhUzg7MrZoxqmPjVqYYOWaW6aANp1Lgbmqps+EPh2OK1pUaCu1jyuoYm2LKqOvZ9mzFzX2Q5ZDodIQ02cxnVNvOCdc3uJB0hDeYyE9Xl5vlpmbgySIMR3Odu3NG/wMy9scqGtcxEvyxkytpgGq0HDEt9L1UJ7mJJC4cjlTKbmmeCe3N1OWUmlZOZe7ppGYj0fXn94njS+92+1L3h6yqss9Iynukgg+PTbIjLtwNUl1tYVVNO95yNO/qcXhwVFqps8zOqJpOi1SGbdmTLMx1kYjqYdnBf6nupSnAXS1qf04cvEGLLqpngV2ZStFQvbiGTPxhmcjqYdloG4KrVdbza7yYQCie8P1VaBiIVM/Ezd6OnzNyZO0TeTObN3JOUQRraaqXWPVuZlEGCzNyFWBSjUmZTS/Ws420O+6IWMrl80dWplenP3Hd2OpgOhnnjUuKVoy6fP43gbp3V0/3siFEpMz+4b2iqontoknDcqtg+p29W24G5ZCFT9tLZpCOe0RlSgrsQWTB6ymxqnp22aHXYF5VzN2rNM5u5OwA4dsE5777pYIipQDhlmsdht+COn7lHS+nWNiSeuXv9IS5FZ5RaR2rcE11MNbTX2Rkcn0r66SKXDpwb49v7z+f95xRKbO/UNNMytXYLZpOStIwQ2Tg9OEG7wz5vV5y22nIG3FOzZrWZyKT1gKHdYaep2sbRXte8+4w8ek2Kmfvc3ZjOjkzS7rDP6xMDsLFpdsWM0xvAFwgtmJZpd5QT1jOBKl9GJqf51MOH+cpTJ3KyccpSkGlaxmRSNFYt3VWqEtzFknZ6cDJW3x6vtbYcfyicsg1vMs4MerkblFLs7HRw9IJr3n2pVqcaHBVWxqdmGpCdG/EkTMnATCrK6DGTqNXvXLFa9zymZrTWfPkHrzHm8RMMaw6fn/9JphQNjk9Ra7dgt6bfobyx2rpkO0NKcBdLVjAU5szQTE+ZeEb/8mwrZnrHIumQhfLXiVy12sG5Ec+89r+xXu5pXFDVOrIdn9aas8OehBdTIbJwqr7SGgvuiTbpmKu9ALXuz7xyiR+9NsAf3LSRMpNi/9mxvP2sQhpwp1/jbmiqWrrNwyS4iyWrZ9SLPxROGNzbahe3OUX30CSNVbaMSiEhclEV4Fifa9ZxV5oz9/jmYcOT00xOB2f1lJlrY3NV3Mw9GtwXyLnne7u9oYkpvvzUa+zsdPC5t2/iivZafn12NC8/q9AGx6doSTPfbljKzcMkuIuiOTUwwWcfPZqwvznEtx1INHOP/BJmO3PvGpqcd5E2Hds7HJgUHJuTd0+1C5PBaJfg8gViF1PXNSUfx8bmKrqGJiMXU10+qmxmauzJu4aUW8poqLTmJS2jteZL338Nrz/E331gB+YyE9etr+d4nwvfEt4oOl2R1ampG4bFa6yyMTrpz/raTz5JcBdF86uuYZ46dpF/fKE74f2nBidQama1ZryGSitWsymr1r9aa7qHJhO+bipVNjObW6rn5d1Ttfs1GDN3l9c/Uwa5wMx9U3MVbl+AkUk/fc5IjXuqPibtdfa8rFJ96thFnnt9kD95x5bY39116xoIhDRHeks77x4MhRmemM48LVNtIxjWuJJsn1hMEtxF0RgdFr/5q7OxxTzxugYnWVNfkfACl1KKttryrNIPQxPTTEwF2ZTgQm06rlrt4PgF16zZmvHLPbeqZy6j9NLpCXAubt/UZOJ3ZepzetO6RpDpQqZAKMx9L55jz1/9lN/655d55pWL80opB8en+O9PvcY1a+r4+JvXxY7vWluHScH+Ek/NDE9OE9ZklZaBpVnrLsFdFI3T66faZsZmLuMvn3l93v2nBifmLV6K15rlakwjh71xgXTIQq7qrMPtC3BudOYNadwXoKbcnLJVrCO6BZ/LF+Ds8GRs39RkjLYLZ4YmU65ONbQ57PQ7U/e711rz/MlBbvu/v+QvnnmdNQ2VXBr38QePHOUtf/MC//B8FyOTka6HX3zyVfyhMP/r/dtnjbe63BLJu58r7YuqsRr3DGfuS7kFQTr93IXIizGPnzaHnfdf08FXnz3Jz94Y5ObLWoDIoqCeEQ+3b1uV9PmtjnJePpP5jNHI5W/Mcua+M7qY6WivK9YTxu0LpNV+uLrcjEnNpGU2Nyd/8wJoqbFRZTNzpNfFxFRwwTJIQ3udHV8gFOl6mWQF7hsD4/zPZ07yYvcI65sque93d3HTlmbCGl54Y4gHXu7hfz93mn/4WTe71tbx0plR/vu7t87qOW/Ys66eB146z1QglLBevxQMjme2gMkgM3chEhjz+KmrtLDvhrWsb6rkL374OtPByIW5cyMegmG9YOqkrTayGjOY4WrM7uFJasrNseXjmdrYVEW1zTxrparLm7r1AEQWvjgqrIxMTtM76mVdkhp3g1KKjc1V/PJ0ZBP5hVanGtqjF5sTXVT1+UP82fdf5V1f/xWv9rv5ynu28uPPvZWbL2tBKUWZSfH2rS08dPcefvpHb2Pv7k6OX3Bx/foGfveGtQl/3p51DfhD4YSLu0rFcpy5S3AXRTPm8dNQacNqNvHn79lGz6iXb714DogsXgJmtfqdqzW6GnMow1lT1+Akm1qqs95gwWRSbO+snRXM3L5ALOWSisNu4dV+N8GwXvBiqmFjc1VssVY6aRnjDWBucPcHw3zy4cN850Av+25Yyy/+5EY+9qZ1WMoSh4GNzVX8xV1XcPjLt/LAx3djSpI+unZdPUrB/nOlm3cfGJ/GWmaKbcqSrppyM1azSWbuQsRzegOxTTDeurmJW7e28I2fdTPgnuL0wARlJpV0gQ/M1LpnWg55Zngy63y74arOOt4YmIiVAKbTEdLgqLBwMtp8LNnq1HjxJZvppGVm+t3P/L0EQ2E+992j/PL0MF/7zSv5ynu2pV3jX25JvnUgRCqEtrbWlHS9+4DbR3ONLeM3fKUUTUu0BYEEd1EUobDG5fXHNp4G+PIdWwmGNX/9o5OcHpxgXWMlNnPyHG5rFpt2OD1+Rib9WVfKGHZ2OgiFNa/2u4H0c+4QaUFgtB9Y35h6HEbFTLnFREMaM8v6SivlFlMsuIfDkQuiz746wH+743J++9rVaY0zE3vWNXC01xVLq5WaTHZgmqux2rYkWxBIcBdFMe4LENbMuuC3uqGCT751PU8du8h/dI8k7CkTL5vt9oz+6BuyqHGPtzOuQ6TWOuOZu3Gb7IJnPKNipt2RusYdomWiDjv9rkjFzF/+2+t873Afn71lE7/3lvVpjTFT162vZzoY5vgFd15eP98Gx6czLoM0yMxdiDjGRtNzc5z/+caNtNWW4/GHEq5MjVdTbqbKZs5o5t4VzeVnszo1XmOVjc56O0d7XfgCIQIhnXZwNxYyLZRyitdeZ8dmNtFel/piauw5jshCpr//aRf/+h89fOxNa/nc2zel/fxM7Y7m3bNJzVwY8/JXz57kP7pH8jCy1LTWDLinaM1y5t5UbWVkgb11i0WCuyiKZC137dYyvnTHVgCuaKtd8DWUUrRmuJCpa2gCu6Uslq9fjKs66zja64r1lUnVNMxgPC6dlAxEdp766PVreM/21rTH1lZr57V+N19/vosPXNPBl+/YmvUF5HQ4Kqxctqomo4uqvaNe/vTxV7jp737Ovb88y8f+9SAvvDGUtzEmMz4VxBcIZVwGaWiqsjHmmY6l2pYKqXMXRWGsTk1UnXDH9lbWN72FLSlm7hDpDplJCwKj7UCyyo9MXLXawdPHL3JqIHJxNO20TPSc07mYajDe8NLVXmcnFNa868pVfO1923NyvqnsWVfPowd78QfDC16APT/q4Rs/6+bJo/2UmRQf2rOaD+5ZzZ987xU++dBh/unDV3PL5S15H6/BqHFPdwemuZqqbYQ1jHqmaa7O7jXyQYK7KApjH9FkOefLW2vSep222nJev5h+nrd7aJLr1jek/fiFGB0ifxGtQU8/LWPM3NMP7pl67852AD75tvUpV83mynXr67n/pR5e6XOxa239vPt9/hBfefo1njgSCeofuW4Nn3rbhtiM+eG79/CR+/bzqYcP808fuoa3by1MgL+U4Q5Mc8Vq3Sf8Syq4S1pGFMWYJ5LKqM+w5e5crbV2Rib9aVVpTEwFuOSeyqphWCJb22qwlpn4+alIKiHdapnt7Q62ttZwzZq6nIwjkdUNFXzmlk0LVhvl2u51kTfN/QlaEQRDYf7wO0d4/HAfH71+DS9+/ib+/M5tswJqbYWFh+7ew9bWGv7Ttw/zkxMDBRn3YJYLmAyxVapLrGJGgrsoCqfXT7nFlNGuN4kY5ZDpbCt3JtpiN1fB3WYuY2tbDT2jkR2S0p25r26o4NnPvoXmLIPJUlVfaWVLS/W8i6paa/78hyf46ckh/vzObXzlPduSnnut3cJDv7eHbW21/OdvH+HfX8t/gDe212vOsN2voXGJbpQtwV0UxZjHv+hZO2S2aYfRMGyxlTLxjE2zgYw3/liO9qyv5/B556yukv/8i7M8/OtePvW2DXz0+rUpX6Om3MKDd+/myo5a/uCRIzz3+mAeRxwJ7vWV1qw/5Rgz96XWgkCCuygKp8efVo13Kpls2tE1NIG1zMTq+vRLClMx8u5lJkXlIj+FLAfXrW/A6w/FFnf94Gg/f/Pvb3DXzjY+f9uWtF+nptzCgx/fzYamKv73T07la7hAJC2TbUoGoNJmpsJatuRm7nJBVRTFmNefcR+PRGZm7qmD+5mhSdY1VmJO0kslG1evjuTNa+2WvJYalord6yIXUn99dpQpf4g/efw4162v52/fn3nFTnW5hduuWMU3ftaF1x+kwpp9uPIHw3zhiVfwBUI4Kiw4Kqw47BYcFRa6hydj3T2z1bgE91KV4C6KYszjpzODRTnJ2K1l1FVYuJhGzr1raDJl7XymOursNFZZqUmxScdK0VhlY2NzFU8fu8g/vXCGdY2V/MtHdmWd8tjZWUtYw2v947E3jmwcPu/kyaP9tDvsTAfDuH1+AqGZuvS3L7L0cqG9VKcCIf7Tw4f5/beu54YNjYv6OZmQ4C6KYsyTm5k7RCpmLqWYuU8FQlwY88ZKBHNFKcWNW5oZX4LbrBXLdevrefjXvbTU2Lj/Y7vTvtCcyPYOBwDHL7gWFdwP9UQqeJ79zFuorbCgtcbrD+HyBXB7AxmtOUikqcrGmWhri7l+fGKAF04NMxUIS3AXy1sgFGZiKjhvdWq22hzl9DkXDu5nhz2Ede4qZeL97fu2IxmZGe+6opVfnB7m3o/sWnALwXQ0VtnoqLNzrM+1qNc5eN7JlpbqWLmqUopKm5lKmzmtrQtTjrPayv5ziWfu3z14AYCXz45yJgcpoHTJBVVRcM5YX5ncpDLS2W7PaBi22G6QiZhMSvLtcW7Y2MivPn9z2gvRUtnRGdmzNluhsObIeSe71uZvXUFTVTlOb2De3rPnRz28dGaUfdevwWxSfGd/b97GMJcEd1FwTmMBU2V2dcVztTrKGZ8K4pkOJn1M9+AEJpV+sy6xdOzoqKXP6cv6guUbA+NMTge5NsGq2VxprI58Ch2d00DsuwcvYFLwqRs3cNu2VTx+pI+pQGHaIktwFwVn9JWpy9HMPZ1NO7qHJ1nTsHB/eLE07Yjm3V/JMjVzMLpiNr8z9/kLmYKhMN873MdNW5pprbXzO3tW4/IGCrIwC9II7kqpTqXUC0qpk0qpE0qpz0aP1yulnlNKdUVv6+Ke80WlVLdS6pRS6rZ8noAoPc4k7X6z1VqbetOOrsHC5TpFbl3RXotJwbEse8UfPO+krbacjhxUZyUz04Jg5v/gC6eGGZ6Y5rev7QTg+vUNrG2o4JECpWbSmbkHgT/WWl8OXAd8Wim1FfgC8LzWehPwfPR7ovftBbYBtwP3KKVkuiRiYh0hc3ZBdeFa90AozLkRT17y7SL/Km1mNrdUZ5V311pzqGcsYSOzXIpvHmZ49EAvTdU2brqsGYhcm/ng7tUc6Bnj9OBEXscDaQR3rfUlrfWR6NcTwEmgHbgLeCD6sAeA90a/vgt4VGs9rbU+B3QDu3M8blHCjF7uuVquv6q2HKVIWut+ftRLMKwXvW+qKJ4dHQ6O97nQOrOe6X1OH4Pj01ybx5QMzG8eNuCe4oVTQ3zgmo5ZG5C//5oOrGWmgszeM8q5K6XWAlcB+4EWrfUliLwBAM3Rh7UDF+Ke1hc9Nve1PqGUOqSUOjQ8PJzF0EWpGvP6qbaZF+z5nQlLmYmmKlvSWvfuocgsSWbupWtHpwOXN8CFscw2Qz/YY+Tb8ztzL7eUUW0zx3Lujx++QFjDb+3qnPW4hiobt1+xiicLcGE17d8upVQV8ATwOa31+EIPTXBs3tut1vperfUurfWupqamdIchloFc9ZWJt7q+gpfOjCasqDAahknOvXTt6IysLM603v1gj5PqcnPKLRtzoSm6UXY4rPnuoQuRHHuC6qzf2bOa8akgz7xyKa/jSSu4K6UsRAL7t7XWT0YPDyqlWqP3twLG/lh9QPzbVQdwMTfDFcvBmDeQ8+D+hXdexqhnmo9+6wDuOatFu4YmaXfYqbTJmr1StbmlmnKLKeO8+8GeMXatqSvIhiWN0RYEL50Z5cKYj727OxM+bs+6ejY0VfLI/vN5HU861TIK+BZwUmv9f+LuehrYF/16H/BU3PG9SimbUmodsAk4kLshi1Ln9PipT3Nji3TtWlvPv3xkF11DE3z8/oN4/TM178bWeqJ0WcpMXNFWm1FwH/P46R6azHtKxtAUbR726MFeau0Wbtu2KuHjlIpcWD3S6+LkpYWSIIuTzsz9TcBHgJuVUseif94FfA24VSnVBdwa/R6t9QngMeB14N+BT2utC1O1L0rCWB7SMgBv29zE/9t7FUd7nXzyocNMB0OEw5ozwxLcl4MdnQ5eu+ietwo0mcPnnQB5XbwUr6naxkWXj5+cGOQ3rmqn3JK8SPD913RgNef3wmo61TIvaq2V1nq71npn9M+zWutRrfUtWutN0duxuOd8VWu9QWu9RWv9o7yNXpQkpzc3G3Uk8s4rW/mb923nV10jfOY7Rzk/5mUqEM7pBh2iOLZ31DIVCKddRnioZwxrmYntHbntBJpMU7WNqUAYfyicNCVjcFRYuePKVn5wtH/Wp8xckhWqoqCmAiG8/lBeZu6GD+zq5Cvv2cqPTwzyqYcOA/lpGCYKy9gY5Xiai5kO9oxxZUftgjPoXGqsivyf3tnp4LJVqfvqfGjPaiamg/zweH4uSUpwFwUVW8CUx+AO8LE3reOPbt3MqegsT4J76VtdX4GjwpJW3n0qENkNqlApGSC2L+zeaxeetRuuWVPH5pYqnj85lPrBWZDyAVFQsb4yBdhv9A9v3og/GOboBafsb7oMKKVii5lSOXbBRSCk8754Kd6bNjTytd+8kt+8uiOtxyuluP9ju2nJ00bpEtxFQeW6r8xClFL81wz27RRL345OR1rb7hmbc1yzpnDB3Wo2sXf36oyes9h+9wuRtIwoqJm0jGxLJzIXv+3eQg72ONncUrWiP7FJcBcF5SxgWkYsP/Hb7iVjbM5RyHz7UiTBXRTUmDeAUrlrGiZWlnS23XtjYJyJPG/OUQokuIuCcnr8OOyWgiwHF8tTqm33DvVEFi/lc3OOUiDBXRTUmDc/q1PFyrGzw7HgtnsHe8ZorS3PycbXpUyCuyioSF8ZCe4ie8aK00Tb7mmtI83C1tav+E3LJbiLgspXXxmxciTbdk9rzfE+N4Pj0+xe4SkZkDp3UWBOrz+24bEQ2TC23TtwbpSfnxri2AUXxy+4ON7njpXaXr+hocijLD4J7qJgtNY4Pbnv5S5Wnh0dDr576AK/PjuGUrC5uZq3X97Mjk4Hu9fWs7E5/5tzLHUS3EXBePwh/KGwLGASi/bpmzayeVU1W1truLKjlirZiGUe+RsRBTM2KQuYRG6sbqjg7jevK/YwljS5oCoKZqyAfWWEWOkkuIuCibUekOAuRN5JcBcFE2saJmkZIfJOgrsoGKPdr8zchcg/Ce6iYMY8fspMippyuY4vRL5JcBcF4/T6qauwrvhl4UIUggR3UTBjHr/UuAtRIBLcRcE4PQEpgxSiQCS4i4IZ8/oluAtRIBLcRcE4PX5ZnSpEgUhwFwURDmucMnMXomAkuIuCGJ8KENbSV0aIQpHgLgoitjpVZu5CFIQEd1EQsjpViMKS4C4KYnRS+soIUUgS3EVBzMzcZRGTEIUgwV0UxJgnAEjOXYhCkeAuCsLp9WMzm7Bbyoo9FCFWBAnuoiAifWWkaZgQhSLBXRSErE4VorBSBnel1H1KqSGl1Gtxx+qVUs8ppbqit3Vx931RKdWtlDqllLotXwMXpUX6yghRWOnM3O8Hbp9z7AvA81rrTcDz0e9RSm0F9gLbos+5RyklSVYRmblLcBeiYFIGd631L4GxOYfvAh6Ifv0A8N64449qrae11ueAbmB3boYqStmYx099hZRBClEo2ebcW7TWlwCit83R4+3AhbjH9UWPzaOU+oRS6pBS6tDw8HCWwxClIBAKMz4VlJm7EAWU6wuqiUohdKIHaq3v1Vrv0lrvampqyvEwxFLi8kZq3BskuAtRMNkG90GlVCtA9HYoerwP6Ix7XAdwMfvhieVA+soIUXjZBvengX3Rr/cBT8Ud36uUsiml1gGbgAOLG6IodbGOkFIKKUTBmFM9QCn1HeBGoFEp1Qd8Bfga8JhS6m6gF/gAgNb6hFLqMeB1IAh8WmsdytPYRYlwemTmLkShpQzuWusPJrnrliSP/yrw1cUMSiwvo9LLXYiCkxWqIu+MmbtDSiGFKBgJ7iLvxrx+qmxmbGZZzyZEoUhwF3l35LyT1fUVxR6GECuKBHeRV6/1uzne5+b913QUeyhCrCgS3EVePXKgF5vZxPuuluAuRCFJcBd5Mzkd5Kmj/bx7exu1cjFViIKS4C7y5qlj/Xj8IX5nz+piD0WIFUeCu8gLrTWP7O/lslXVXL3aUezhCLHiSHAXefFKn5sTF8f50J7VsrWeEEUgwV3kxSP7e7FbyrjrqoQdn4UQeZay/YBYmbTW7D83hi8wvzWQ2aS4dm095ZbEi5LGpwI8ffwid+5oo6ZcLqQKUQwS3EVCPz05xO8/eCjp/bvW1HH/x3dTZZv/X+gHR/vxBUJ86Dq5kCpEsUhwFwl979AFGqts3PvRa+btwHJqYIIv/eA19t13gPs/di3VcbNz40LqFe01bO9wFHTMQogZEtzFPGMePy+cGmLf9Wu5enXdvPuvWl1Hrd3CH37nKB+97wAPfHx3LP1ypNfFGwMT/NVvXFnoYQsh4sgFVTHP08f6CYQ071ugZcA7r2zlG79zNa/2ufnotw4wPhXZSu+R/b1UWsu4c2dboYYrhEhAgruY54kj/WxtreHy1poFH3f7Fau450NXc+Kim4986wAXxrw888pF3ntVe8JcvBCicCS4i1lOD07war97wVl7vHdsW8U9H7qG1y+6edf/+xXTwbCsSBViCZDgLmZ54nAfZpPirgzSKrdubeGfP3wN04EwOzsdbGurzeMIhRDpkM/OIiYYCvP9o/3cuKWZxipbRs+95fIW/u0zb6aqXP5LCbEUyG+iiHmxe4ShiWnef012q0o3tVTneERCiGxJWkbEPHGkH0eFhZsuay72UIQQiyTBXQDg9gX48YkB7tzRJnudCrEMSHAXAPzbK5fwB8OyY5IQy4QEdwHAE0f62NhcxfYOqXQRYjmQ4C44N+Lh8Hkn77u6Q3qvC7FMSHAXPHmkD5OC35De60IsGxLcV7hwWPPkkX7etLGRVbXlxR6OECJHJLivYOGw5vHDffS7fLw/zXYDQojSUNKLmJweP5959Cjv3t7K7Ve0UmtPvetP76iXc6MeXF4/Lm8g8sfnx+0NUFVuZu+1q9natnDDrFI3PhXge4f6eOjlHnpGvaxvrOQdW1cVe1hCiBwq6eB+wenlwpiXP33iVb781Alu3tLMXTvbuOmy5tgWcD5/iF+fG+UXp4b5xelhzo145r1Olc1Mrd3CqGeaB18+z+619ey7YS3v2NaCpax4H268/iDHL7i5Zk0dVvPix9E1OMH9L/Xw/aP9eP0hrllTxx+9Ywu3b1uVk9cXQiwdSmtd7DGwa9cufehQ8i3dFqK15nifm6eO9fPD45cYmZymutzMrVtbGJn0s//sKNPBMDazies3NHDj5iauaK/FUWHFUWGh1m6JBXCX189jhy7w4Mvn6XP6WFVTzoevW83e3asz7rUSLxTWnBme5NgFFzXlZm65fOE3jXBY8/2j/fztj99gcHyadY2VfPGdl3Hr1paU1SzBUJiLrinOjXo4P+qhZ8RLz6iHnhEPZ0c8WM0m7tzRxu/esJYr2qXsUYhSppQ6rLXelfC+Ug/u8YKhMC+fHeUHRy/ykxMDNNXYuHFzM2/b0sSedck3dJ4rFNb87I0hHniphxe7RwBorS1nTUMF6xorWdNQydqGCjrrKxKu5tQ6EsyPXnBx/IKLV/vcePwzG0231Nj40J41fHD3apqqZ79pHOwZ4y+feZ1X+tzs6Kjlt67t5L4Xz3Fm2MMNGxr4b3dsnZc2Coc1B3rGeOrYRZ599RJuXyB2n91SFhv3zk4HH9jVSX2lNe2/UyHE0rVigns+dA9N8KNXB6IzYS89Ix5GPf60nmspU2xtrWFHp4OdnQ62dzjoHfNw/0vn+eXpYaxlJu7Y3spHr19DY5WNv/7RSZ59dYBVNeX86Tu3cNeOdkwmRSAU5pH9vfzfn57G7Qvw27s6+aN3bGZkws9Tx/v54bGLXHRPUWEt4x1bW7hhQyNrGyNvQE3VNqldF2KZkuCeY+NTAXpHI/n+QDjx319nnZ2tbTVJ+7ScGZ7koZfP8/jhPiang5gU2MxlfOptG/jEW9djt85/ntsb4OvPd/Hgyz0ABMMas0nx1s1N3LWzjVu3tlBhLenLKEKIDEhwX8Imp4M8eSRSjvixG9alVWt+dniSB18+z4amSt51ZSsNi7geIIQoXUUJ7kqp24GvA2XAN7XWX0v22JUc3IUQIlsLBfe81L8ppcqAfwTeCWwFPqiU2pqPnyWEEGK+fBU37wa6tdZntdZ+4FHgrjz9LCGEEHPkK7i3Axfivu+LHotRSn1CKXVIKXVoeHg4T8MQQoiVKV/BPVHt3azkvtb6Xq31Lq31rqampjwNQwghVqZ8Bfc+oDPu+w7gYp5+lhBCiDnyFdwPApuUUuuUUlZgL/B0nn6WEEKIOfKy4kVrHVRK/QHwYyKlkPdprU/k42cJIYSYL2/LGbXWzwLP5uv1hRBCJLckVqgqpYaB8xk8pREYydNwlrqVeu5y3iuLnHd61mitE1akLIngniml1KFkq7KWu5V67nLeK4uc9+LJDg1CCLEMSXAXQohlqFSD+73FHkARrdRzl/NeWeS8F6kkc+5CCCEWVqozdyGEEAuQ4C6EEMtQyQV3pdTtSqlTSqlupdQXij2efFFKdSqlXlBKnVRKnVBKfTZ6vF4p9ZxSqit6W1fsseaDUqpMKXVUKfVM9Ptlf95KKYdS6nGl1BvRf/frV8h5/5fo//HXlFLfUUqVL9fzVkrdp5QaUkq9Fncs6bkqpb4YjXWnlFK3ZfKzSiq4r7BNQILAH2utLweuAz4dPdcvAM9rrTcBz0e/X44+C5yM+34lnPfXgX/XWl8G7CBy/sv6vJVS7cBngF1a6yuItCvZy/I97/uB2+ccS3iu0d/3vcC26HPuicbAtJRUcGcFbQKitb6ktT4S/XqCyC96O5HzfSD6sAeA9xZlgHmklOoA7gC+GXd4WZ+3UqoGeCvwLQCttV9r7WKZn3eUGbArpcxABZEOssvyvLXWvwTG5hxOdq53AY9qrae11ueAbiIxMC2lFtxTbgKyHCml1gJXAfuBFq31JYi8AQDNRRxavvw98HkgHHdsuZ/3emAY+NdoOuqbSqlKlvl5a637gb8DeoFLgFtr/ROW+XnPkexcFxXvSi24p9wEZLlRSlUBTwCf01qPF3s8+aaUejcwpLU+XOyxFJgZuBr4J631VYCH5ZOKSCqaX74LWAe0AZVKqQ8Xd1RLxqLiXakF9xW1CYhSykIksH9ba/1k9PCgUqo1en8rMFSs8eXJm4A7lVI9RNJuNyulHmb5n3cf0Ke13h/9/nEiwX65n/fbgXNa62GtdQB4EriB5X/e8ZKd66LiXakF9xWzCYhSShHJv57UWv+fuLueBvZFv94HPFXoseWT1vqLWusOrfVaIv++P9Naf5jlf94DwAWl1JbooVuA11nm500kHXOdUqoi+n/+FiLXl5b7ecdLdq5PA3uVUjal1DpgE3Ag7VfVWpfUH+BdwGngDPClYo8nj+f5ZiIfwV4BjkX/vAtoIHJFvSt6W1/ssebx7+BG4Jno18v+vIGdwKHov/kPgLoVct7/A3gDeA14CLAt1/MGvkPk2kKAyMz87oXOFfhSNNadAt6Zyc+S9gNCCLEMlVpaRgghRBokuAshxDIkwV0IIZYhCe5CCLEMSXAXQohlSIK7EEIsQxLchRBiGfr/GapyeY885eAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import random\n",
    "# Keras for building the model\n",
    "from keras import Sequential\n",
    "# use the deque to save the results after every episode\n",
    "from collections import deque\n",
    "from keras.layers import Dense\n",
    "#optimizer \n",
    "from tensorflow.keras.optimizers import Adam\n",
    "# for the graph\n",
    "import matplotlib.pyplot as plt\n",
    "# Activations layers\n",
    "from keras.activations import relu, linear\n",
    "\n",
    "import numpy as np\n",
    "env.seed(110)\n",
    "np.random.seed(10)\n",
    "\n",
    "\n",
    "class DQN:\n",
    "\n",
    "    \"\"\" Implementation of deep q learning algorithm \"\"\"\n",
    "    # intialize the hyperparameters\n",
    "    def __init__(self, action_space, state_space):\n",
    "\n",
    "        self.action_space = action_space\n",
    "        self.state_space = state_space\n",
    "        self.epsilon = 1\n",
    "        self.gamma = .95\n",
    "        self.batch_size = 64\n",
    "        self.epsilon_min = .01\n",
    "        self.epsilon_decay = .995\n",
    "        self.learning_rate = 0.001\n",
    "        self.memory = deque(maxlen=10000)\n",
    "        self.model = self.build_model()\n",
    "    \n",
    "    # building the model with the 3 layers(2 activations layers and the output layer)\n",
    "    def build_model(self):\n",
    "    \n",
    "        model = Sequential()\n",
    "        model.add(Dense(24, input_shape=(self.state_space,), activation='relu'))\n",
    "        model.add(Dense(24, activation='relu'))\n",
    "        model.add(Dense(self.action_space, activation='linear'))\n",
    "        model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))\n",
    "        return model\n",
    "    # To save the actual_state, action , reward , next state in a deque for the replay buffer \n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def act(self, state):\n",
    "\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.action_space)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])\n",
    "\n",
    "    def replay(self):\n",
    "\n",
    "        if len(self.memory) < self.batch_size:\n",
    "            return\n",
    "\n",
    "        minibatch = random.sample(self.memory, self.batch_size)\n",
    "        states = np.array([i[0] for i in minibatch])\n",
    "        actions = np.array([i[1] for i in minibatch])\n",
    "        rewards = np.array([i[2] for i in minibatch])\n",
    "        next_states = np.array([i[3] for i in minibatch])\n",
    "        dones = np.array([i[4] for i in minibatch])\n",
    "\n",
    "        states = np.squeeze(states)\n",
    "        next_states = np.squeeze(next_states)\n",
    "\n",
    "        targets = rewards + self.gamma*(np.amax(self.model.predict_on_batch(next_states), axis=1))*(1-dones)\n",
    "        targets_full = self.model.predict_on_batch(states)\n",
    "\n",
    "        ind = np.array([i for i in range(self.batch_size)])\n",
    "        targets_full[[ind], [actions]] = targets\n",
    "\n",
    "        self.model.fit(states, targets_full, epochs=1, verbose=0)\n",
    "        if self.epsilon > self.epsilon_min:\n",
    "            self.epsilon *= self.epsilon_decay\n",
    "\n",
    "\n",
    "def train_dqn(episode):\n",
    "\n",
    "    loss = []\n",
    "    agent = DQN(env.action_space.n, env.observation_space.shape[0])\n",
    "    for e in range(episode):\n",
    "        state = env.reset()\n",
    "        state = np.reshape(state, (1, 4))\n",
    "        score = 0\n",
    "        max_steps = 1000\n",
    "        for i in range(max_steps):\n",
    "            env.render()\n",
    "            action = agent.act(state)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            score += reward\n",
    "            next_state = np.reshape(next_state, (1, 4))\n",
    "            agent.remember(state, action, reward, next_state, done)\n",
    "            state = next_state\n",
    "            agent.replay()\n",
    "            if done:\n",
    "                print(\"episode: {}/{}, score: {}\".format(e, episode, score))\n",
    "                break\n",
    "        loss.append(score)\n",
    "    return loss\n",
    "\n",
    "\n",
    "def random_policy(episode, step):\n",
    "\n",
    "    for i_episode in range(episode):\n",
    "        env.reset()\n",
    "        for t in range(step):\n",
    "            env.render()\n",
    "            action = env.action_space.sample()\n",
    "            state, reward, done, info = env.step(action)\n",
    "            if done:\n",
    "                print(\"Episode finished after {} timesteps\".format(t+1))\n",
    "                break\n",
    "            print(\"Starting next episode\")\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\n",
    "    ep = 100\n",
    "    loss = train_dqn(ep)\n",
    "    plt.plot([i+1 for i in range(0, ep, 2)], loss[::2])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c74815c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}