notes/04_deepq_learning/cartpole_dqn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Q Leaning\n",
    "\n",
    "In this notebook, we will implement dqn algorithm to play Cartpole.\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple, deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Create our environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\deepanshu\\.conda\\envs\\ai\\lib\\site-packages\\gym\\logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "env = gym.make('CartPole-v1').unwrapped\n",
    "env.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Viewing our Enviroment\n",
    "Execute the code cell below to play Cartpole with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The size of state is:  (4,)\n",
      "No. of Actions:  2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEICAYAAAB/Dx7IAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVEklEQVR4nO3df5Bd5X3f8ffHQgbH4CCihRGSiFQs2ghPI+Id1Q1pS4NjCK0rnKkzchtHmSEjpoXUbt3WyJ460KmmTsbY/WkPcmCiCbYV+QdF8TixMbXHZYYgrzC/BMgoRoAsFQkIARJXscS3f9wjcy1W2ru/WD2779fMnXvuc55z7verkT57ePZcbqoKSVI7XjfTBUiSxsfglqTGGNyS1BiDW5IaY3BLUmMMbklqjMGtk1aSDyX5vameO8C5Ksmbp+Jc0nSI93HrtZDkN4APAOcDLwC3ARuq6vmZrGs0SQpYUVW7R9n3TeBtwOG+4V+qqrtfo/Ikr7g1/ZJ8APgd4N8BP0kv+H4auCPJ649zzCmvXYXjdm1Vnd73eFVon+T1q3EGt6ZVkjcBNwC/VVV/UlU/rKo9wK/SC+9f6+Zdn+QLSW5N8gLwG93YrX3n+vUkTyR5Nsl/SLInydv7jr+1217WLXesS/JkkmeSfLjvPKuT3J3k+ST7k/yP4/0AGWevleSaJI8Bj3Vj/zXJU0leSLIjyd/rm399ks93Pb+Y5MEkFyTZkORAd9w7+ub/ZJKbu5q/n+Q/JZk32brVHoNb0+3ngdOAL/UPVtVLwB8Dv9Q3vAb4AnAm8Jn++UlWAp8E/jmwiN6V++Ix3vsXgL8JXAp8JMnPdONHgH8NLAT+brf/X46zr+O5Evg7wMru9beBVcBZwGeBzyc5rW/+O4E/ABYA3wG+Su/f5WLgPwI39c3dTG+J5s3ARcA7gN+corrVEINb020h8ExVHR5l3/5u/1F3V9X/qqqXq+oHx8z9p8AfVdVdVfXXwEeAsX5Bc0NV/aCq7gfuB34WoKp2VNWfVtXh7ur/JuAfjKOn/9ZdrT+f5N5j9v3nqnruaP1VdWtVPdu9143AqfR+mBz1f6rqq92fz+eBIeCjVfVDYAuwLMmZSc4Bfhl4f1X9ZVUdAD4BrB1H3ZolXIfTdHsGWJjklFHCe1G3/6inTnCec/v3V9VfJXl2jPf+v33bfwWcDpDkAuDjwDDwE/T+HewY41z9/lVVHe8Olh/roVvf/82u/gLexI//sHq6b/sH9H7IHel7TVf3ucB8YH+So/Nfd+z7aW7wilvT7W7gEPAr/YNJ3kjvCvLOvuETXUHvB5b0Hf8G4KcmWNOngEfp3TnyJuBDQE58yMB+1EO3nv1Beuv5C6rqTOAvJvheT9H7c1xYVWd2jzdV1YVTUbTaYnBrWlXVX9D75eR/T3J5kvlJltFbFthLb313EF8A3pnk57tfJN7AxMP2DHq3JL6U5G8B/2KC5xnkfQ4DB4FTknyE3hX3uFXVfuBrwI1J3pTkdUnOTzKeJR7NEga3pl1V/S69q9qP0QvMe+hdQV5aVYcGPMdO4LforfvuB14EDtC7Ch2vfwv8s+4cnwb+cALnGMRX6f0C9rvAE8D/Y3JLG78OvB54GPhzej/MFk2yRjXID+CoSUlOB56nt9zx+EzXI72WvOJWM5K8M8lPdOvjHwMeBPbMbFXSa2/agrtbz9yVZHeS66brfTSnrAH2dY8VwNryPxk1B03LUkn3aa7v0vtwxV56H0J4T1U9POVvJklzzHRdca8GdlfV97oPS2yhd7UkSZqk6foAzmJ+/Lfne+l9DHhUCxcurGXLlk1TKZLUnj179vDMM8+MesvrdAX3aG/2Y2sySdYD6wHOO+88RkZGpqkUSWrP8PDwcfdN11LJXmBp3+sl9H6h9CNVtamqhqtqeGhoaJrKkKTZZ7qC+9vAiiTLu0+5rQW2TdN7SdKcMi1LJVV1OMm19D45Ng+4pfvkmyRpkqbt/w5YVV8BvjJd55ekucpPTkpSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNmdR3TibZA7wIHAEOV9VwkrOAPwSWAXuAX62qP59cmZKko6biivsfVtWqqhruXl8H3FlVK4A7u9eSpCkyHUsla4DN3fZm4MppeA9JmrMmG9wFfC3JjiTru7Fzqmo/QPd89mgHJlmfZCTJyMGDBydZhiTNHZNa4wYurqp9Sc4G7kjy6KAHVtUmYBPA8PBwTbIOSZozJnXFXVX7uucDwG3AauDpJIsAuucDky1SkvSKCQd3kjcmOePoNvAO4CFgG7Cum7YOuH2yRUqSXjGZpZJzgNuSHD3PZ6vqT5J8G9ia5CrgSeDdky9TknTUhIO7qr4H/Owo488Cl06mKEnS8fnJSUlqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjRkzuJPckuRAkof6xs5KckeSx7rnBX37NiTZnWRXksumq3BJmqsGueL+feDyY8auA+6sqhXAnd1rkqwE1gIXdsd8Msm8KatWkjR2cFfVt4DnjhleA2zutjcDV/aNb6mqQ1X1OLAbWD1FtUqSmPga9zlVtR+gez67G18MPNU3b2839ipJ1icZSTJy8ODBCZYhSXPPVP9yMqOM1WgTq2pTVQ1X1fDQ0NAUlyFJs9dEg/vpJIsAuucD3fheYGnfvCXAvomXJ0k61kSDexuwrtteB9zeN742yalJlgMrgO2TK1GS1O+UsSYk+RxwCbAwyV7gt4GPAluTXAU8CbwboKp2JtkKPAwcBq6pqiPTVLskzUljBndVvec4uy49zvyNwMbJFCVJOj4/OSlJjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTFjBneSW5IcSPJQ39j1Sb6f5L7ucUXfvg1JdifZleSy6SpckuaqQa64fx+4fJTxT1TVqu7xFYAkK4G1wIXdMZ9MMm+qipUkDRDcVfUt4LkBz7cG2FJVh6rqcWA3sHoS9UmSjjGZNe5rkzzQLaUs6MYWA0/1zdnbjb1KkvVJRpKMHDx4cBJlSNLcMtHg/hRwPrAK2A/c2I1nlLk12gmqalNVDVfV8NDQ0ATLkKS5Z0LBXVVPV9WRqnoZ+DSvLIfsBZb2TV0C7JtciZKkfhMK7iSL+l6+Czh6x8k2YG2SU5MsB1YA2ydXoiSp3yljTUjyOeASYGGSvcBvA5ckWUVvGWQPcDVAVe1MshV4GDgMXFNVR6andEmam8YM7qp6zyjDN59g/kZg42SKkiQdn5+clKTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0Z83ZAaS7YsenqV429df1NM1CJNDavuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS8cx2p0m0snA4JakxhjcktQYg1uSGmNwS1JjDG5JasyYwZ1kaZJvJHkkyc4k7+vGz0pyR5LHuucFfcdsSLI7ya4kl01nA5I01wxyxX0Y+EBV/QzwNuCaJCuB64A7q2oFcGf3mm7fWuBC4HLgk0nmTUfxkjQXjRncVbW/qu7ttl8EHgEWA2uAzd20zcCV3fYaYEtVHaqqx4HdwOqpLlyS5qpxrXEnWQZcBNwDnFNV+6EX7sDZ3bTFwFN9h+3txo491/okI0lGDh48OP7KJWmOGji4k5wOfBF4f1W9cKKpo4zVqwaqNlXVcFUNDw0NDVqGJM15AwV3kvn0QvszVfWlbvjpJIu6/YuAA934XmBp3+FLgH1TU64kaZC7SgLcDDxSVR/v27UNWNdtrwNu7xtfm+TUJMuBFcD2qStZkua2Qb667GLgvcCDSe7rxj4EfBTYmuQq4Eng3QBVtTPJVuBhenekXFNVR6a8ckmao8YM7qq6i9HXrQEuPc4xG4GNk6hLknQcfnJSkhpjcEtSYwxuSWqMwS0Bb11/00yXIA3M4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjRnky4KXJvlGkkeS7Ezyvm78+iTfT3Jf97ii75gNSXYn2ZXksulsQJLmmkG+LPgw8IGqujfJGcCOJHd0+z5RVR/rn5xkJbAWuBA4F/h6kgv8wmBJmhpjXnFX1f6qurfbfhF4BFh8gkPWAFuq6lBVPQ7sBlZPRbGSpHGucSdZBlwE3NMNXZvkgSS3JFnQjS0Gnuo7bC8nDnrppLVj09UzXYL0KgMHd5LTgS8C76+qF4BPAecDq4D9wI1Hp45yeI1yvvVJRpKMHDx4cNyFS9JcNVBwJ5lPL7Q/U1VfAqiqp6vqSFW9DHyaV5ZD9gJL+w5fAuw79pxVtamqhqtqeGhoaDI9SNKcMshdJQFuBh6pqo/3jS/qm/Yu4KFuexuwNsmpSZYDK4DtU1eyJM1tg9xVcjHwXuDBJPd1Yx8C3pNkFb1lkD3A1QBVtTPJVuBhenekXOMdJZI0dcYM7qq6i9HXrb9ygmM2AhsnUZck6Tj85KQkNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1vqvHX9TTNdgjQQg1uSGmNwa9ZLMvBjOo6XpprBLUmNGeSLFKQ55Y/2rf/R9jvP3TSDlUij84pb6tMf2qO9lk4GBrckNcbglqTGDPIt76cl2Z7k/iQ7k9zQjZ+V5I4kj3XPC/qO2ZBkd5JdSS6bzgakqXTsmrZr3DoZpapOPKF3j9Mbq+qlJPOBu4D3Ab8CPFdVH01yHbCgqj6YZCXwOWA1cC7wdeCCE33T+/DwcI2MjExNR9IxXsvb9Mb69yQNanh4mJGRkVH/8g7yLe8FvNS9nN89ClgDXNKNbwa+CXywG99SVYeAx5Psphfidx/vPXbs2OE9sJoV/Hus18JAtwMmmQfsAN4M/M+quifJOVW1H6Cq9ic5u5u+GPjTvsP3dmPHnnM9sB7gvPPO44knnph4F9IJeMWtFg0PDx9330C/nKyqI1W1ClgCrE7ylhNMH+1fyav+NlfVpqoarqrhoaGhQcqQJDHOu0qq6nl6SyKXA08nWQTQPR/opu0FlvYdtgTYN+lKJUnAYHeVDCU5s9t+A/B24FFgG7Cum7YOuL3b3gasTXJqkuXACmD7VBcuSXPVIGvci4DN3Tr364CtVfXlJHcDW5NcBTwJvBugqnYm2Qo8DBwGrjnRHSWSpPEZ5K6SB4CLRhl/Frj0OMdsBDZOujpJ0qv4yUlJaozBLUmN8X/rqlnPe6s123jFLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMG+bLg05JsT3J/kp1JbujGr0/y/ST3dY8r+o7ZkGR3kl1JLpvOBiRprhnkixQOAb9YVS8lmQ/cleSPu32fqKqP9U9OshJYC1wInAt8PckFfmGwJE2NMa+4q+el7uX87nGirxRZA2ypqkNV9TiwG1g96UolScCAa9xJ5iW5DzgA3FFV93S7rk3yQJJbkizoxhYDT/UdvrcbO/ac65OMJBk5ePDgJFqQpLlloOCuqiNVtQpYAqxO8hbgU8D5wCpgP3BjNz2jnWKUc26qquGqGh4aGppQ8ZI0F43rrpKqeh74JnB5VT3dBfrLwKd5ZTlkL7C077AlwL4pqFWSxGB3lQwlObPbfgPwduDRJIv6pr0LeKjb3gasTXJqkuXACmD71JYtSXPXIHeVLAI2J5lHL+i3VtWXk/xBklX0lkH2AFcDVNXOJFuBh4HDwDXeUSJJU2fM4K6qB4CLRhl/7wmO2QhsnFxpkqTR+MlJSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMakqma6BpIcBP4SeGama5kGC7Gv1szW3uyrLT9dVUOj7TgpghsgyUhVDc90HVPNvtozW3uzr9nDpRJJaozBLUmNOZmCe9NMFzBN7Ks9s7U3+5olTpo1bknSYE6mK25J0gAMbklqzIwHd5LLk+xKsjvJdTNdz3gluSXJgSQP9Y2dleSOJI91zwv69m3oet2V5LKZqXpsSZYm+UaSR5LsTPK+brzp3pKclmR7kvu7vm7oxpvu66gk85J8J8mXu9ezpa89SR5Mcl+SkW5sVvQ2IVU1Yw9gHvBnwN8AXg/cD6ycyZom0MPfB34OeKhv7HeB67rt64Df6bZXdj2eCizvep830z0cp69FwM9122cA3+3qb7o3IMDp3fZ84B7gba331dffvwE+C3x5tvxd7OrdAyw8ZmxW9DaRx0xfca8GdlfV96rqr4EtwJoZrmlcqupbwHPHDK8BNnfbm4Er+8a3VNWhqnoc2E3vz+CkU1X7q+rebvtF4BFgMY33Vj0vdS/nd4+i8b4AkiwB/hHwe33Dzfd1ArO5txOa6eBeDDzV93pvN9a6c6pqP/QCEDi7G2+y3yTLgIvoXZ0231u3nHAfcAC4o6pmRV/AfwH+PfBy39hs6At6P1y/lmRHkvXd2GzpbdxOmeH3zyhjs/n+xOb6TXI68EXg/VX1QjJaC72po4ydlL1V1RFgVZIzgduSvOUE05voK8k/Bg5U1Y4klwxyyChjJ11ffS6uqn1JzgbuSPLoCea21tu4zfQV915gad/rJcC+GaplKj2dZBFA93ygG2+q3yTz6YX2Z6rqS93wrOgNoKqeB74JXE77fV0M/JMke+gtOf5ikltpvy8Aqmpf93wAuI3e0ses6G0iZjq4vw2sSLI8yeuBtcC2Ga5pKmwD1nXb64Db+8bXJjk1yXJgBbB9BuobU3qX1jcDj1TVx/t2Nd1bkqHuSpskbwDeDjxK431V1YaqWlJVy+j9O/rfVfVrNN4XQJI3Jjnj6DbwDuAhZkFvEzbTvx0FrqB3x8KfAR+e6XomUP/ngP3AD+n9pL8K+CngTuCx7vmsvvkf7nrdBfzyTNd/gr5+gd5/Xj4A3Nc9rmi9N+BvA9/p+noI+Eg33nRfx/R4Ca/cVdJ8X/TuOru/e+w8mhOzobeJPvzIuyQ1ZqaXSiRJ42RwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMb8f4iVaWTpYabeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"The size of state is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.render(mode='rgb_array'))\n",
    "env.close()\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render(mode='rgb_array')\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        time.sleep(0.05)\n",
    "        if done:\n",
    "            env.close()\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Creating Replay Memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Transition = namedtuple('Transition',\n",
    "                        ('state', 'action', 'reward', 'next_state'))\n",
    "\n",
    "\n",
    "class ReplayMemory(object):\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.position = 0\n",
    "\n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition.\"\"\"\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)\n",
    "        self.memory[self.position] = Transition(*args)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        experiences = random.sample(self.memory, batch_size)\n",
    "        \n",
    "        states = torch.FloatTensor(np.array([e.state for e in experiences])).float().to(device)\n",
    "        actions = torch.from_numpy(np.array([e.action for e in experiences])).long().to(device)\n",
    "        rewards = torch.from_numpy(np.array([e.reward for e in experiences])).float().to(device)\n",
    "        next_states = torch.FloatTensor(np.array([e.next_state for e in experiences])).float().to(device)\n",
    "        \n",
    "        return (states, actions, rewards, next_states)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Creating Deep Learning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, state_size = 4, action_size=2):\n",
    "        super(DQN, self).__init__()\n",
    "        self.fc1 = nn.Linear(state_size, 32)\n",
    "        self.fc2 = nn.Linear(32, 64)\n",
    "        self.fc3 = nn.Linear(64, action_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return self.fc3(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Creating out agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = int(1e5)  # replay buffer size\n",
    "BATCH_SIZE = 64         # minibatch size\n",
    "GAMMA = 0.99            # discount factor\n",
    "TAU = 1e-3              # for soft update of target parameters\n",
    "LR = 0.001              # learning rate \n",
    "UPDATE_EVERY = 5        # how often to update the network\n",
    "class Agent():\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.action_size = action_size\n",
    "        self.state_size = state_size\n",
    "        \n",
    "        # Q-Network\n",
    "        self.policy_net = DQN(state_size, action_size).to(device)\n",
    "        self.target_net = DQN(state_size, action_size).to(device)\n",
    "        self.optimizer = optim.Adam(self.policy_net.parameters(), lr=LR)\n",
    "        \n",
    "        # Replay memory\n",
    "        self.memory = ReplayMemory(BUFFER_SIZE)\n",
    "        \n",
    "        self.t_step = 0\n",
    "    \n",
    "    def step(self, state, action, reward, next_state):\n",
    "        # Save experience in replay memory\n",
    "        self.memory.push(state, action, reward, next_state)\n",
    "        \n",
    "        # Learn every UPDATE_EVERY time steps.\n",
    "        self.t_step = (self.t_step + 1) % UPDATE_EVERY\n",
    "        if self.t_step == 0:\n",
    "            # If enough samples are available in memory, get random subset and learn\n",
    "            if len(self.memory) > BATCH_SIZE:\n",
    "                experiences = self.memory.sample(BATCH_SIZE)\n",
    "                self.learn(experiences, GAMMA)\n",
    "                \n",
    "    def act(self, state, eps=0.):\n",
    "        \"\"\"Returns actions for given state as per current policy.\"\"\"\n",
    "        \n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        self.policy_net.eval()\n",
    "        with torch.no_grad():\n",
    "            action_values = self.policy_net(state)\n",
    "        self.policy_net.train()\n",
    "        \n",
    "        # Epsilon-greedy action selection\n",
    "        if random.random() > eps:\n",
    "            return np.argmax(action_values.cpu().data.numpy())\n",
    "        else:\n",
    "            return random.choice(np.arange(self.action_size))\n",
    "        \n",
    "    def learn(self, experiences, gamma):\n",
    "        states, actions, rewards, next_states = experiences\n",
    "        # Get max predicted Q values (for next states) from target model\n",
    "        Q_targets_next = self.target_net(next_states).detach().max(1)[0].unsqueeze(1)\n",
    "        \n",
    "        # Compute Q targets for current states \n",
    "        Q_targets = rewards + (gamma * Q_targets_next)\n",
    "\n",
    "        # Get expected Q values from local model\n",
    "        Q_expected = self.policy_net(states).gather(1, actions.unsqueeze(1))\n",
    "        \n",
    "        # Compute loss\n",
    "        loss = F.mse_loss(Q_expected, Q_targets)\n",
    "        # Minimize the loss\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "\n",
    "        # ------------------- update target network ------------------- #\n",
    "        self.soft_update(self.policy_net, self.target_net, TAU)\n",
    "        \n",
    "    def soft_update(self, local_model, target_model, tau):\n",
    "        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):\n",
    "            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = Agent(state_size=4, action_size= env.action_space.n)\n",
    "\n",
    "# watch an untrained agent\n",
    "state = env.reset()\n",
    "for j in range(200):\n",
    "    action = agent.act(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    time.sleep(0.05)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8: Loading Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=100)\n",
    "\n",
    "# To Load checkpoint uncomment code\n",
    "# checkpoint = torch.load('cartpole_2000.pth')\n",
    "# agent.policy_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.target_net.load_state_dict(checkpoint['state_dict'])\n",
    "# agent.optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "# start_epoch = checkpoint['epoch']\n",
    "# scores = checkpoint['scores']\n",
    "\n",
    "# index = 1\n",
    "# for i in reversed(scores):\n",
    "#     scores_window.append(i)\n",
    "#     if index == 100:\n",
    "#         break\n",
    "#     index += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=2000, eps_start=1.0, eps_end=0.05, eps_decay=0.995):\n",
    "    \"\"\"\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        eps_start (float): starting value of epsilon, for epsilon-greedy action selection\n",
    "        eps_end (float): minimum value of epsilon\n",
    "        eps_decay (float): multiplicative factor (per episode) for decreasing epsilon\n",
    "    \"\"\"\n",
    "    eps = eps_start\n",
    "    for i_episode in range(start_epoch+1, n_episodes+1):\n",
    "        state = env.reset()\n",
    "        score = 0\n",
    "        while True:\n",
    "            action = agent.act(state.copy(), eps)\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            score += reward\n",
    "            if done:\n",
    "                reward = -100\n",
    "            agent.step(state, action, reward, next_state)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                env.close()\n",
    "                break\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        eps = max(eps_end, eps_decay*eps) # decrease epsilon\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "        \n",
    "        if i_episode % 500 == 0:\n",
    "            print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)))\n",
    "            state = {'epoch': i_episode,'state_dict': agent.policy_net.state_dict(),'optimizer': agent.optimizer.state_dict(),\n",
    "                     'scores': scores }\n",
    "            torch.save(state, \"cartpole_{}.pth\".format(i_episode))\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Episode 1\tAverage Score: 17.00\r",
      "Episode 2\tAverage Score: 15.00\r",
      "Episode 3\tAverage Score: 33.33"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\deepanshu\\.conda\\envs\\ai\\lib\\site-packages\\ipykernel_launcher.py:61: UserWarning: Using a target size (torch.Size([64, 64])) that is different to the input size (torch.Size([64, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 500\tAverage Score: 10.66\n",
      "Episode 1000\tAverage Score: 49.08\n",
      "Episode 1500\tAverage Score: 249.52\n",
      "Episode 2000\tAverage Score: 625.36\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU5b3H8c8vCYR9k0UMIFJxrwIi6nVp3SpqK9baqrXV29py22rtXsFW29pya+tS661LrVpRW7faVioKKqIoohBAVtkJi2xhh4RsM7/7x5yZzCSTnCRmJgG+79crr5x55jlnnpwk53ee9Zi7IyIiUp+cli6AiIi0fgoWIiISSsFCRERCKViIiEgoBQsREQmV19IFyJSePXv6wIEDW7oYIiL7ldmzZ29191410w/YYDFw4EAKCwtbuhgiIvsVM1uTLl3NUCIiEkrBQkREQilYiIhIKAULEREJpWAhIiKhFCxERCSUgoWIiIRSsBAROUCsKt7Luyu3ZuTYB+ykPBGRg825d78FQNEdlzT7sVWzEBGRUAoWIiISSsFCRERCKViIiEgoBQsREQmlYCEiIqEULEREJJSChYiIhFKwEBGRUAoWIiISSsFCRERCKViIiEgoBQsREQmlYCEiIqEULEREJJSChYiIhFKwEBGRUAoWIiISSsFCRERCKViIiEiojAcLM8s1s7lm9lLwuoeZvWZmy4Pv3ZPyjjWzFWa21MwuTEo/2cwWBO/dZ2aW6XKLiEi1bNQsvgd8mPR6DDDF3QcDU4LXmNlxwFXA8cBI4AEzyw32eRAYDQwOvkZmodwiIhLIaLAws37AJcAjScmjgPHB9njgsqT0Z9y93N1XAyuAEWbWF+ji7jPc3YEnkvYREZEsyHTN4l7gp0A0Ka2Pu28ECL73DtILgHVJ+dYHaQXBds30WsxstJkVmllhcXFx8/wEIiKSuWBhZp8Ftrj77IbukibN60mvnej+sLsPd/fhvXr1auDHiohImLwMHvsM4FIzuxhoB3Qxs6eAzWbW1903Bk1MW4L864H+Sfv3AzYE6f3SpIuISJZkrGbh7mPdvZ+7DyTWcf2Gu38FmABcF2S7Dngx2J4AXGVm+WZ2BLGO7JlBU9UeMzstGAV1bdI+IiKSBZmsWdTlDuA5M7seWAt8EcDdF5nZc8BioAq4wd0jwT7fBh4H2gOvBF8iIpIlWQkW7v4m8GawvQ04r45844BxadILgRMyV0IREamPZnCLiEgoBQsREQmlYCEiIqEULEREJJSChYiIhFKwEBGRUAoWIiISSsFCRERCKViIiEgoBQsREQmlYCEiIqEULEREJJSChYiIhFKwEBGRUAoWIiISSsFCROQA4+7NfkwFCxGRA0wGYoWChYiIhFOwEBE5wGSgYqFgISJyoFGfhYiIhFLNQkREWoSChYjIAUajoUREJJRnoCFKwUJEREIpWIiIHGDUDCUiInUq6NYegLwca/Zj5zX7EUVEpEWc1L8rHdrmkpfb/PUA1SxERA4QmWh+ilOwEBE5gFjzt0ABChYiItIAChYiIhJKwUJEREIpWIiIHCDUwS0iIg1iZKaHO2PBwszamdlMM5tnZovM7FdBeg8ze83MlgffuyftM9bMVpjZUjO7MCn9ZDNbELx3n1mm+vtFRCSdTNYsyoFz3f0kYAgw0sxOA8YAU9x9MDAleI2ZHQdcBRwPjAQeMLPc4FgPAqOBwcHXyAyWW0REashYsPCYvcHLNsGXA6OA8UH6eOCyYHsU8Iy7l7v7amAFMMLM+gJd3H2Gxx7/9ETSPiIikgUZ7bMws1wz+wDYArzm7u8Dfdx9I0DwvXeQvQBYl7T7+iCtINiumZ7u80abWaGZFRYXFzfvDyMi0splYmnyuIwGC3ePuPsQoB+xWsIJ9WRP1w/h9aSn+7yH3X24uw/v1atX4wssIrKf269ncLv7TuBNYn0Nm4OmJYLvW4Js64H+Sbv1AzYE6f3SpIuISJZkcjRULzPrFmy3B84HlgATgOuCbNcBLwbbE4CrzCzfzI4g1pE9M2iq2mNmpwWjoK5N2kdERLIgk0uU9wXGByOacoDn3P0lM5sBPGdm1wNrgS8CuPsiM3sOWAxUATe4eyQ41reBx4H2wCvBl4iIZEnGgoW7zweGpknfBpxXxz7jgHFp0guB+vo7REQOeprBLSIiLUrBQkREQilYiIhIKAULEREJpWAhInKAyGD/toKFiMiBJFOLcitYiIhIKAULEREJpWAhIiKhFCxERA4QmsEtIiINkqlnTitYiIhIKAULEREJpWAhIiKhGhwszOxMM/tasN0reECRiIi0Gi38DG4z+wVwMzA2SGoDPJWpQomISNO09DO4Pw9cCpQAuPsGoHNmiiQiIq1NQ4NFhbs7QR3HzDpmrkgiItLaNDRYPGdmfwa6mdk3gdeBv2SuWCIi0liZnJTXoGdwu/tdZnYBsBs4GrjN3V/LXLFERKQpMtVnERoszCwXmOzu5wMKECIiB6HQZih3jwClZtY1C+UREZFWqEHNUEAZsMDMXiMYEQXg7jdlpFQiItKqNDRYTAy+RESklcrkY1Ub2sE93szaAkcFSUvdvTJzxRIRkaawDK0726BgYWafBsYDRcRWwO1vZte5+7SMlEpERFqVhjZD3Q18xt2XApjZUcDTwMmZKpiIiLQeDZ2U1yYeKADcfRmx9aFEROQg0NCaRaGZPQo8Gby+BpidmSKJiEhTeAancDc0WHwbuAG4iVifxTTggUwVSkREmqbFZnAn5fuju98TK4zlAvmZKZKIiLQ2De2zmAK0T3rdnthigiIichBoaLBo5+574y+C7Q6ZKZKIiLQ2DQ0WJWY2LP7CzIYD+zJTJBERaYpMzuBuaLD4PvC8mb1tZtOAZ4Ab69vBzPqb2VQz+9DMFpnZ94L0Hmb2mpktD753T9pnrJmtMLOlZnZhUvrJZrYgeO8+s0x14YiI7N8ydXGsN1iY2Slmdqi7zwKOAZ4FqoBJwOqQY1cBP3L3Y4HTgBvM7DhgDDDF3QcT6wsZE3zWccBVwPHASOCBoCMd4EFgNDA4+BrZ2B9URESaLqxm8WegItg+HbgFuB/YATxc347uvtHd5wTbe4APgQJgFLGlQwi+XxZsjwKecfdyd18NrABGmFlfoIu7zwge7fpE0j4iIpIFYUNnc919e7B9JfCwu78AvGBmHzT0Q8xsIDAUeB/o4+4bIRZQzKx3kK0AeC9pt/VBWmWwXTM93eeMJlYDYcCAAQ0tnoiIhAirWeSaWTygnAe8kfReQxch7AS8AHzf3XfXlzVNmteTXjvR/WF3H+7uw3v16tWQ4omIHDBa8hncTwNvmdlWYqOf3gYwsyOBXWEHN7M2xALF39z9n0HyZjPrG9Qq+gJbgvT1QP+k3fsBG4L0fmnSRUSkpgyN/6m3ZuHu44AfAY8DZ3r1wiM5wHfr2zcYsfQo8GF85ndgAnBdsH0d8GJS+lVmlm9mRxDryJ4ZNFntMbPTgmNem7SPiIhkQWhTkru/lyZtWQOOfQbwVWKPY433b9wC3AE8Z2bXA2uBLwbHXGRmzwGLiY2kuiF4/jfE1qZ6nNjM8VeCLxERyZKGrg3VaO7+DnUP+T2vjn3GAePSpBcCJzRf6UREpDEaOilPRERasdFPFPLWsuKMHV/BQkTkAPDq4s1AC83gFhERAQULERFpAAULEREJpWAhIiKhFCxERA4gmXqAg4KFiIiEUrAQEZFQChYiIhJKwUJEREIpWIiItGJ7y6salV8zuEVEDjILP9rFCb+YzMT5G2u996k7p/KXaauyVhYFCxGRVmr++tgz5t5enrpA4P1TV7BmWynjXv4wa2VRsBARaaUiwfPmcnKqG5fKKiPcOXlp1suiYCEi0kpFo7FgkduImXbWEo9VFRGRlhOJB4ucTHVbN5yChYhIKxWNN0Ml1RYytZxHmIw9VlVERD6e6mBRnRYkJZRXRVKCSaaoZiEi0kpVBc1Q20oqOPnXr7Fiy95aeY7++STOv+etjJdFwUJEpJWK1yImL9rEtpIKnpxRlDbfmm2liW1NyhMROcjEO7jjzUyZGunUEAoWIiKtVLxmYWnSsk3BQkSklYpXJKJJEcJpmWihYCEi0krFaxTx8ODuPF+4vkXKoqGzIiL7iecK17OvMlJvHj1WVUTkIFOzGSosUGSSgoWISCsVH/3UUp3ayRQsREQklIKFiEgrp5qFiIjUKd3Q2dB9MjSHW8FCRKSVa0zFojwSzUgZFCxERA4g89btzMhxFSxERFqpeJNSY5qhMkXBQkSklYr3WbSCWJG5YGFmj5nZFjNbmJTWw8xeM7PlwffuSe+NNbMVZrbUzC5MSj/ZzBYE791nLbnsoohIFjwxo4h563ZmbLnxpshkzeJxYGSNtDHAFHcfDEwJXmNmxwFXAccH+zxgZrnBPg8Co4HBwVfNY4qIHFBue3ERo+6f3tLFSJGxYOHu04DtNZJHAeOD7fHAZUnpz7h7ubuvBlYAI8ysL9DF3We4uwNPJO0jInJAa03tKNnus+jj7hsBgu+9g/QCYF1SvvVBWkGwXTM9LTMbbWaFZlZYXFzcrAUXEcm2TM2ZaIrW0sGd7ox4PelpufvD7j7c3Yf36tWr2QonInKwy3aw2Bw0LRF83xKkrwf6J+XrB2wI0vulSRcROeAdzM1QE4Drgu3rgBeT0q8ys3wzO4JYR/bMoKlqj5mdFoyCujZpHxFpRQqLtrNh576WLoZkSCaHzj4NzACONrP1ZnY9cAdwgZktBy4IXuPui4DngMXAJOAGd48v3P5t4BFind4rgVcyVWYRSfXkjCIGjpnI7rLK0LxXPDSDT9/5ZsbLJC0jY0/Kc/er63jrvDryjwPGpUkvBE5oxqKJSAM9/m4RAFt2l9GlXZvQ/BUZWpcozAfrdnL7fxbx92+eRrs2ueE7SKO1lg5uEZEmu+3FhcxZu5Olm/a0dFGaVWuag6xgISISIhp1ylrgkaatJ1QoWIhIPRq6JNGLH3yU0XK0tNsmLOSYWycRjbaCRZpaiIKFiHwsc9bu4HvPfNDSxciov7+/Fsj+6q+tqBVKwUJE6pbuWhWNOuMmLmbttlIA9pZVZbdQ9cj0pTzb9Yo1wTluDRQsRKRRlm3Zw1/eXs3Zd06lKhJt9rvtqUu38NVH38dbw7rcNWS7ZhEfjdYaZGzorIgcmJKvl1VRb/a77dFPFFIZcSoiUfLzGjcMNtOtNtmIFa0xSIJqFiLSSLXa0Zv52hYfLtqUa2amL7PZqFm00lihYCEidWvIdSusbrFmWwn/nLO+3jzJcoJgFGmhkUe/eWkx1z42M+17YUWqjER5Yfb6j1U7eHPZlvBMLUDNUCLSAJa0Vb3tHn4nfMl977C3vIphA7ozsGfH0E/KCWoWVS0QLCYv2sQj76yu8/2wIPDgmyu557Vl5OUao4bU+TSFen398cIm7ZdpqlmISAOkv0hG3UODxd7y2GipT9/1ZoM+KR4sWqJm8T9Pzq73/bAiFe8pB2DXvvC1tPY3ChYiUqewDmOn+fsJrIWbodKp7kepLtOzs9byr7kNb17b3ylYiEijJHdwx2oWqRf1ssoI/5rb9Hb7lqxZhEku0s0vLOAHz85rucJkmYKFiNQp3eU6ubbhXjvP3a8u5QfPzuPNZU17tHG8g7sq2jIr2Mbtq6i9FlRjR0Ot2LI3MXkxEnV2ldbfPBVvsmuNFCxEpE6rikuCraRO7aT3PU2fxabdsXb73U1st28tNYsfPV97CZPGBovz73mLs++cCsDvJi3hpNtfrffZIHdNXtq4QmaRgoWINEryBTO2mXoBjTc/NXV5bUsTLKYu3cKeei6ymZiM98HanbXSGhorauarjESZ8EHsidAl9dQe6nuvpSlYiEgDVF/9kluHou7MX78rbc6wC/jO0goGjpnI1CWp8wpqzrPYsHMfX/vrrFbRPxAWLOLxsaIqyk//UV3eSNQTQ4Fz0wTRWUXbufrh91pkuHBDKViISKjJizazo6QCSK1ZvLdqOw+8uTI1cwOvd0uCBxV97fFZjBj3eiK95jyL0orY3fbqrXsTeTbtKmuR5cIb2gw1ZclmniusHikVda/eN00U/eFzHzBj1TY+asXPMFewEJEU67aX8tHOfZySdAG/c/JSvvlEbLJY8vVy+ZbaT6aLPyQorBUqJynDlmB+Qiw99j1es4jHhHj+ddtLOe23U/jT1BUN+4GaKF1YaOpyH1VRT/w8NQ/x0vwNrNseCxKt+XkZChYikuKs30/ljDveSEwwiyvaVkJlJMqtLy5MpKW7uE1ZUvdyFd9+ajYVVVG+/dRsbn9pUcp72/aWs6OkolafRTTRBxLL90Zw/LfSjLZqzkX4Nu4q4/4aAamhh6+KpGaMRj0xUS8SdcqrIsxdu4PdZZXc+Pe5iXyR1rowFAoWIge97z49l4FjJobm27q3gndXbuODddUdv2Ft7EVbS1Jev7JwE/PX7+SVhZtY+NHulPdO/s3rDP31a+TkpB473keybHOsGeoXE2JBZvaaHZz4y8kpx2jspXZ7SQWz1+yo8/07Jy/l/6YsTwlckxZuYvnm+p/1XVjjmFv3VgfeqDu3/2cxn3/g3Vr9NZWRlh0uXB+tDSVykHl5wUZO6t+Ngm7tAfjPvA0N3rdD29Qlw+sb3vrUe2s4a3CvWunlVfVfEONNMpGoM/LeaYm+jXR2Bw9eWhUEpRkrt3FSv27k5jRsfNQF97zFtpIKiu64pM48d7+2LLH9qTvfBGDgIR1q5dtZWsETM9akPUZyUC2rjLDwo9iggJrzKmoG0NZENQuRg4i7852/zeHyB6Y3af/2bVKDxZ+nraoz73urtnNnmnkD1zzyfoM+60t/nlErUGzcVbsD+J5Xl7InCBp3Tl7KS/PrDn6VkSj/9dspTJy/EYBtQad9Y+d0FCU9we6JGUXsLK1I6eOpqSIpQJ5/zzTmBSPIVmzZW9curY6ChchBYEdJBfsqIonO4s27y+vfoQ4tPcP49N++USvtvjdS+xXquwDvKatiw64yxv5zfkp/S2lFVejs6rrc9uIiLv7j21RG6g44l/4pfXD+6/SiJn1mS1CwEDkIDP31a1x2//TQJTTCLpiPvF338t2tRbzJZ0dJBXPXpvYdxDvAd5dVMeiWlxPppRURTrr91SZ/5oZdZU3ed3+hPguRg8TSzXuoL1b88fXlTF+xtd5jbNnT+i+Ks1ZvZ9nmPVw/fhbrtu+jT5d8fnrhMXzh5H513v235pnTrYWChchBpL6axR9eX1bne3Gri0tC87S0wjU7+MwfpiVeb95dzo+en0dpZYTKOjrXz737rWwVb7+lZiiRDHplwcaMzcp98r01rCqOtc8v2bQ7Meyy5tyH5LkHq5OGskaizguzG/c8hj378R34rf9eyO0vLW7pYjTZTy48ukU/X8FCJEMWrN/Ft5s48mjRhl28u7LuJqHKSJRb/72QL/15Bmu3lTLy3re545Ul3PPqUgbd8jLvJjUnJQ/bTN7++/tr+NHzLb/ekjRMRY1aUbs22b18K1iIAKuK93LflOXNOgP4c396B0g/8mjtttJEZ/LK4r0s3pA6vv6S+97hy3+JDTF98r01vL08dbZyafCshV37Krnr1djw1NlrdiRGBk1csDGRN778BqTOLC7eW9G0H6wZXHBcnwbn/dd3/qtW2rSfnMPi2y/kfz416GOX5eoRAxqUb8qPPsVZg3t+7M+D9D9TmOTfI8Btnz0+bb5enfObVKYw6rMQAa54aAbbSyq46IRDGdync4P2Kdpawt7yKk4o6Nrozzv7zqkUdGvP9DHncl7QXp5uYtiIca8n1k1a/duLuevVpXx+aAGd8tsAkJtjTAgm1ZlB1/Zt2LWvkiN7d0ocI3kS3B2vfJjYrhmAMmnk8YcyadGmxOtTj+jBa4s318r33tjz+Mqj76cMfx06oDtLfzOSifM3ct4xfVhRvIcBwaS4sRcdy43nHMk9ry1j9podLNqwmz9cOYSHp61MTHCbeNOZHN2nM5c9MJ2rThnAvooI05YXs3jDbkYNKeDmi46mTa5x47lH8tfpRVw2pIBDu7Rjd1klzxeu4743VjCoZ0c+0asTT15/KuVVES5/4F0WBQH+gWuG8Z2/zQHg++cPplN+Hl89/XA27Spj2rJiunZoy01Pz+XEfl0TK/Tm5dS+Tz+2bxc+3Jh+Ut5/bjyTf8xel5LWpX0eL333TB59ZzX/mvtRIn3GmHPDfyFNoGAhGTFp4UZOH9STrh3aNPuxo1GnIhKlXY0JYpt2lbF08x4+dVTtWcN1iUSdbSXlbA8mZ8VnBDfEp+96E4CFv7qQTvnV/0p7yiprLTtRFYmSl5t6gfho575ad4s1JS+wN3vNDu6fupJ/z93AE9ePAFKXuy4pr6JTfh679lWSl2PMW7eTzu3yaJtX/blzkp7RMDfN8xoypX2Nmd95NWZY/3Tk0XzmuEM5tGs7Xv/hpxLLj8RrDvl5uVw+rB8AJx/eI2Xfzu3a8IvPpd5lX3rSYbg7+yojdGgb+9289N2zEu9/8+zUGsnto04A4OaRxyTSunZoww8/czQ/uOColLz5eblMvOksKqqiVESidMrP48FrhrF+x76U4x5+SEe+enpHlmzanXj9+aEFfLRjX8oii2cf1Ysnvj6CVcV7Offut/jTl4cy8JCOfPb/3knkOaGgC04/xs9Yw7AB3ZizdidllVFOKOjKV08/PCVY1Pw7ay4KFgegqkiU0soI4176kAGHdOCGc47M6udv3l3Gt56aw5lH9uSpb5za7Mf/3eQl/PmtVfTqnM+sn52fSB91/zts3l1e6w7d3Xlm1jo+e2JfOrdLDV4/ePaDxJ05wD9mr2Pi/I3c9rnj0n72va8vY3DvzlxyYt9E2gm/mMx9Vw/l0pMOA+DkX79ORY01fh5/t4hvnDWIe19fxjGHVtdcxk38MCVfJOp1LlVxxUMzgFhNIf7Iz5KkR3/G104CWFlcwq0vpi7UlykvfffMlAtbOvElRQb16sgph/dIuYl4+puncfonDkm739iLjm1yucwsESg+jroe4tQ2LycRiC/6ZN+0eQCOObQLf7xqCOcd2ydxUxF/Wt69Vw7hsqEFAAzq1Slt7XL6mHMxM07s142iOy5hzAvzmbN2Z6IPY9iA7om8y8dd1ISfsGHUZ1HDuu2lTFpY3d5bc2TJnZOXcNEf3852sRKmr9jKt56cnWhb315SwZf+PCOxDEJpRRVfH1/Iib98lWcL19VabmFl8V7KqyLN9sjKdO3t8bvlxRt34+4p5zAa9cSInGjUQ++s03kqWH+n5qqo8b6BmouxPfL2asb+cwGf/OWrPPpO6qSyCTXWRXp65joem7467XyDSNS59/Xl3PD3ObXeu+npuYmfs2agACjeW85vXlrMva8v51tPVe+/KukZDW8vL+YTt7xcZ1NE3Na95aHzAuatb55awy0XHxOap1/39imvnwxqPQBXDu/P/V8eluhYH9KvG7+74sSU5cnTBYpnRp/G1B9/uomlbn1GDSlIqX12adeGojsuSQSK+sTX8IqLn7p0y6W3yVCtAvajYGFmI81sqZmtMLMxmfqcq//yHt96ag4rtuzhx8/PY9AtL7OqeG/iQnD/1JUp/8xVkSiFRdtrjVRorIUf1T/6ZdvecsoqI1zzyPtMWrQpcUf5j9nrmLl6O48GM2uPu20y02os3VxWGWH6iq0MHDOR8+5+i6N/PomvPT4LiD2U/sE3Vzbpog1w3t1vcfF9qcEzvt5OVSTKfVNWMOiWl9myp4z3Vm3jj1OWc85db/LkjCK+9vgsjrl1Uui521NWyRtLqtu3k++mI8FzAn6cNKrnd68sSQRTd2fcy9V3779u4NDJm1+YXytt5urtie0FNZ4OBzB77Y6U1UWTlVdGeeSd2rOfd++rvuh/9dGZQMP6Eq58+L1631+TtHZRUw3u3YlvnjWIojsuYdiAbinv5Sc1bXXr0Dax/dh/D09ZPPB3V5zIJSf2pXO72IXyJyNjwz8vrudOHOC0QYdwRM+OH/tnOBDFBym0yc3Ew2Trtl80Q5lZLnA/cAGwHphlZhPcvdkHTa/fEbtDP/+e6kk96Sbs/HX6agb37sxXHk1dFG1Qz4588+xB3P3qUrYGo00uH1rAZUMLuPaxmdw+6niuPKU/01dsZfnmvRzWrT3/98byRBPC1884gls/eyx7y6vYXVbFoV3akZtjnPyb11NWutxRUsHO0orEjNSibSV1Bpvx7xbx21eWpKRNW1bMlx6awcyi2AXwsG7tGDUkdpcTjTqllRE65ecRjTo5Sc0i7p5oKkm+IE35cDP9undgcO9OjH1hAQD5bXL509TlAIwYNwWAo/rEOl6Tm0iKtsVqGrk5Rv/uHWiTa+zaV0lpRQQzuOCeaewtr2Lmz86jd+d2KT/HvsoIO0oq+EfSfIFH3lnNl07pz1F9OvN+0gU++dx179i2Vnqy+N9B8j7JgSY+0inZF4NmonQef7cobfqCj2oHnf99eUmanI0T74MJE2//Tvb7K07kp/+YzycLuiaaYJ76xqm8v2p74iZj4a8uZPDPXuGkftWd+z06tuXcY9KPcrrzihP535eX0LNTbKRO/A645iq2Ei7eKpCbppM8k6w5hwpmipmdDvzS3S8MXo8FcPff1rXP8OHDvbCwsFGfU1EV5aifv/JxipoRPTu1TQSeTOncLo9O+Xnk5Vpiiei4Lu3yyMvNITfH2FVaScSdHKPehdMyJT8vp9YS1zlW/TS1ZN07tKG8KkppRYSrR/Tn6ZnVo0l6dGxLZVU0McnskI5t6dU5P2WV09wcIy/HcGqPcY87/rAuXHv64dwcBMi4r59xBG3yjO4d2nJHUqB+6CvDUpqhkp19VC8Wb9jF1r0V9O6cz76KCGcf3YvLhxZw/fjY3/IJBV04uk8XXpiznmtOHcCnjurF6CdnJ45xyYl9KauIMGXJFv7rE4fw7sptHN2nM/dfM4yyygiLguOXVUbo3qEtlw8rYP2Offzt/TV87qTD2LizjEuHHMYdryzhC8P6cdxhXVLKuPCjXazfUcrIE/oyq2g7fbu2o1/3DuwsraBNbg4dg6aW8e8WMaBHB845pnfan6PZT5gAAAprSURBVBViHfYd83M55tAudeY5mB176yTKqyKs+m1qP8bsNTv4woPvMmPsufTtGmuiGjhmIjlGrbxNYWaz3X14rfT9JFhcAYx0928Er78KnOruN9bINxoYDTBgwICT16xJv7Z8fZZu2sP7q7cxedEmjurTmStP6c8DU1cyYd4GThnYnf49OoDH7jy37ClLLFV89Yj+7KuI0LV9G2av3ZGyLn3bvBw+d+JhzCzaRjQaa1OPj3I5qV9XenbKZ8qSLYkL4TlH92Lq0lhTxIiBPTiyTyfeWlpMx/xcCrq157Bu7Zm2vJi2uTkM6d+dHaUVzCrazjWnHs5xh3Xhln8u4IZzjmRPWSWrikvo260dry7azOmfOIR3lm/lrME96dQuj+MP68qRvTvxzvJiNuwqo7IqSlXUKdpWwty1O/ncSYfx7oqtnDm4J53b5RGJxuYj7NpXSZ8u7ejdOZ/DD+mAe2zJ5k75ucxbv4tjDu3MGUf25N2VW9lXEWH5lr2UVkTIzTFOOKwL63bso3/39mzZU055VZSjgqGq01ds5YwjexKJRumYn8ekhZu44Lg+dMzPY+22Urp3bEPH/Dzycozzju3Dfz82k1MG9qB3l3wWbdjNMYd25uyjevHRjn0ps6aP6NmRr51xBDNWbuMn/5hHv+7t+USvTuyriLCnvIp7vnQSndu1YUdJBSUVVfTo2JbfT1pKftKkp7lrd5Kfl8PNI4/hhIKuVEaiKe3DZZURyiuj7KuMcGjX1NoPxM5b9w5t6d6xLeVVEdrm5tTZcerutd4rq4yQn1e9T0VVlLwcIyfH2FNWSce2eZRUVNG5XRsiUWf++p2xDtFtJXRr34ZDOmVm7L1kzqrivbRvm5sICPWZvWYHfbu247Bu4XnD7O/B4ovAhTWCxQh3/25d+zSlZiEicrCrK1jsLx3c64H+Sa/7AQ1/vJeIiHws+0uwmAUMNrMjzKwtcBUwoYXLJCJy0NgvRkO5e5WZ3QhMBnKBx9w9OzOORERk/wgWAO7+MvByaEYREWl2+0szlIiItCAFCxERCaVgISIioRQsREQk1H4xKa8pzKwYaPwU7pieQN2r+rUclatxVK7GUbka50At1+HuXuuhMAdssPg4zKww3QzGlqZyNY7K1TgqV+McbOVSM5SIiIRSsBARkVAKFuk93NIFqIPK1TgqV+OoXI1zUJVLfRYiIhJKNQsREQmlYCEiIqEULJKY2UgzW2pmK8xsTJY/u7+ZTTWzD81skZl9L0j/pZl9ZGYfBF8XJ+0zNijrUjO7MINlKzKzBcHnFwZpPczsNTNbHnzvns1ymdnRSefkAzPbbWbfb6nzZWaPmdkWM1uYlNboc2RmJwfneoWZ3Wd1PU7v45XrTjNbYmbzzexfZtYtSB9oZvuSzt1DWS5Xo393WSrXs0llKjKzD4L0rJyveq4N2f37cnd9xfptcoGVwCCgLTAPOC6Ln98XGBZsdwaWAccBvwR+nCb/cUEZ84EjgrLnZqhsRUDPGmm/B8YE22OA32W7XDV+d5uAw1vqfAFnA8OAhR/nHAEzgdMBA14BLspAuT4D5AXbv0sq18DkfDWOk41yNfp3l41y1Xj/buC2bJ4v6r42ZPXvSzWLaiOAFe6+yt0rgGeAUdn6cHff6O5zgu09wIdAQT27jAKecfdyd18NrCD2M2TLKGB8sD0euKwFy3UesNLd65uxn9Fyufs0YHuaz2zwOTKzvkAXd5/hsf/sJ5L2abZyufur7l4VvHyP2JMn65StctWjRc9XXHAX/iXg6fqO0dzlqufakNW/LwWLagXAuqTX66n/Yp0xZjYQGAq8HyTdGDQZPJZU1cxmeR141cxmm9noIK2Pu2+E2B8z0LsFyhV3Fan/wC19vuIae44Kgu1slvHrxO4w444ws7lm9paZnRWkZbNcjfndZft8nQVsdvflSWlZPV81rg1Z/ftSsKiWru0u6+OKzawT8ALwfXffDTwIfAIYAmwkVg2G7Jb3DHcfBlwE3GBmZ9eTN6vn0WKP2b0UeD5Iag3nK0xdZcn2ufsZUAX8LUjaCAxw96HAD4G/m1mXLJarsb+7bP9Oryb1piSr5yvNtaHOrHV8/scql4JFtfVA/6TX/YAN2SyAmbUh9sfwN3f/J4C7b3b3iLtHgb9Q3XSStfK6+4bg+xbgX0EZNgfV2ni1e0u2yxW4CJjj7puDMrb4+UrS2HO0ntQmoYyV0cyuAz4LXBM0SRA0W2wLtmcTa+s+KlvlasLvLpvnKw+4HHg2qbxZO1/prg1k+e9LwaLaLGCwmR0R3K1eBUzI1ocH7aGPAh+6+z1J6X2Tsn0eiI/SmABcZWb5ZnYEMJhY51Vzl6ujmXWObxPrHF0YfP51QbbrgBezWa4kKXd7LX2+amjUOQqaEvaY2WnB38O1Sfs0GzMbCdwMXOrupUnpvcwsN9geFJRrVRbL1ajfXbbKFTgfWOLuiWacbJ2vuq4NZPvvq6k99AfiF3AxsZEGK4GfZfmzzyRWJZwPfBB8XQw8CSwI0icAfZP2+VlQ1qV8zFEg9ZRrELGRFfOARfHzAhwCTAGWB997ZLNcwed0ALYBXZPSWuR8EQtYG4FKYndw1zflHAHDiV0kVwJ/IlhloZnLtYJYm3b87+yhIO8Xgt/xPGAO8Lksl6vRv7tslCtIfxz4Vo28WTlf1H1tyOrfl5b7EBGRUGqGEhGRUAoWIiISSsFCRERCKViIiEgoBQsREQmlYCFSg5lFLHVF23pXIDazb5nZtc3wuUVm1rMJ+11osRVbu5vZyx+3HCLp5LV0AURaoX3uPqShmd39ofBcGXUWMJXYiqnTW7gscoBSsBBpIDMrIrbcwzlB0pfdfYWZ/RLY6+53mdlNwLeIrbm02N2vMrMewGPEJjiWAqPdfb6ZHUJsElgvYrPJLemzvgLcRGy5/PeB77h7pEZ5rgTGBscdBfQBdpvZqe5+aSbOgRy81AwlUlv7Gs1QVya9t9vdRxCb/Xpvmn3HAEPd/URiQQPgV8DcIO0WYktDA/wCeMdjC9FNAAYAmNmxwJXEFnAcAkSAa2p+kLs/S/WzFz5JbGbuUAUKyQTVLERqq68Z6umk739I8/584G9m9m/g30HamcSWhsDd3zCzQ8ysK7Fmo8uD9IlmtiPIfx5wMjAreJBZe6oXiatpMLGlGwA6eOx5ByLNTsFCpHG8ju24S4gFgUuBW83seOpfGjrdMQwY7+5j6yuIxR5x2xPIM7PFQF+LPfLzu+7+dv0/hkjjqBlKpHGuTPo+I/kNM8sB+rv7VOCnQDegEzCNoBnJzD4NbPXY8wiS0y8C4g/7mQJcYWa9g/d6mNnhNQvi7sOBicT6K35PbJHHIQoUkgmqWYjU1j64Q4+b5O7x4bP5ZvY+sRutq2vslws8FTQxGfAHd98ZdID/1czmE+vgji8r/SvgaTObA7wFrAVw98Vm9nNiTyfMIbYC6g1AusfGDiPWEf4d4J4074s0C606K9JAwWio4e6+taXLIpJtaoYSEZFQqlmIiEgo1SxERCSUgoWIiIRSsBARkVAKFiIiEkrBQkREQv0/62sZA/nUug4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = train()\n",
    "# plot the scores\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your total score is:  225.0\n"
     ]
    }
   ],
   "source": [
    "# watch an trained agent\n",
    "score = 0\n",
    "state = env.reset()\n",
    "while True:\n",
    "    action = agent.act(state)\n",
    "    env.render()\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = next_state\n",
    "    score += reward\n",
    "    time.sleep(0.05)\n",
    "    if done:\n",
    "        print(\"Your total score is: \", score)\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}