DeepQNetworksInOpenAIGym.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gym: 0.10.5\n",
      "Tensorflow: 1.9.0\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from collections import deque\n",
    "print(\"Gym:\", gym.__version__)\n",
    "print(\"Tensorflow:\", tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Observation space: Box(4,)\n",
      "Action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env_name = \"CartPole-v0\"\n",
    "env = gym.make(env_name)\n",
    "print(\"Observation space:\", env.observation_space)\n",
    "print(\"Action space:\", env.action_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QNetwork():\n",
    "    def __init__(self, state_dim, action_size):\n",
    "        self.state_in = tf.placeholder(tf.float32, shape=[None, *state_dim])\n",
    "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
    "        self.q_target_in = tf.placeholder(tf.float32, shape=[None])\n",
    "        action_one_hot = tf.one_hot(self.action_in, depth=action_size)\n",
    "        \n",
    "        self.hidden1 = tf.layers.dense(self.state_in, 100, activation=tf.nn.relu)\n",
    "        self.q_state = tf.layers.dense(self.hidden1, action_size, activation=None)\n",
    "        self.q_state_action = tf.reduce_sum(tf.multiply(self.q_state, action_one_hot), axis=1)\n",
    "        \n",
    "        self.loss = tf.reduce_mean(tf.square(self.q_state_action - self.q_target_in))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(self.loss)\n",
    "        \n",
    "    def update_model(self, session, state, action, q_target):\n",
    "        feed = {self.state_in: state, self.action_in: action, self.q_target_in: q_target}\n",
    "        session.run(self.optimizer, feed_dict=feed)\n",
    "        \n",
    "    def get_q_state(self, session, state):\n",
    "        q_state = session.run(self.q_state, feed_dict={self.state_in: state})\n",
    "        return q_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer():\n",
    "    def __init__(self, maxlen):\n",
    "        self.buffer = deque(maxlen=maxlen)\n",
    "        \n",
    "    def add(self, experience):\n",
    "        self.buffer.append(experience)\n",
    "        \n",
    "    def sample(self, batch_size):\n",
    "        sample_size = min(len(self.buffer), batch_size)\n",
    "        samples = random.choices(self.buffer, k=sample_size)\n",
    "        return map(list, zip(*samples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent():\n",
    "    def __init__(self, env):\n",
    "        self.state_dim = env.observation_space.shape\n",
    "        self.action_size = env.action_space.n\n",
    "        self.q_network = QNetwork(self.state_dim, self.action_size)\n",
    "        self.replay_buffer = ReplayBuffer(maxlen=10000)\n",
    "        self.gamma = 0.97\n",
    "        self.eps = 1.0\n",
    "        \n",
    "        self.sess = tf.Session()\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "    def get_action(self, state):\n",
    "        q_state = self.q_network.get_q_state(self.sess, [state])\n",
    "        action_greedy = np.argmax(q_state)\n",
    "        action_random = np.random.randint(self.action_size)\n",
    "        action = action_random if random.random() < self.eps else action_greedy\n",
    "        return action\n",
    "    \n",
    "    def train(self, state, action, next_state, reward, done):\n",
    "        self.replay_buffer.add((state, action, next_state, reward, done))\n",
    "        states, actions, next_states, rewards, dones = self.replay_buffer.sample(50)\n",
    "        q_next_states = self.q_network.get_q_state(self.sess, next_states)\n",
    "        q_next_states[dones] = np.zeros([self.action_size])\n",
    "        q_targets = rewards + self.gamma * np.max(q_next_states, axis=1)\n",
    "        self.q_network.update_model(self.sess, states, actions, q_targets)\n",
    "        \n",
    "        if done: self.eps = max(0.1, 0.99*self.eps)\n",
    "    \n",
    "    def __del__(self):\n",
    "        self.sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, total_reward: 17.00\n",
      "Episode: 1, total_reward: 11.00\n",
      "Episode: 2, total_reward: 14.00\n",
      "Episode: 3, total_reward: 22.00\n",
      "Episode: 4, total_reward: 19.00\n",
      "Episode: 5, total_reward: 37.00\n",
      "Episode: 6, total_reward: 13.00\n",
      "Episode: 7, total_reward: 13.00\n",
      "Episode: 8, total_reward: 18.00\n",
      "Episode: 9, total_reward: 21.00\n",
      "Episode: 10, total_reward: 20.00\n",
      "Episode: 11, total_reward: 41.00\n",
      "Episode: 12, total_reward: 21.00\n",
      "Episode: 13, total_reward: 25.00\n",
      "Episode: 14, total_reward: 20.00\n",
      "Episode: 15, total_reward: 11.00\n",
      "Episode: 16, total_reward: 13.00\n",
      "Episode: 17, total_reward: 19.00\n",
      "Episode: 18, total_reward: 14.00\n",
      "Episode: 19, total_reward: 39.00\n",
      "Episode: 20, total_reward: 12.00\n",
      "Episode: 21, total_reward: 37.00\n",
      "Episode: 22, total_reward: 23.00\n",
      "Episode: 23, total_reward: 15.00\n",
      "Episode: 24, total_reward: 40.00\n",
      "Episode: 25, total_reward: 14.00\n",
      "Episode: 26, total_reward: 11.00\n",
      "Episode: 27, total_reward: 30.00\n",
      "Episode: 28, total_reward: 13.00\n",
      "Episode: 29, total_reward: 13.00\n",
      "Episode: 30, total_reward: 12.00\n",
      "Episode: 31, total_reward: 14.00\n",
      "Episode: 32, total_reward: 17.00\n",
      "Episode: 33, total_reward: 15.00\n",
      "Episode: 34, total_reward: 10.00\n",
      "Episode: 35, total_reward: 10.00\n",
      "Episode: 36, total_reward: 13.00\n",
      "Episode: 37, total_reward: 12.00\n",
      "Episode: 38, total_reward: 11.00\n",
      "Episode: 39, total_reward: 9.00\n",
      "Episode: 40, total_reward: 15.00\n",
      "Episode: 41, total_reward: 17.00\n",
      "Episode: 42, total_reward: 15.00\n",
      "Episode: 43, total_reward: 31.00\n",
      "Episode: 44, total_reward: 10.00\n",
      "Episode: 45, total_reward: 19.00\n",
      "Episode: 46, total_reward: 8.00\n",
      "Episode: 47, total_reward: 22.00\n",
      "Episode: 48, total_reward: 11.00\n",
      "Episode: 49, total_reward: 10.00\n",
      "Episode: 50, total_reward: 13.00\n",
      "Episode: 51, total_reward: 33.00\n",
      "Episode: 52, total_reward: 10.00\n",
      "Episode: 53, total_reward: 11.00\n",
      "Episode: 54, total_reward: 12.00\n",
      "Episode: 55, total_reward: 12.00\n",
      "Episode: 56, total_reward: 19.00\n",
      "Episode: 57, total_reward: 65.00\n",
      "Episode: 58, total_reward: 15.00\n",
      "Episode: 59, total_reward: 34.00\n",
      "Episode: 60, total_reward: 43.00\n",
      "Episode: 61, total_reward: 72.00\n",
      "Episode: 62, total_reward: 45.00\n",
      "Episode: 63, total_reward: 27.00\n",
      "Episode: 64, total_reward: 43.00\n",
      "Episode: 65, total_reward: 62.00\n",
      "Episode: 66, total_reward: 45.00\n",
      "Episode: 67, total_reward: 60.00\n",
      "Episode: 68, total_reward: 60.00\n",
      "Episode: 69, total_reward: 57.00\n",
      "Episode: 70, total_reward: 19.00\n",
      "Episode: 71, total_reward: 67.00\n",
      "Episode: 72, total_reward: 99.00\n",
      "Episode: 73, total_reward: 131.00\n",
      "Episode: 74, total_reward: 23.00\n",
      "Episode: 75, total_reward: 39.00\n",
      "Episode: 76, total_reward: 90.00\n",
      "Episode: 77, total_reward: 58.00\n",
      "Episode: 78, total_reward: 62.00\n",
      "Episode: 79, total_reward: 91.00\n",
      "Episode: 80, total_reward: 48.00\n",
      "Episode: 81, total_reward: 47.00\n",
      "Episode: 82, total_reward: 68.00\n",
      "Episode: 83, total_reward: 51.00\n",
      "Episode: 84, total_reward: 92.00\n",
      "Episode: 85, total_reward: 75.00\n",
      "Episode: 86, total_reward: 67.00\n",
      "Episode: 87, total_reward: 100.00\n",
      "Episode: 88, total_reward: 52.00\n",
      "Episode: 89, total_reward: 50.00\n",
      "Episode: 90, total_reward: 122.00\n",
      "Episode: 91, total_reward: 65.00\n",
      "Episode: 92, total_reward: 62.00\n",
      "Episode: 93, total_reward: 61.00\n",
      "Episode: 94, total_reward: 76.00\n",
      "Episode: 95, total_reward: 89.00\n",
      "Episode: 96, total_reward: 11.00\n",
      "Episode: 97, total_reward: 99.00\n",
      "Episode: 98, total_reward: 147.00\n",
      "Episode: 99, total_reward: 156.00\n",
      "Episode: 100, total_reward: 100.00\n",
      "Episode: 101, total_reward: 88.00\n",
      "Episode: 102, total_reward: 167.00\n",
      "Episode: 103, total_reward: 124.00\n",
      "Episode: 104, total_reward: 123.00\n",
      "Episode: 105, total_reward: 55.00\n",
      "Episode: 106, total_reward: 124.00\n",
      "Episode: 107, total_reward: 81.00\n",
      "Episode: 108, total_reward: 129.00\n",
      "Episode: 109, total_reward: 165.00\n",
      "Episode: 110, total_reward: 123.00\n",
      "Episode: 111, total_reward: 131.00\n",
      "Episode: 112, total_reward: 63.00\n",
      "Episode: 113, total_reward: 169.00\n",
      "Episode: 114, total_reward: 198.00\n",
      "Episode: 115, total_reward: 141.00\n",
      "Episode: 116, total_reward: 174.00\n",
      "Episode: 117, total_reward: 135.00\n",
      "Episode: 118, total_reward: 196.00\n",
      "Episode: 119, total_reward: 200.00\n",
      "Episode: 120, total_reward: 200.00\n",
      "Episode: 121, total_reward: 200.00\n",
      "Episode: 122, total_reward: 200.00\n",
      "Episode: 123, total_reward: 200.00\n",
      "Episode: 124, total_reward: 131.00\n",
      "Episode: 125, total_reward: 200.00\n",
      "Episode: 126, total_reward: 200.00\n",
      "Episode: 127, total_reward: 200.00\n",
      "Episode: 128, total_reward: 125.00\n",
      "Episode: 129, total_reward: 200.00\n",
      "Episode: 130, total_reward: 200.00\n",
      "Episode: 131, total_reward: 200.00\n",
      "Episode: 132, total_reward: 191.00\n",
      "Episode: 133, total_reward: 200.00\n",
      "Episode: 134, total_reward: 200.00\n",
      "Episode: 135, total_reward: 200.00\n",
      "Episode: 136, total_reward: 200.00\n",
      "Episode: 137, total_reward: 200.00\n",
      "Episode: 138, total_reward: 200.00\n",
      "Episode: 139, total_reward: 200.00\n",
      "Episode: 140, total_reward: 200.00\n",
      "Episode: 141, total_reward: 200.00\n",
      "Episode: 142, total_reward: 200.00\n",
      "Episode: 143, total_reward: 200.00\n",
      "Episode: 144, total_reward: 200.00\n",
      "Episode: 145, total_reward: 200.00\n",
      "Episode: 146, total_reward: 200.00\n",
      "Episode: 147, total_reward: 200.00\n",
      "Episode: 148, total_reward: 200.00\n",
      "Episode: 149, total_reward: 200.00\n",
      "Episode: 150, total_reward: 200.00\n",
      "Episode: 151, total_reward: 200.00\n",
      "Episode: 152, total_reward: 200.00\n",
      "Episode: 153, total_reward: 200.00\n",
      "Episode: 154, total_reward: 200.00\n",
      "Episode: 155, total_reward: 200.00\n",
      "Episode: 156, total_reward: 200.00\n",
      "Episode: 157, total_reward: 200.00\n",
      "Episode: 158, total_reward: 200.00\n",
      "Episode: 159, total_reward: 200.00\n",
      "Episode: 160, total_reward: 200.00\n",
      "Episode: 161, total_reward: 200.00\n",
      "Episode: 162, total_reward: 200.00\n",
      "Episode: 163, total_reward: 200.00\n",
      "Episode: 164, total_reward: 200.00\n",
      "Episode: 165, total_reward: 200.00\n",
      "Episode: 166, total_reward: 200.00\n",
      "Episode: 167, total_reward: 200.00\n",
      "Episode: 168, total_reward: 200.00\n",
      "Episode: 169, total_reward: 200.00\n",
      "Episode: 170, total_reward: 200.00\n",
      "Episode: 171, total_reward: 200.00\n",
      "Episode: 172, total_reward: 200.00\n",
      "Episode: 173, total_reward: 200.00\n",
      "Episode: 174, total_reward: 200.00\n",
      "Episode: 175, total_reward: 200.00\n",
      "Episode: 176, total_reward: 200.00\n",
      "Episode: 177, total_reward: 200.00\n",
      "Episode: 178, total_reward: 200.00\n",
      "Episode: 179, total_reward: 200.00\n",
      "Episode: 180, total_reward: 200.00\n",
      "Episode: 181, total_reward: 200.00\n",
      "Episode: 182, total_reward: 200.00\n",
      "Episode: 183, total_reward: 174.00\n",
      "Episode: 184, total_reward: 200.00\n",
      "Episode: 185, total_reward: 200.00\n",
      "Episode: 186, total_reward: 200.00\n",
      "Episode: 187, total_reward: 200.00\n",
      "Episode: 188, total_reward: 200.00\n",
      "Episode: 189, total_reward: 200.00\n",
      "Episode: 190, total_reward: 200.00\n",
      "Episode: 191, total_reward: 200.00\n",
      "Episode: 192, total_reward: 196.00\n",
      "Episode: 193, total_reward: 200.00\n",
      "Episode: 194, total_reward: 200.00\n",
      "Episode: 195, total_reward: 200.00\n",
      "Episode: 196, total_reward: 200.00\n",
      "Episode: 197, total_reward: 200.00\n",
      "Episode: 198, total_reward: 200.00\n",
      "Episode: 199, total_reward: 200.00\n",
      "Episode: 200, total_reward: 200.00\n",
      "Episode: 201, total_reward: 200.00\n",
      "Episode: 202, total_reward: 158.00\n",
      "Episode: 203, total_reward: 200.00\n",
      "Episode: 204, total_reward: 200.00\n",
      "Episode: 205, total_reward: 198.00\n",
      "Episode: 206, total_reward: 200.00\n",
      "Episode: 207, total_reward: 200.00\n",
      "Episode: 208, total_reward: 199.00\n",
      "Episode: 209, total_reward: 200.00\n",
      "Episode: 210, total_reward: 200.00\n",
      "Episode: 211, total_reward: 200.00\n",
      "Episode: 212, total_reward: 180.00\n",
      "Episode: 213, total_reward: 164.00\n",
      "Episode: 214, total_reward: 138.00\n",
      "Episode: 215, total_reward: 177.00\n",
      "Episode: 216, total_reward: 200.00\n",
      "Episode: 217, total_reward: 200.00\n",
      "Episode: 218, total_reward: 200.00\n",
      "Episode: 219, total_reward: 200.00\n",
      "Episode: 220, total_reward: 185.00\n",
      "Episode: 221, total_reward: 196.00\n",
      "Episode: 222, total_reward: 200.00\n",
      "Episode: 223, total_reward: 200.00\n",
      "Episode: 224, total_reward: 200.00\n",
      "Episode: 225, total_reward: 200.00\n",
      "Episode: 226, total_reward: 200.00\n",
      "Episode: 227, total_reward: 200.00\n",
      "Episode: 228, total_reward: 200.00\n",
      "Episode: 229, total_reward: 200.00\n",
      "Episode: 230, total_reward: 122.00\n",
      "Episode: 231, total_reward: 193.00\n",
      "Episode: 232, total_reward: 180.00\n",
      "Episode: 233, total_reward: 200.00\n",
      "Episode: 234, total_reward: 200.00\n",
      "Episode: 235, total_reward: 200.00\n",
      "Episode: 236, total_reward: 194.00\n",
      "Episode: 237, total_reward: 200.00\n",
      "Episode: 238, total_reward: 200.00\n",
      "Episode: 239, total_reward: 200.00\n",
      "Episode: 240, total_reward: 200.00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 241, total_reward: 199.00\n",
      "Episode: 242, total_reward: 200.00\n",
      "Episode: 243, total_reward: 200.00\n",
      "Episode: 244, total_reward: 200.00\n",
      "Episode: 245, total_reward: 200.00\n",
      "Episode: 246, total_reward: 200.00\n",
      "Episode: 247, total_reward: 200.00\n",
      "Episode: 248, total_reward: 200.00\n",
      "Episode: 249, total_reward: 200.00\n",
      "Episode: 250, total_reward: 200.00\n",
      "Episode: 251, total_reward: 200.00\n",
      "Episode: 252, total_reward: 200.00\n",
      "Episode: 253, total_reward: 200.00\n",
      "Episode: 254, total_reward: 200.00\n",
      "Episode: 255, total_reward: 200.00\n",
      "Episode: 256, total_reward: 200.00\n",
      "Episode: 257, total_reward: 200.00\n",
      "Episode: 258, total_reward: 200.00\n",
      "Episode: 259, total_reward: 200.00\n",
      "Episode: 260, total_reward: 200.00\n",
      "Episode: 261, total_reward: 200.00\n",
      "Episode: 262, total_reward: 200.00\n",
      "Episode: 263, total_reward: 200.00\n",
      "Episode: 264, total_reward: 200.00\n",
      "Episode: 265, total_reward: 200.00\n",
      "Episode: 266, total_reward: 170.00\n",
      "Episode: 267, total_reward: 200.00\n",
      "Episode: 268, total_reward: 181.00\n",
      "Episode: 269, total_reward: 200.00\n",
      "Episode: 270, total_reward: 200.00\n",
      "Episode: 271, total_reward: 200.00\n",
      "Episode: 272, total_reward: 200.00\n",
      "Episode: 273, total_reward: 200.00\n",
      "Episode: 274, total_reward: 200.00\n",
      "Episode: 275, total_reward: 200.00\n",
      "Episode: 276, total_reward: 200.00\n",
      "Episode: 277, total_reward: 200.00\n",
      "Episode: 278, total_reward: 200.00\n",
      "Episode: 279, total_reward: 158.00\n",
      "Episode: 280, total_reward: 200.00\n",
      "Episode: 281, total_reward: 200.00\n",
      "Episode: 282, total_reward: 200.00\n",
      "Episode: 283, total_reward: 200.00\n",
      "Episode: 284, total_reward: 200.00\n",
      "Episode: 285, total_reward: 200.00\n",
      "Episode: 286, total_reward: 198.00\n",
      "Episode: 287, total_reward: 200.00\n",
      "Episode: 288, total_reward: 200.00\n",
      "Episode: 289, total_reward: 200.00\n",
      "Episode: 290, total_reward: 200.00\n",
      "Episode: 291, total_reward: 200.00\n",
      "Episode: 292, total_reward: 200.00\n",
      "Episode: 293, total_reward: 200.00\n",
      "Episode: 294, total_reward: 200.00\n",
      "Episode: 295, total_reward: 200.00\n",
      "Episode: 296, total_reward: 200.00\n",
      "Episode: 297, total_reward: 200.00\n",
      "Episode: 298, total_reward: 200.00\n",
      "Episode: 299, total_reward: 200.00\n",
      "Episode: 300, total_reward: 158.00\n",
      "Episode: 301, total_reward: 200.00\n",
      "Episode: 302, total_reward: 200.00\n",
      "Episode: 303, total_reward: 148.00\n",
      "Episode: 304, total_reward: 200.00\n",
      "Episode: 305, total_reward: 200.00\n",
      "Episode: 306, total_reward: 200.00\n",
      "Episode: 307, total_reward: 200.00\n",
      "Episode: 308, total_reward: 200.00\n",
      "Episode: 309, total_reward: 172.00\n",
      "Episode: 310, total_reward: 200.00\n",
      "Episode: 311, total_reward: 195.00\n",
      "Episode: 312, total_reward: 200.00\n",
      "Episode: 313, total_reward: 200.00\n",
      "Episode: 314, total_reward: 200.00\n",
      "Episode: 315, total_reward: 200.00\n",
      "Episode: 316, total_reward: 200.00\n",
      "Episode: 317, total_reward: 199.00\n",
      "Episode: 318, total_reward: 200.00\n",
      "Episode: 319, total_reward: 200.00\n",
      "Episode: 320, total_reward: 200.00\n",
      "Episode: 321, total_reward: 200.00\n",
      "Episode: 322, total_reward: 187.00\n",
      "Episode: 323, total_reward: 200.00\n",
      "Episode: 324, total_reward: 200.00\n",
      "Episode: 325, total_reward: 200.00\n",
      "Episode: 326, total_reward: 200.00\n",
      "Episode: 327, total_reward: 200.00\n",
      "Episode: 328, total_reward: 187.00\n",
      "Episode: 329, total_reward: 156.00\n",
      "Episode: 330, total_reward: 200.00\n",
      "Episode: 331, total_reward: 200.00\n",
      "Episode: 332, total_reward: 200.00\n",
      "Episode: 333, total_reward: 200.00\n",
      "Episode: 334, total_reward: 200.00\n",
      "Episode: 335, total_reward: 166.00\n",
      "Episode: 336, total_reward: 200.00\n",
      "Episode: 337, total_reward: 200.00\n",
      "Episode: 338, total_reward: 145.00\n",
      "Episode: 339, total_reward: 149.00\n",
      "Episode: 340, total_reward: 200.00\n",
      "Episode: 341, total_reward: 200.00\n",
      "Episode: 342, total_reward: 164.00\n",
      "Episode: 343, total_reward: 159.00\n",
      "Episode: 344, total_reward: 178.00\n",
      "Episode: 345, total_reward: 200.00\n",
      "Episode: 346, total_reward: 200.00\n",
      "Episode: 347, total_reward: 200.00\n",
      "Episode: 348, total_reward: 200.00\n",
      "Episode: 349, total_reward: 154.00\n",
      "Episode: 350, total_reward: 188.00\n",
      "Episode: 351, total_reward: 176.00\n",
      "Episode: 352, total_reward: 200.00\n",
      "Episode: 353, total_reward: 199.00\n",
      "Episode: 354, total_reward: 200.00\n",
      "Episode: 355, total_reward: 193.00\n",
      "Episode: 356, total_reward: 200.00\n",
      "Episode: 357, total_reward: 163.00\n",
      "Episode: 358, total_reward: 200.00\n",
      "Episode: 359, total_reward: 200.00\n",
      "Episode: 360, total_reward: 200.00\n",
      "Episode: 361, total_reward: 200.00\n",
      "Episode: 362, total_reward: 191.00\n",
      "Episode: 363, total_reward: 179.00\n",
      "Episode: 364, total_reward: 175.00\n",
      "Episode: 365, total_reward: 200.00\n",
      "Episode: 366, total_reward: 146.00\n",
      "Episode: 367, total_reward: 200.00\n",
      "Episode: 368, total_reward: 200.00\n",
      "Episode: 369, total_reward: 200.00\n",
      "Episode: 370, total_reward: 200.00\n",
      "Episode: 371, total_reward: 200.00\n",
      "Episode: 372, total_reward: 159.00\n",
      "Episode: 373, total_reward: 200.00\n",
      "Episode: 374, total_reward: 200.00\n",
      "Episode: 375, total_reward: 196.00\n",
      "Episode: 376, total_reward: 200.00\n",
      "Episode: 377, total_reward: 116.00\n",
      "Episode: 378, total_reward: 156.00\n",
      "Episode: 379, total_reward: 200.00\n",
      "Episode: 380, total_reward: 200.00\n",
      "Episode: 381, total_reward: 200.00\n",
      "Episode: 382, total_reward: 200.00\n",
      "Episode: 383, total_reward: 200.00\n",
      "Episode: 384, total_reward: 200.00\n",
      "Episode: 385, total_reward: 74.00\n",
      "Episode: 386, total_reward: 200.00\n",
      "Episode: 387, total_reward: 200.00\n",
      "Episode: 388, total_reward: 200.00\n",
      "Episode: 389, total_reward: 200.00\n",
      "Episode: 390, total_reward: 200.00\n",
      "Episode: 391, total_reward: 200.00\n",
      "Episode: 392, total_reward: 200.00\n",
      "Episode: 393, total_reward: 200.00\n",
      "Episode: 394, total_reward: 200.00\n",
      "Episode: 395, total_reward: 200.00\n",
      "Episode: 396, total_reward: 200.00\n",
      "Episode: 397, total_reward: 200.00\n",
      "Episode: 398, total_reward: 200.00\n",
      "Episode: 399, total_reward: 200.00\n"
     ]
    }
   ],
   "source": [
    "agent = DQNAgent(env)\n",
    "num_episodes = 400\n",
    "\n",
    "for ep in range(num_episodes):\n",
    "    state = env.reset()\n",
    "    total_reward = 0\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        agent.train(state, action, next_state, reward, done)\n",
    "        env.render()\n",
    "        total_reward += reward\n",
    "        state = next_state\n",
    "        \n",
    "    print(\"Episode: {}, total_reward: {:.2f}\".format(ep, total_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}