FasterQLearningTrainingWithExperienceReplay.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using OpenAI Gym: 0.10.5\n",
      "Using Tensorflow: 1.9.0\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import time\n",
    "from gym.envs.registration import register\n",
    "from IPython.display import clear_output\n",
    "from collections import deque\n",
    "print(\"Using OpenAI Gym:\", gym.__version__)\n",
    "print(\"Using Tensorflow:\", tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Observation space: Discrete(16)\n",
      "Action space: Discrete(4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "gym.spaces.discrete.Discrete"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://github.com/openai/gym/blob/master/gym/envs/toy_text/frozen_lake.py\n",
    "# https://github.com/openai/gym/blob/master/gym/envs/__init__.py\n",
    "try:\n",
    "    register(\n",
    "        id='FrozenLakeNoSlip-v0',\n",
    "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
    "        kwargs={'map_name' : '4x4', 'is_slippery':False},\n",
    "        max_episode_steps=100,\n",
    "        reward_threshold=0.78, # optimum = .8196\n",
    "    )\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# env_name = \"CartPole-v1\"\n",
    "# env_name = \"MountainCar-v0\"\n",
    "# env_name = \"MountainCarContinuous-v0\"\n",
    "# env_name = \"Acrobot-v1\"\n",
    "# env_name = \"Pendulum-v0\"\n",
    "env_name = \"FrozenLake-v0\"\n",
    "env_name = \"FrozenLakeNoSlip-v0\"\n",
    "env = gym.make(env_name)\n",
    "print(\"Observation space:\", env.observation_space)\n",
    "print(\"Action space:\", env.action_space)\n",
    "type(env.action_space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    def __init__(self, env):\n",
    "        self.is_discrete = \\\n",
    "            type(env.action_space) == gym.spaces.discrete.Discrete\n",
    "        \n",
    "        if self.is_discrete:\n",
    "            self.action_size = env.action_space.n\n",
    "            print(\"Action size:\", self.action_size)\n",
    "        else:\n",
    "            self.action_low = env.action_space.low\n",
    "            self.action_high = env.action_space.high\n",
    "            self.action_shape = env.action_space.shape\n",
    "            print(\"Action range:\", self.action_low, self.action_high)\n",
    "        \n",
    "    def get_action(self, state):\n",
    "        if self.is_discrete:\n",
    "            action = random.choice(range(self.action_size))\n",
    "        else:\n",
    "            action = np.random.uniform(self.action_low,\n",
    "                                       self.action_high,\n",
    "                                       self.action_shape)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action size: 4\n",
      "State size: 16\n"
     ]
    }
   ],
   "source": [
    "class QNAgent(Agent):\n",
    "    def __init__(self, env, discount_rate=0.97, learning_rate=0.001):\n",
    "        super().__init__(env)\n",
    "        self.state_size = env.observation_space.n\n",
    "        print(\"State size:\", self.state_size)\n",
    "        \n",
    "        self.eps = 1.0\n",
    "        self.discount_rate = discount_rate\n",
    "        self.learning_rate = learning_rate\n",
    "        self.build_model()\n",
    "        \n",
    "        self.sess = tf.Session()\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "        self.replay_buffer = deque(maxlen=1000)\n",
    "        \n",
    "    def build_model(self):\n",
    "        tf.reset_default_graph()\n",
    "        self.state_in = tf.placeholder(tf.int32, shape=[None])\n",
    "        self.action_in = tf.placeholder(tf.int32, shape=[None])\n",
    "        self.target_in = tf.placeholder(tf.float32, shape=[None])\n",
    "        \n",
    "        self.state = tf.one_hot(self.state_in, depth=self.state_size)\n",
    "        self.action = tf.one_hot(self.action_in, depth=self.action_size)\n",
    "        \n",
    "        self.q_state = tf.layers.dense(self.state, units=self.action_size, name=\"q_table\")\n",
    "        self.q_action = tf.reduce_sum(tf.multiply(self.q_state, self.action), axis=1)\n",
    "        \n",
    "        self.loss = tf.reduce_sum(tf.square(self.target_in - self.q_action))\n",
    "        self.optimizer = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)\n",
    "        \n",
    "    def get_action(self, state):\n",
    "        q_state = self.sess.run(self.q_state, feed_dict={self.state_in: [state]})\n",
    "        action_greedy = np.argmax(q_state)\n",
    "        action_random = super().get_action(state)\n",
    "        return action_random if random.random() < self.eps else action_greedy\n",
    "    \n",
    "    def train(self, experience, batch_size=50):\n",
    "        self.replay_buffer.append(experience)\n",
    "        samples = random.choices(self.replay_buffer, k=batch_size)\n",
    "        state, action, next_state, reward, done = (list(col) for col in zip(experience, *samples))\n",
    "#         state, action, next_state, reward, done = ([exp] for exp in experience)\n",
    "        \n",
    "        q_next = self.sess.run(self.q_state, feed_dict={self.state_in: next_state})\n",
    "        q_next[done] = np.zeros([self.action_size])\n",
    "        q_target = reward + self.discount_rate * np.max(q_next, axis=1)\n",
    "        \n",
    "        feed = {self.state_in: state, self.action_in: action, self.target_in: q_target}\n",
    "        self.sess.run(self.optimizer, feed_dict=feed)\n",
    "        \n",
    "        if experience[4]:\n",
    "            self.eps = self.eps * 0.99\n",
    "            \n",
    "    def __del__(self):\n",
    "        self.sess.close()\n",
    "        \n",
    "agent = QNAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s: 15 a: 2\n",
      "Episode: 99, Total reward: 73.0, eps: 0.13397967485796175\n",
      "  (Right)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFF\u001b[41mG\u001b[0m\n",
      "[[ 0.2915219   0.4591379   0.2689546   0.31623498]\n",
      " [ 0.31743488 -0.3748523   0.04215977  0.00409675]\n",
      " [ 0.1477144   0.48421127  0.04322524  0.31961307]\n",
      " [ 0.00712917  0.08716363  0.0486892   0.02223596]\n",
      " [ 0.31576723  0.48447582 -0.51515913  0.311779  ]\n",
      " [ 0.0684554   0.04598576  0.05356741 -0.01328123]\n",
      " [-0.42640948  0.5349467  -0.41976473  0.30681062]\n",
      " [ 0.25691265  0.30407184 -0.5459194   0.25798124]\n",
      " [ 0.34918264 -0.38702574  0.41161922  0.34177908]\n",
      " [ 0.35890004  0.27314752  0.437125   -0.5167427 ]\n",
      " [ 0.2928187   0.56286657 -0.51294297  0.39509773]\n",
      " [-0.36676133  0.50335765 -0.16498888 -0.16828948]\n",
      " [ 0.01138669  0.49019372 -0.12844104  0.17367804]\n",
      " [-0.53203475  0.01639959  0.24097243 -0.0812238 ]\n",
      " [ 0.08020027  0.47967038  0.49306536  0.0293826 ]\n",
      " [ 0.00829726  0.12029958 -0.30489913 -0.42049965]]\n"
     ]
    }
   ],
   "source": [
    "total_reward = 0\n",
    "for ep in range(100):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = agent.get_action(state)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        agent.train((state,action,next_state,reward,done))\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "        \n",
    "        print(\"s:\", state, \"a:\", action)\n",
    "        print(\"Episode: {}, Total reward: {}, eps: {}\".format(ep,total_reward,agent.eps))\n",
    "        env.render()\n",
    "        with tf.variable_scope(\"q_table\", reuse=True):\n",
    "            weights = agent.sess.run(tf.get_variable(\"kernel\"))\n",
    "            print(weights)\n",
    "        time.sleep(0.05)\n",
    "        clear_output(wait=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}