cgames/01_ping_pong/ping_pong_ppo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "LN0nZwyMGadB"
   },
   "outputs": [],
   "source": [
    "# Pong with Poximal Policy Optimization\n",
    "\n",
    "## Step 1: Import the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import clear_output\n",
    "import math\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "from algos.agents import PPOAgent\n",
    "from algos.models import ActorCnn, CriticCnn\n",
    "from algos.preprocessing.stack_frame import preprocess_frame, stack_frame"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "tfo8jleHGadK"
   },
   "outputs": [],
   "source": [
    "## Step 2: Create our environment\n",
    "\n",
    "Initialize the environment in the code cell below.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('Pong-v0')\n",
    "env.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"Device: \", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "nS221MgXGadP"
   },
   "outputs": [],
   "source": [
    "## Step 3: Viewing our Enviroment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The size of frame is: \", env.observation_space.shape)\n",
    "print(\"No. of Actions: \", env.action_space.n)\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(env.reset())\n",
    "plt.title('Original Frame')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Execute the code cell below to play Pong with a random policy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_play():\n",
    "    score = 0\n",
    "    env.reset()\n",
    "    while True:\n",
    "        env.render()\n",
    "        action = env.action_space.sample()\n",
    "        state, reward, done, _ = env.step(action)\n",
    "        score += reward\n",
    "        if done:\n",
    "            env.close()\n",
    "            print(\"Your Score at end of game is: \", score)\n",
    "            break\n",
    "random_play()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "Sr52nmcpGada"
   },
   "outputs": [],
   "source": [
    "## Step 4:Preprocessing Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(preprocess_frame(env.reset(), (30, -4, -12, 4), 84), cmap=\"gray\")\n",
    "plt.title('Pre Processed image')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "colab_type": "text",
    "id": "mJMc3HA8Gade"
   },
   "outputs": [],
   "source": [
    "## Step 5: Stacking Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stack_frames(frames, state, is_new=False):\n",
    "    frame = preprocess_frame(state, (30, -4, -12, 4), 84)\n",
    "    frames = stack_frame(frames, frame, is_new)\n",
    "\n",
    "    return frames\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 6: Creating our Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SHAPE = (4, 84, 84)\n",
    "ACTION_SIZE = env.action_space.n\n",
    "SEED = 0\n",
    "GAMMA = 0.99           # discount factor\n",
    "ALPHA= 0.0001          # Actor learning rate\n",
    "BETA = 0.0001          # Critic learning rate\n",
    "TAU = 0.95\n",
    "BATCH_SIZE = 32\n",
    "PPO_EPOCH = 5\n",
    "CLIP_PARAM = 0.2\n",
    "UPDATE_EVERY = 1000     # how often to update the network \n",
    "\n",
    "\n",
    "agent = PPOAgent(INPUT_SHAPE, ACTION_SIZE, SEED, device, GAMMA, ALPHA, BETA, TAU, UPDATE_EVERY, BATCH_SIZE, PPO_EPOCH, CLIP_PARAM, ActorCnn, CriticCnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 7: Watching untrained agent play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# watch an untrained agent\n",
    "state = stack_frames(None, env.reset(), True) \n",
    "for j in range(200):\n",
    "    env.render()\n",
    "    action, _, _ = agent.act(state)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        break \n",
    "        \n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 8: Loading Agent\n",
    "Uncomment line to load a pretrained agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_epoch = 0\n",
    "scores = []\n",
    "scores_window = deque(maxlen=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 9: Train the Agent with DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=1000):\n",
    "    \"\"\"\n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "    \"\"\"\n",
    "    for i_episode in range(start_epoch + 1, n_episodes+1):\n",
    "        state = stack_frames(None, env.reset(), True)\n",
    "        score = 0\n",
    "        while True:\n",
    "            action, log_prob, value = agent.act(state)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            score += reward\n",
    "            next_state = stack_frames(state, next_state, False)\n",
    "            agent.step(state, action, value, log_prob, reward, done, next_state)\n",
    "            if done:\n",
    "                break\n",
    "            else:\n",
    "                state = next_state\n",
    "        scores_window.append(score)       # save most recent score\n",
    "        scores.append(score)              # save most recent score\n",
    "        \n",
    "        clear_output(True)\n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111)\n",
    "        plt.plot(np.arange(len(scores)), scores)\n",
    "        plt.ylabel('Score')\n",
    "        plt.xlabel('Episode #')\n",
    "        plt.show()\n",
    "        print('\\rEpisode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_window)), end=\"\")\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = train(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(len(scores)), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 10: Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = 0\n",
    "state = stack_frames(None, env.reset(), True)\n",
    "while True:\n",
    "    env.render()\n",
    "    action, _, _ = agent.act(state)\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    score += reward\n",
    "    state = stack_frames(state, next_state, False)\n",
    "    if done:\n",
    "        print(\"You Final score is:\", score)\n",
    "        break \n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "space_invader.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}