hw4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "hw4.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "sqJgIUtjlZMC",
        "0VpS2T7_lj-z",
        "mDwvKHJvmMD-"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqJgIUtjlZMC",
        "colab_type": "text"
      },
      "source": [
        "# Importing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AwENq7aUUIte",
        "colab_type": "text"
      },
      "source": [
        "If you are a Mac user:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mLo-jSpoUKfq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install x11-utils"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NcRIRpZ1lVh2",
        "colab_type": "text"
      },
      "source": [
        "Whether you are a Windows or Mac user, you will need the following code blocks:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GOulpQDhTtiC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!py -m pip install gym\n",
        "\n",
        "!py -m apt-get install python-opengl -y\n",
        "!py -m apt install xvfb -y\n",
        "\n",
        "# Special gym environment\n",
        "!py -m pip install gym[atari]\n",
        "\n",
        "# For rendering environment, you can use pyvirtualdisplay.\n",
        "!py -m pip install pyvirtualdisplay\n",
        "!py -m pip install piglet\n",
        "\n",
        "# To activate virtual display \n",
        "# need to run a script once for training an agent as follows\n",
        "from pyvirtualdisplay import Display\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()\n",
        "\n",
        "\n",
        "'''\n",
        "# This code creates a virtual display to draw game images on. \n",
        "# If you are running locally, just ignore it\n",
        "import os\n",
        "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\"))==0:\n",
        "    !bash ../xvfb start\n",
        "    %env DISPLAY=:1\n",
        "'''"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: gym in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (0.18.0)\n",
            "Requirement already satisfied: scipy in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym) (1.5.2)\n",
            "Requirement already satisfied: numpy>=1.10.4 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym) (1.18.2)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym) (1.5.0)\n",
            "Requirement already satisfied: Pillow<=7.2.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym) (7.2.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym) (1.6.0)\n",
            "Requirement already satisfied: future in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from pyglet<=1.5.0,>=1.4.0->gym) (0.18.2)\n",
            "WARNING: You are using pip version 20.1.1; however, version 21.0.1 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User1\\AppData\\Local\\Programs\\Python\\Python38-32\\python.exe -m pip install --upgrade pip' command.\n",
            "'apt-get' is not recognized as an internal or external command,\n",
            "operable program or batch file.\n",
            "'apt' is not recognized as an internal or external command,\n",
            "operable program or batch file.\n",
            "Requirement already satisfied: gym[atari] in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (0.18.0)\n",
            "Requirement already satisfied: scipy in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (1.5.2)\n",
            "Requirement already satisfied: numpy>=1.10.4 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (1.18.2)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (1.5.0)\n",
            "Requirement already satisfied: Pillow<=7.2.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (7.2.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (1.6.0)\n",
            "Requirement already satisfied: atari_py~=0.2.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (0.2.6)\n",
            "Requirement already satisfied: opencv-python>=3. in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from gym[atari]) (4.5.1.48)\n",
            "Requirement already satisfied: future in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari]) (0.18.2)\n",
            "Requirement already satisfied: six in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from atari_py~=0.2.0->gym[atari]) (1.14.0)\n",
            "WARNING: You are using pip version 20.1.1; however, version 21.0.1 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User1\\AppData\\Local\\Programs\\Python\\Python38-32\\python.exe -m pip install --upgrade pip' command.\n",
            "Requirement already satisfied: pyvirtualdisplay in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (2.1)\n",
            "Requirement already satisfied: EasyProcess in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from pyvirtualdisplay) (0.3)\n",
            "WARNING: You are using pip version 20.1.1; however, version 21.0.1 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User1\\AppData\\Local\\Programs\\Python\\Python38-32\\python.exe -m pip install --upgrade pip' command.\n",
            "Requirement already satisfied: piglet in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (1.0.0)\n",
            "Requirement already satisfied: piglet-templates in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from piglet) (1.1.0)\n",
            "Requirement already satisfied: markupsafe in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from piglet-templates->piglet) (1.1.1)\n",
            "Requirement already satisfied: attrs in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from piglet-templates->piglet) (19.3.0)\n",
            "Requirement already satisfied: Parsley in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from piglet-templates->piglet) (1.3)\n",
            "Requirement already satisfied: astunparse in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from piglet-templates->piglet) (1.6.3)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from astunparse->piglet-templates->piglet) (0.36.2)\n",
            "Requirement already satisfied: six<2.0,>=1.6.1 in c:\\users\\user1\\appdata\\local\\programs\\python\\python38-32\\lib\\site-packages (from astunparse->piglet-templates->piglet) (1.14.0)\n",
            "WARNING: You are using pip version 20.1.1; however, version 21.0.1 is available.\n",
            "You should consider upgrading via the 'C:\\Users\\User1\\AppData\\Local\\Programs\\Python\\Python38-32\\python.exe -m pip install --upgrade pip' command.\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "EasyProcessError",
          "evalue": "start error <EasyProcess cmd_param=['Xvfb', '-help'] cmd=['Xvfb', '-help'] oserror=[WinError 2] The system cannot find the file specified return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\easyprocess\\__init__.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    167\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 168\u001b[1;33m             self.popen = subprocess.Popen(\n\u001b[0m\u001b[0;32m    169\u001b[0m                 \u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstdout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstderr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcwd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcwd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)\u001b[0m\n\u001b[0;32m    853\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 854\u001b[1;33m             self._execute_child(args, executable, preexec_fn, close_fds,\n\u001b[0m\u001b[0;32m    855\u001b[0m                                 \u001b[0mpass_fds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcwd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[1;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session)\u001b[0m\n\u001b[0;32m   1306\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1307\u001b[1;33m                 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,\n\u001b[0m\u001b[0;32m   1308\u001b[0m                                          \u001b[1;31m# no special security\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 2] The system cannot find the file specified",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[1;31mEasyProcessError\u001b[0m                          Traceback (most recent call last)",
            "\u001b[1;32m<ipython-input-6-d2fe7a674ffa>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;31m# need to run a script once for training an agent as follows\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mpyvirtualdisplay\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDisplay\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m \u001b[0mdisplay\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvisible\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1400\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m900\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\pyvirtualdisplay\\display.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, backend, visible, size, color_depth, bgcolor, use_xauth, retries, extra_args, manage_global_env, **kwargs)\u001b[0m\n\u001b[0;32m     52\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"unknown backend: %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 54\u001b[1;33m         self._obj = cls(\n\u001b[0m\u001b[0;32m     55\u001b[0m             \u001b[0msize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m             \u001b[0mcolor_depth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolor_depth\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\pyvirtualdisplay\\xvfb.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, size, color_depth, bgcolor, use_xauth, fbdir, dpi, retries, extra_args, manage_global_env)\u001b[0m\n\u001b[0;32m     42\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dpi\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdpi\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 44\u001b[1;33m         AbstractDisplay.__init__(\n\u001b[0m\u001b[0;32m     45\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m             \u001b[0mPROGRAM\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\pyvirtualdisplay\\abstractdisplay.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, program, use_xauth, retries, extra_args, manage_global_env)\u001b[0m\n\u001b[0;32m     86\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_retries_current\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 88\u001b[1;33m         \u001b[0mhelptext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_helptext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprogram\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     89\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_displayfd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"-displayfd\"\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mhelptext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     90\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_has_displayfd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\pyvirtualdisplay\\util.py\u001b[0m in \u001b[0;36mget_helptext\u001b[1;34m(program)\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menable_stdout_log\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menable_stderr_log\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mhelptext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mhelptext\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\easyprocess\\__init__.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m         \"\"\"\n\u001b[1;32m--> 141\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    142\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_alive\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    143\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38-32\\lib\\site-packages\\easyprocess\\__init__.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    172\u001b[0m             \u001b[0mlog\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"OSError exception: %s\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moserror\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    173\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moserror\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moserror\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 174\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mEasyProcessError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"start error\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    175\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_started\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m         \u001b[0mlog\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"process was started (pid=%s)\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;31mEasyProcessError\u001b[0m: start error <EasyProcess cmd_param=['Xvfb', '-help'] cmd=['Xvfb', '-help'] oserror=[WinError 2] The system cannot find the file specified return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3V2x5ptVKt6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#\n",
        "# Import libraries\n",
        "#\n",
        "import gym\n",
        "from gym import logger as gymlogger\n",
        "from gym.wrappers import Monitor\n",
        "gymlogger.set_level(40) # error only\n",
        "#import tensorflow as tf\n",
        "import numpy as np\n",
        "import random\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import math\n",
        "import glob\n",
        "import io\n",
        "import base64\n",
        "from IPython.display import HTML\n",
        "\n",
        "from IPython import display as ipythondisplay\n",
        "\n",
        "\"\"\"\n",
        "Utility functions to enable video recording of gym environment and displaying it\n",
        "To enable video, just do \"env = wrap_env(env)\"\"\n",
        "\"\"\"\n",
        "\n",
        "def show_video():\n",
        "  mp4list = glob.glob('video/*.mp4')\n",
        "  if len(mp4list) > 0:\n",
        "    mp4 = mp4list[0]\n",
        "    video = io.open(mp4, 'r+b').read()\n",
        "    encoded = base64.b64encode(video)\n",
        "    ipythondisplay.display(HTML(data='''<video alt=\"test\" autoplay \n",
        "                loop controls style=\"height: 400px;\">\n",
        "                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "             </video>'''.format(encoded.decode('ascii'))))\n",
        "  else: \n",
        "    print(\"Could not find video\")\n",
        "    \n",
        "\n",
        "def wrap_env(env):\n",
        "  env = Monitor(env, './video', force=True)\n",
        "  return env"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0VpS2T7_lj-z",
        "colab_type": "text"
      },
      "source": [
        "# Problem 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mjqyC-7gV7UC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Specify which environment to use.\n",
        "env = gym.make(\"Taxi-v3\").env\n",
        "env.reset()\n",
        "\n",
        "# Initialize table of Q-values\n",
        "# Hint: to access a specific value in the q_table, do this:\n",
        "#            q_table[state, action]\n",
        "q_table = np.zeros([env.observation_space.n, env.action_space.n])\n",
        "\n",
        "##########################################\n",
        "# Initialize RL Parameters\n",
        "##########################################\n",
        "\n",
        "\n",
        "# For plotting metrics\n",
        "cumulative_reward_each_episode = []\n",
        "epsilon_each_episode = []\n",
        "\n",
        "# For each episode\n",
        "maxNumEpisodes = 2000\n",
        "for i in range(maxNumEpisodes):\n",
        "\n",
        "  # Reset to initial conditions\n",
        "  state = env.reset()\n",
        "\t\n",
        "  # The variable 'cumulative_reward' will store the sum of the accumulated \n",
        "  # reward for an entire episode. Set this value to zero at the start of each \n",
        "  # episode.\n",
        "  cumulative_reward = 0\n",
        "  done = False\n",
        "\n",
        "  # While the episode is not finished\n",
        "  while not done:\n",
        "\n",
        "    ##########################################\n",
        "    # For every time step, using epsilon-greedy to choose between\n",
        "    # exploration and exploitation.\n",
        "    # Implement epsilon-greedy exploration.\n",
        "    # Hint: to return a random action, do this:\n",
        "    #           action = env.action_space.sample()\n",
        "    ##########################################\n",
        "\n",
        "\n",
        "\t\t# Take the action.\n",
        "\t\t# This moves the agent to a new state and earns a reward\n",
        "    next_state, reward, done, info = env.step(action)\n",
        "\n",
        "    # Add the reward just earned to the cumulative reward variable\n",
        "    cumulative_reward += reward\n",
        "\n",
        "\n",
        "    ##########################################\n",
        "    # Update your estimate of Q(s,a)\n",
        "    # Hint: to access a specific value in the q_table, do this:\n",
        "    #            q_table[state, action]\n",
        "    ##########################################\n",
        "\n",
        "\n",
        "    # Set your state variable to next_state for the next loop.\n",
        "    state = next_state\n",
        "\n",
        "    # If this episode is finished, take care of a few things:\n",
        "    if done:\n",
        "      # Save the cumulative reward from the previous episode to an array.\n",
        "      cumulative_reward_each_episode.append(cumulative_reward)\n",
        "\n",
        "      # Save the epsilon used in this episode.\n",
        "      epsilon_each_episode.append(epsilon)\n",
        "\n",
        "      ##########################################\n",
        "      # Decay epsilon,\n",
        "      # If you want to decay or change the value of epsilon at the end of\n",
        "      # each episode, do so here.\n",
        "      ##########################################\n",
        "\n",
        "\n",
        "  if i % 100 == 0:\n",
        "    print('Episode: {0}'.format(i)) \n",
        "\n",
        "print(\"Training finished.\\n\")\n",
        "\n",
        "# Plot the Cumulative Reward and Epsilon value through time.\n",
        "fsize = 15\n",
        "\n",
        "plt.plot(cumulative_reward_each_episode)\n",
        "plt.title('Cumulative Reward through Time', fontsize=fsize)\n",
        "plt.xlabel('Episode', fontsize=fsize)\n",
        "plt.ylabel('Cumulative Reward', fontsize=fsize)\n",
        "plt.show() \n",
        "\n",
        "plt.plot(epsilon_each_episode)\n",
        "plt.title('Exploration (epsilon) through Time', fontsize=fsize)\n",
        "plt.xlabel('Episode', fontsize=fsize)\n",
        "plt.ylabel('epsilon', fontsize=fsize)\n",
        "plt.show() \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGf-KDaVYk22",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Once training is finished, run through one more episode using exploitation only. \n",
        "\n",
        "done = False\n",
        "\n",
        "state = env.reset()\n",
        "\n",
        "while True:\n",
        "  env.render()\n",
        "\n",
        "  ##########################################\n",
        "  # Choose an action based on exploitation.\n",
        "  ##########################################\n",
        "  action = \n",
        "  state, reward, done, info = env.step(action)\n",
        "   \n",
        "  if done: \n",
        "    break;\n",
        "env.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mDwvKHJvmMD-",
        "colab_type": "text"
      },
      "source": [
        "# Problem 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJ94Jq6GzR-z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# I have included code to help you discretize the state space.\n",
        "# You DO NOT need to keep these specific bin ranges.\n",
        "# In fact, you may not want to keep these bin ranges.\n",
        "# I have provided this code to make it easier for you to modify and to save\n",
        "# you time.\n",
        "# You can alter or discretize the state space however you wish.\n",
        "# You do not need to keep all 4 state features if you have an argument for \n",
        "# eliminating features.\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "# Discretize input state to make Q-table and to reduce dimensionality\n",
        "def discretize(state):\n",
        "\n",
        "  #print ( state )\n",
        "\n",
        "  # First, set up arrays of the left bin edges\n",
        "  # Note: your bin sizes do not need to be of uniform width.\n",
        "  bins_pos = [-4.8, -1, 1, 4.8]\n",
        "  bins_vel = [-3.4*10**38, -5, 5, 3.4*10**38 ]\n",
        "  bins_w = [-3.4*10**38, -100, -50, -40, -30, -20, -10, -5, -2, 0, 2, 5, 10, 20, 30, 40, 50, 100, 3.4*10**38]\n",
        "\n",
        "  angle_range = 0.43*2\n",
        "  num_angle_bins = 12\n",
        "  angle_bin_stepsize = angle_range / num_angle_bins\n",
        "  bins_ang = [-0.43]\n",
        "  for i in range(1, num_angle_bins):\n",
        "    bins_ang.append( bins_ang[i-1]+ angle_bin_stepsize )\n",
        "\n",
        "  cart_position_bin = pd.cut([state[0]], bins=bins_pos, include_lowest=True)\n",
        "  cart_velocity_bin = pd.cut([state[1]], bins=bins_vel, include_lowest=True)\n",
        "  pole_angle_bin    = pd.cut([state[2]], bins=bins_ang, include_lowest=True)\n",
        "  angle_rate_bin    = pd.cut([state[3]], bins=bins_w  , include_lowest=True)\n",
        "\n",
        "  # To verify the order of the state variables:\n",
        "  #   https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py\n",
        "\n",
        "  return [cart_position_bin[0].left, cart_velocity_bin[0].left, pole_angle_bin[0].left, angle_rate_bin[0].left]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvFkLsSeFQCN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Simple code to test your discretization.\n",
        "\n",
        "# Specify which environment to use.\n",
        "env = gym.make('CartPole-v0')\n",
        "state = env.reset()\n",
        "\n",
        "action = env.action_space.sample() # Explore action space\n",
        "state, reward, done, info = env.step(action)\n",
        "print ( 'Continuous state: ')\n",
        "print ( state )\n",
        "\n",
        "discretized_state = discretize(state)\n",
        "print ( 'Discretized state: ')\n",
        "print ( discretized_state )\n",
        "\n",
        "env.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uiwbIE02mQeB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Specify which environment to use.\n",
        "env = gym.make('CartPole-v0')\n",
        "#env = wrap_env(env) \n",
        "state = env.reset()\n",
        "\n",
        "##########################################\n",
        "# Initialize your Q-values.\n",
        "# Note: you may use whichever data structure you wish.\n",
        "#       I used a dictionary, but a list works, too.\n",
        "##########################################\n",
        "\n",
        "\n",
        "\n",
        "##########################################\n",
        "# Initialize RL Parameters\n",
        "##########################################\n",
        "\n",
        "\n",
        "\n",
        "# For plotting metrics\n",
        "cumulative_reward_each_episode = []\n",
        "epsilon_each_episode = []\n",
        "\n",
        "\n",
        "# To start off wish debugging your code, use 1 episode. Increase this once\n",
        "# your code starts to work.\n",
        "maxNumEpisodes = 1\n",
        "\n",
        "# For each episode\n",
        "for i in range(maxNumEpisodes):\n",
        "\n",
        "  # Reset to initial conditions\n",
        "  state = env.reset()\n",
        "\n",
        "  ##########################################\n",
        "  # Discretize the state\n",
        "  # Note: you'll need to modify the discretize function\n",
        "  #       provided above.\n",
        "  ##########################################\n",
        "  state = discretize(state)\n",
        "\n",
        "\t# At the beginning of each episode, set the cumulative reward variable to zero.\n",
        "  cumulative_reward = 0\n",
        "  done = False\n",
        "\n",
        "  # For every step in the episode\n",
        "  while not done:\n",
        "    #env.render()\n",
        "\n",
        "    ##########################################\n",
        "    # For every time step, using epsilon-greedy to choose between\n",
        "    # exploration and exploitation.\n",
        "    # Implement epsilon-greedy exploration.\n",
        "    # Hint: to return a random action, do this:\n",
        "    #           action = env.action_space.sample()\n",
        "    ##########################################\n",
        "\n",
        "\n",
        "\n",
        "\t\t# Take the action.\n",
        "\t\t# This moves the agent to a new state and earns a reward\n",
        "    next_state, reward, done, info = env.step(action)\n",
        "\n",
        "    # Discrete the state\n",
        "    next_state = discretize(next_state)\n",
        "\n",
        "    # Add the reward just earned to the cumulative reward variable\n",
        "    cumulative_reward += reward\n",
        "\n",
        "\t\t##########################################\n",
        "    # Update your estimate of Q(s,a)\n",
        "    ##########################################\n",
        "\n",
        "\n",
        "\n",
        "    state = next_state\n",
        "\n",
        "    # If the episode is finished, do a few things.\n",
        "    if done:\n",
        "      # Save the cumulative reward from the previous episode to an array.\n",
        "      cumulative_reward_each_episode.append(cumulative_reward)\n",
        "\n",
        "      # Save the epsilon used in this episode.\n",
        "      epsilon_each_episode.append(epsilon)\n",
        "\n",
        "      ##########################################\n",
        "      # Decay epsilon.\n",
        "      # If you want to decay or change the value of epsilon at the end of\n",
        "      # each episode, do so here.\n",
        "      ##########################################\n",
        "\n",
        "\n",
        "  if i % 100 == 0:\n",
        "    print('Episode: {0}'.format(i)) \n",
        "\t\n",
        "env.close()\n",
        "#show_video()\n",
        "print(\"Training finished.\\n\")\n",
        "\n",
        "\n",
        "# Plot the Cumulative Reward and Epsilon value through time.\n",
        "fsize = 15\n",
        "\n",
        "plt.plot(cumulative_reward_each_episode)\n",
        "plt.title('Cumulative Reward through Time', fontsize=fsize)\n",
        "plt.xlabel('Episode', fontsize=fsize)\n",
        "plt.ylabel('Cumulative Reward', fontsize=fsize)\n",
        "plt.show() \n",
        "\n",
        "plt.plot(epsilon_each_episode)\n",
        "plt.title('Exploration (epsilon) through Time', fontsize=fsize)\n",
        "plt.xlabel('Episode', fontsize=fsize)\n",
        "plt.ylabel('epsilon', fontsize=fsize)\n",
        "plt.show() \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gIEBUKPg8w9m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Once training is finished, run through one more episode using exploitation only. \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}