LunarLanderAssignment HA 5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting box2d\n",
      "  Using cached Box2D-2.3.2.tar.gz (427 kB)\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Building wheels for collected packages: box2d\n",
      "  Building wheel for box2d (setup.py): started\n",
      "  Building wheel for box2d (setup.py): finished with status 'error'\n",
      "  Running setup.py clean for box2d\n",
      "Failed to build box2d\n",
      "Installing collected packages: box2d\n",
      "  Running setup.py install for box2d: started\n",
      "  Running setup.py install for box2d: finished with status 'error'\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  error: subprocess-exited-with-error\n",
      "  \n",
      "  × python setup.py bdist_wheel did not run successfully.\n",
      "  │ exit code: 1\n",
      "  ╰─> [28 lines of output]\n",
      "      Using setuptools (version 67.6.0).\n",
      "      running bdist_wheel\n",
      "      running build\n",
      "      running build_py\n",
      "      creating build\n",
      "      creating build\\lib.win-amd64-cpython-310\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      running build_ext\n",
      "      building 'Box2D._Box2D' extension\n",
      "      swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "      swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "      Box2D\\Common\\b2Math.h(67) : Warning 302: Identifier 'b2Vec2' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(47) : Warning 302: %extend definition of 'b2Vec2'.\n",
      "      Box2D\\Common\\b2Math.h(158) : Warning 302: Identifier 'b2Vec3' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(168) : Warning 302: %extend definition of 'b2Vec3'.\n",
      "      Box2D\\Common\\b2Math.h(197) : Warning 302: Identifier 'b2Mat22' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(301) : Warning 302: %extend definition of 'b2Mat22'.\n",
      "      Box2D\\Common\\b2Math.h(271) : Warning 302: Identifier 'b2Mat33' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(372) : Warning 302: %extend definition of 'b2Mat33'.\n",
      "      Box2D\\Collision\\b2DynamicTree.h(44) : Warning 312: Nested union not currently supported (ignored).\n",
      "      Box2D\\Common\\b2Settings.h(144) : Warning 506: Can't wrap varargs with keyword arguments enabled\n",
      "      Box2D\\Common\\b2Math.h(91) : Warning 509: Overloaded method b2Vec2::operator ()(int32) effectively ignored,\n",
      "      Box2D\\Common\\b2Math.h(85) : Warning 509: as it is shadowed by b2Vec2::operator ()(int32) const.\n",
      "      error: Microsoft Visual C++ 14.0 or greater is required. Get it with \"Microsoft C++ Build Tools\": https://visualstudio.microsoft.com/visual-cpp-build-tools/\n",
      "      [end of output]\n",
      "  \n",
      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "  ERROR: Failed building wheel for box2d\n",
      "  error: subprocess-exited-with-error\n",
      "  \n",
      "  × Running setup.py install for box2d did not run successfully.\n",
      "  │ exit code: 1\n",
      "  ╰─> [30 lines of output]\n",
      "      Using setuptools (version 67.6.0).\n",
      "      running install\n",
      "      c:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\setuptools\\command\\install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n",
      "        warnings.warn(\n",
      "      running build\n",
      "      running build_py\n",
      "      creating build\n",
      "      creating build\\lib.win-amd64-cpython-310\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      running build_ext\n",
      "      building 'Box2D._Box2D' extension\n",
      "      swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "      swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "      Box2D\\Common\\b2Math.h(67) : Warning 302: Identifier 'b2Vec2' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(47) : Warning 302: %extend definition of 'b2Vec2'.\n",
      "      Box2D\\Common\\b2Math.h(158) : Warning 302: Identifier 'b2Vec3' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(168) : Warning 302: %extend definition of 'b2Vec3'.\n",
      "      Box2D\\Common\\b2Math.h(197) : Warning 302: Identifier 'b2Mat22' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(301) : Warning 302: %extend definition of 'b2Mat22'.\n",
      "      Box2D\\Common\\b2Math.h(271) : Warning 302: Identifier 'b2Mat33' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(372) : Warning 302: %extend definition of 'b2Mat33'.\n",
      "      Box2D\\Collision\\b2DynamicTree.h(44) : Warning 312: Nested union not currently supported (ignored).\n",
      "      Box2D\\Common\\b2Settings.h(144) : Warning 506: Can't wrap varargs with keyword arguments enabled\n",
      "      Box2D\\Common\\b2Math.h(91) : Warning 509: Overloaded method b2Vec2::operator ()(int32) effectively ignored,\n",
      "      Box2D\\Common\\b2Math.h(85) : Warning 509: as it is shadowed by b2Vec2::operator ()(int32) const.\n",
      "      error: Microsoft Visual C++ 14.0 or greater is required. Get it with \"Microsoft C++ Build Tools\": https://visualstudio.microsoft.com/visual-cpp-build-tools/\n",
      "      [end of output]\n",
      "  \n",
      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "error: legacy-install-failure\n",
      "\n",
      "× Encountered error while trying to install package.\n",
      "╰─> box2d\n",
      "\n",
      "note: This is an issue with the package mentioned above, not pip.\n",
      "hint: See above for output from the failure.\n"
     ]
    }
   ],
   "source": [
    "pip install box2d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gymnasium[box2d] in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (0.27.1)\n",
      "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (1.22.4)\n",
      "Requirement already satisfied: gymnasium-notices>=0.0.1 in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (0.0.1)\n",
      "Requirement already satisfied: typing-extensions>=4.3.0 in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (4.5.0)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (2.2.1)\n",
      "Requirement already satisfied: jax-jumpy>=0.2.0 in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (0.2.0)\n",
      "Collecting pygame==2.1.3.dev8\n",
      "  Using cached pygame-2.1.3.dev8-cp310-cp310-win_amd64.whl (10.6 MB)\n",
      "Requirement already satisfied: swig==4.* in c:\\users\\giorg\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gymnasium[box2d]) (4.1.1)\n",
      "Collecting box2d-py==2.3.5\n",
      "  Using cached box2d-py-2.3.5.tar.gz (374 kB)\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Building wheels for collected packages: box2d-py\n",
      "  Building wheel for box2d-py (setup.py): started\n",
      "  Building wheel for box2d-py (setup.py): finished with status 'error'\n",
      "  Running setup.py clean for box2d-py\n",
      "Failed to build box2d-py\n",
      "Installing collected packages: box2d-py, pygame\n",
      "  Running setup.py install for box2d-py: started\n",
      "  Running setup.py install for box2d-py: finished with status 'error'\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  error: subprocess-exited-with-error\n",
      "  \n",
      "  × python setup.py bdist_wheel did not run successfully.\n",
      "  │ exit code: 1\n",
      "  ╰─> [28 lines of output]\n",
      "      Using setuptools (version 67.6.0).\n",
      "      running bdist_wheel\n",
      "      running build\n",
      "      running build_py\n",
      "      creating build\n",
      "      creating build\\lib.win-amd64-cpython-310\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      running build_ext\n",
      "      building 'Box2D._Box2D' extension\n",
      "      swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "      swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "      Box2D\\Common\\b2Math.h(67) : Warning 302: Identifier 'b2Vec2' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(47) : Warning 302: %extend definition of 'b2Vec2'.\n",
      "      Box2D\\Common\\b2Math.h(158) : Warning 302: Identifier 'b2Vec3' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(168) : Warning 302: %extend definition of 'b2Vec3'.\n",
      "      Box2D\\Common\\b2Math.h(197) : Warning 302: Identifier 'b2Mat22' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(301) : Warning 302: %extend definition of 'b2Mat22'.\n",
      "      Box2D\\Common\\b2Math.h(271) : Warning 302: Identifier 'b2Mat33' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(372) : Warning 302: %extend definition of 'b2Mat33'.\n",
      "      Box2D\\Collision\\b2DynamicTree.h(44) : Warning 312: Nested union not currently supported (ignored).\n",
      "      Box2D\\Common\\b2Settings.h(144) : Warning 506: Can't wrap varargs with keyword arguments enabled\n",
      "      Box2D\\Common\\b2Math.h(91) : Warning 509: Overloaded method b2Vec2::operator ()(int32) effectively ignored,\n",
      "      Box2D\\Common\\b2Math.h(85) : Warning 509: as it is shadowed by b2Vec2::operator ()(int32) const.\n",
      "      error: Microsoft Visual C++ 14.0 or greater is required. Get it with \"Microsoft C++ Build Tools\": https://visualstudio.microsoft.com/visual-cpp-build-tools/\n",
      "      [end of output]\n",
      "  \n",
      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "  ERROR: Failed building wheel for box2d-py\n",
      "  error: subprocess-exited-with-error\n",
      "  \n",
      "  × Running setup.py install for box2d-py did not run successfully.\n",
      "  │ exit code: 1\n",
      "  ╰─> [30 lines of output]\n",
      "      Using setuptools (version 67.6.0).\n",
      "      running install\n",
      "      c:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\setuptools\\command\\install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.\n",
      "        warnings.warn(\n",
      "      running build\n",
      "      running build_py\n",
      "      creating build\n",
      "      creating build\\lib.win-amd64-cpython-310\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\Box2D.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      copying library\\Box2D\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\n",
      "      creating build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      copying library\\Box2D\\b2\\__init__.py -> build\\lib.win-amd64-cpython-310\\Box2D\\b2\n",
      "      running build_ext\n",
      "      building 'Box2D._Box2D' extension\n",
      "      swigging Box2D\\Box2D.i to Box2D\\Box2D_wrap.cpp\n",
      "      swig.exe -python -c++ -IBox2D -small -O -includeall -ignoremissing -w201 -globals b2Globals -outdir library\\Box2D -keyword -w511 -D_SWIG_KWARGS -o Box2D\\Box2D_wrap.cpp Box2D\\Box2D.i\n",
      "      Box2D\\Common\\b2Math.h(67) : Warning 302: Identifier 'b2Vec2' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(47) : Warning 302: %extend definition of 'b2Vec2'.\n",
      "      Box2D\\Common\\b2Math.h(158) : Warning 302: Identifier 'b2Vec3' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(168) : Warning 302: %extend definition of 'b2Vec3'.\n",
      "      Box2D\\Common\\b2Math.h(197) : Warning 302: Identifier 'b2Mat22' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(301) : Warning 302: %extend definition of 'b2Mat22'.\n",
      "      Box2D\\Common\\b2Math.h(271) : Warning 302: Identifier 'b2Mat33' redefined by %extend (ignored),\n",
      "      Box2D\\Box2D_math.i(372) : Warning 302: %extend definition of 'b2Mat33'.\n",
      "      Box2D\\Collision\\b2DynamicTree.h(44) : Warning 312: Nested union not currently supported (ignored).\n",
      "      Box2D\\Common\\b2Settings.h(144) : Warning 506: Can't wrap varargs with keyword arguments enabled\n",
      "      Box2D\\Common\\b2Math.h(91) : Warning 509: Overloaded method b2Vec2::operator ()(int32) effectively ignored,\n",
      "      Box2D\\Common\\b2Math.h(85) : Warning 509: as it is shadowed by b2Vec2::operator ()(int32) const.\n",
      "      error: Microsoft Visual C++ 14.0 or greater is required. Get it with \"Microsoft C++ Build Tools\": https://visualstudio.microsoft.com/visual-cpp-build-tools/\n",
      "      [end of output]\n",
      "  \n",
      "  note: This error originates from a subprocess, and is likely not a problem with pip.\n",
      "error: legacy-install-failure\n",
      "\n",
      "× Encountered error while trying to install package.\n",
      "╰─> box2d-py\n",
      "\n",
      "note: This is an issue with the package mentioned above, not pip.\n",
      "hint: See above for output from the failure.\n"
     ]
    }
   ],
   "source": [
    "pip install gymnasium[box2d]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lunar Lander with REINFORCE\n",
    "### Christian Igel, 2023\n",
    "\n",
    "If you have suggestions for improvements, [let me know](mailto:igel@diku.dk).\n",
    "\n",
    "Imports:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "\n",
    "from tqdm.notebook import tqdm, trange  # Progress bar\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need [the `gymnasium` package](https://gymnasium.farama.org/).\n",
    "From this package, we create the Cart-Pole game environment:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "DependencyNotInstalled",
     "evalue": "Box2D is not installed, run `pip install gymnasium[box2d]`",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gymnasium\\envs\\box2d\\bipedal_walker.py:15\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 15\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mBox2D\u001b[39;00m\n\u001b[0;32m     16\u001b[0m     \u001b[39mfrom\u001b[39;00m \u001b[39mBox2D\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mb2\u001b[39;00m \u001b[39mimport\u001b[39;00m (\n\u001b[0;32m     17\u001b[0m         circleShape,\n\u001b[0;32m     18\u001b[0m         contactListener,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     22\u001b[0m         revoluteJointDef,\n\u001b[0;32m     23\u001b[0m     )\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'Box2D'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mDependencyNotInstalled\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\giorg\\Desktop\\CODES\\LunarLanderAssignment.ipynb Cell 6\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/giorg/Desktop/CODES/LunarLanderAssignment.ipynb#W5sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m env_visual \u001b[39m=\u001b[39m gym\u001b[39m.\u001b[39;49mmake(\u001b[39m'\u001b[39;49m\u001b[39mLunarLander-v2\u001b[39;49m\u001b[39m'\u001b[39;49m, render_mode\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mhuman\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/giorg/Desktop/CODES/LunarLanderAssignment.ipynb#W5sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m action_size \u001b[39m=\u001b[39m \u001b[39m4\u001b[39m\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/giorg/Desktop/CODES/LunarLanderAssignment.ipynb#W5sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m state_size \u001b[39m=\u001b[39m \u001b[39m8\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gymnasium\\envs\\registration.py:604\u001b[0m, in \u001b[0;36mmake\u001b[1;34m(id, max_episode_steps, autoreset, apply_api_compatibility, disable_env_checker, **kwargs)\u001b[0m\n\u001b[0;32m    601\u001b[0m     env_creator \u001b[39m=\u001b[39m spec_\u001b[39m.\u001b[39mentry_point\n\u001b[0;32m    602\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    603\u001b[0m     \u001b[39m# Assume it's a string\u001b[39;00m\n\u001b[1;32m--> 604\u001b[0m     env_creator \u001b[39m=\u001b[39m load(spec_\u001b[39m.\u001b[39;49mentry_point)\n\u001b[0;32m    606\u001b[0m render_modes \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m    607\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(env_creator, \u001b[39m\"\u001b[39m\u001b[39mmetadata\u001b[39m\u001b[39m\"\u001b[39m):\n",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gymnasium\\envs\\registration.py:58\u001b[0m, in \u001b[0;36mload\u001b[1;34m(name)\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[39m\"\"\"Loads an environment with name and returns an environment creation function.\u001b[39;00m\n\u001b[0;32m     50\u001b[0m \n\u001b[0;32m     51\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[39m    Calls the environment constructor\u001b[39;00m\n\u001b[0;32m     56\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m     57\u001b[0m mod_name, attr_name \u001b[39m=\u001b[39m name\u001b[39m.\u001b[39msplit(\u001b[39m\"\u001b[39m\u001b[39m:\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> 58\u001b[0m mod \u001b[39m=\u001b[39m importlib\u001b[39m.\u001b[39;49mimport_module(mod_name)\n\u001b[0;32m     59\u001b[0m fn \u001b[39m=\u001b[39m \u001b[39mgetattr\u001b[39m(mod, attr_name)\n\u001b[0;32m     60\u001b[0m \u001b[39mreturn\u001b[39;00m fn\n",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\importlib\\__init__.py:126\u001b[0m, in \u001b[0;36mimport_module\u001b[1;34m(name, package)\u001b[0m\n\u001b[0;32m    124\u001b[0m             \u001b[39mbreak\u001b[39;00m\n\u001b[0;32m    125\u001b[0m         level \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m--> 126\u001b[0m \u001b[39mreturn\u001b[39;00m _bootstrap\u001b[39m.\u001b[39;49m_gcd_import(name[level:], package, level)\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:1050\u001b[0m, in \u001b[0;36m_gcd_import\u001b[1;34m(name, package, level)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:1027\u001b[0m, in \u001b[0;36m_find_and_load\u001b[1;34m(name, import_)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:992\u001b[0m, in \u001b[0;36m_find_and_load_unlocked\u001b[1;34m(name, import_)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:241\u001b[0m, in \u001b[0;36m_call_with_frames_removed\u001b[1;34m(f, *args, **kwds)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:1050\u001b[0m, in \u001b[0;36m_gcd_import\u001b[1;34m(name, package, level)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:1027\u001b[0m, in \u001b[0;36m_find_and_load\u001b[1;34m(name, import_)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:1006\u001b[0m, in \u001b[0;36m_find_and_load_unlocked\u001b[1;34m(name, import_)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:688\u001b[0m, in \u001b[0;36m_load_unlocked\u001b[1;34m(spec)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap_external>:883\u001b[0m, in \u001b[0;36mexec_module\u001b[1;34m(self, module)\u001b[0m\n",
      "File \u001b[1;32m<frozen importlib._bootstrap>:241\u001b[0m, in \u001b[0;36m_call_with_frames_removed\u001b[1;34m(f, *args, **kwds)\u001b[0m\n",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gymnasium\\envs\\box2d\\__init__.py:1\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgymnasium\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39menvs\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mbox2d\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mbipedal_walker\u001b[39;00m \u001b[39mimport\u001b[39;00m BipedalWalker, BipedalWalkerHardcore\n\u001b[0;32m      2\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgymnasium\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39menvs\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mbox2d\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcar_racing\u001b[39;00m \u001b[39mimport\u001b[39;00m CarRacing\n\u001b[0;32m      3\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mgymnasium\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39menvs\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mbox2d\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mlunar_lander\u001b[39;00m \u001b[39mimport\u001b[39;00m LunarLander, LunarLanderContinuous\n",
      "File \u001b[1;32mc:\\Users\\giorg\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\gymnasium\\envs\\box2d\\bipedal_walker.py:25\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[39mfrom\u001b[39;00m \u001b[39mBox2D\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mb2\u001b[39;00m \u001b[39mimport\u001b[39;00m (\n\u001b[0;32m     17\u001b[0m         circleShape,\n\u001b[0;32m     18\u001b[0m         contactListener,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     22\u001b[0m         revoluteJointDef,\n\u001b[0;32m     23\u001b[0m     )\n\u001b[0;32m     24\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mImportError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m---> 25\u001b[0m     \u001b[39mraise\u001b[39;00m DependencyNotInstalled(\n\u001b[0;32m     26\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mBox2D is not installed, run `pip install gymnasium[box2d]`\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m     27\u001b[0m     ) \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n\u001b[0;32m     30\u001b[0m \u001b[39mif\u001b[39;00m TYPE_CHECKING:\n\u001b[0;32m     31\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mpygame\u001b[39;00m\n",
      "\u001b[1;31mDependencyNotInstalled\u001b[0m: Box2D is not installed, run `pip install gymnasium[box2d]`"
     ]
    }
   ],
   "source": [
    "env_visual = gym.make('LunarLander-v2', render_mode=\"human\")\n",
    "action_size = 4\n",
    "state_size = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's just test the environment first:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_episodes = 5\n",
    "for _ in range(test_episodes):\n",
    "    R = 0\n",
    "    state, _ = env_visual.reset()  # Environment starts in a random state, cart and pole are moving\n",
    "    print(\"initial state:\", state)\n",
    "    while True:  # Environment sets \"truncated\" to true after 500 steps \n",
    "        # Uncomment the line below to watch the simulation\n",
    "        env_visual.render()\n",
    "        state, reward, terminated, truncated, _ = env_visual.step(env_visual.action_space.sample()) #  Take a random action\n",
    "        R += reward  # Accumulate reward\n",
    "        if terminated or truncated:\n",
    "            print(\"return: \", R)\n",
    "            env_visual.reset()\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REINFORCE\n",
    "\n",
    "Let's define a policy class for a simple softmax policy for real-valued feature vectors and discrete actions.\n",
    "The preference for an action is just a linear function of the input features.\n",
    "It is not trivial that this simple policy is powerful enough to solve the tasks without addional processing of the input features. However, it is indeed possible to get reasonable policies in this setting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Softmax_policy:\n",
    "    def __init__(self, no_actions, no_features):\n",
    "        \"\"\"\n",
    "        Initialize softmax policy for discrete actions\n",
    "        :param no_actions: number of actions\n",
    "        :param no_features: dimensionality of feature vector representing a state\n",
    "        \"\"\"        \n",
    "        self.no_actions = no_actions\n",
    "        self.no_features = no_features\n",
    "\n",
    "        # Initialize policy parameters to zero\n",
    "        self.theta = np.zeros([no_actions, no_features])\n",
    "        \n",
    "    def pi(self, s):\n",
    "        \"\"\"\n",
    "        Compute action probabilities in a given state\n",
    "        :param s: state feature vector\n",
    "        :return: an array of action probabilities\n",
    "        \"\"\"\n",
    "        # Compute action preferences for the given feature vector\n",
    "        preferences = self.theta.dot(s)\n",
    "        # Convert overflows to underflows\n",
    "        preferences = preferences - preferences.max()\n",
    "        # Convert the preferences into probabilities\n",
    "        exp_prefs = np.exp(preferences)\n",
    "        return exp_prefs / np.sum(exp_prefs)\n",
    "    \n",
    "    def inc(self, delta):\n",
    "        \"\"\"\n",
    "        Change the parameters by addition, e.g. for initialization or parameter updates \n",
    "        :param delta: values to be added to parameters\n",
    "        \"\"\"\n",
    "        self.theta += delta\n",
    "\n",
    "    def sample_action(self, s):\n",
    "        \"\"\"\n",
    "        Sample an action in a given state\n",
    "        :param s: state feature vector\n",
    "        :return: action\n",
    "        \"\"\"\n",
    "        return np.random.choice(self.no_actions, p=self.pi(s))\n",
    "\n",
    "    def gradient_log_pi(self, s, a):\n",
    "        \"\"\"\n",
    "        Computes the gradient of the logarithm of the policy\n",
    "        :param s: state feature vector\n",
    "        :param a: action\n",
    "        :return: gradient of the logarithm of the policy\n",
    "        \"\"\"\n",
    "        return 0\n",
    "\n",
    "    def gradient_log_pi_test(self, s, a, eps=0.1):\n",
    "        \"\"\"\n",
    "        Numerically approximates the gradient of the logarithm of the policy\n",
    "        :param s: state feature vector\n",
    "        :param a: action\n",
    "        :return: approximate gradient of the logarithm of the policy\n",
    "        \"\"\"\n",
    "        theta_correct = np.copy(self.theta)\n",
    "        log_pi = np.log(self.pi(s)[a])\n",
    "        d = np.zeros([self.no_actions, self.no_features])\n",
    "        for i in range(self.no_actions):\n",
    "            for j in range(self.no_features):\n",
    "                self.theta[i,j] += eps\n",
    "                log_pi_eps = np.log(self.pi(s)[a])\n",
    "                d[i,j] = (log_pi_eps - log_pi) / eps\n",
    "                self.theta = np.copy(theta_correct)\n",
    "        return d\n",
    "  \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify gradient implementation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "s = env.reset()[0]\n",
    "pi = Softmax_policy(action_size, state_size)\n",
    "tolerance = 0.001  # Absolute tolerance for difference in each gradient component\n",
    "epsilon = 0.0001\n",
    "for _ in range(10):\n",
    "    pi.inc(10.*np.random.rand(action_size, state_size))\n",
    "    for a in range(action_size):\n",
    "        if not np.isclose(pi.gradient_log_pi(s, a), pi.gradient_log_pi_test(s, a, epsilon), atol=tolerance).all():\n",
    "            print(\"derivative test for action\", a)\n",
    "            print(pi.gradient_log_pi(s, 0))\n",
    "            print(pi.gradient_log_pi_test(s, 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do the learning:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.00005  # Learning rate\n",
    "\n",
    "no_episodes = 20000  # Number of episodes\n",
    "total_reward_list = []  # Returns for the individual episodes\n",
    "pi = Softmax_policy(action_size, state_size)  # Policy\n",
    "\n",
    "# Do the learning\n",
    "for e in trange(no_episodes):  #  Loop over episodes\n",
    "    R = []  # Store rewards r_1, ..., r_T\n",
    "    S = []  # Store actions a_0, ..., a_{T-1}\n",
    "    A = []  # Store states s_0, ..., s_{T-1}\n",
    "    state = env.reset()[0]  # Environment starts in a random state, cart and pole are moving\n",
    "    while True:  # Environment sets \"done\" to true after 200 steps \n",
    "        S.append(state)\n",
    "        \n",
    "        action = pi.sample_action(state)  # Take an action following pi\n",
    "        A.append(action)\n",
    "        \n",
    "        state, reward, terminated, truncated, _ = env.step(action)  # Observe reward and new state\n",
    "        R.append(reward)\n",
    "                \n",
    "        if terminated or truncated:  # Failed or succeeded?\n",
    "            break\n",
    "            \n",
    "    R = np.array(R)\n",
    "    total_reward_list.append((e, R.sum()))\n",
    "    \n",
    "    for t in range(R.size):\n",
    "        R_t = R[t:].sum()  # Accumulated future reward\n",
    "        Delta = alpha * R_t * pi.gradient_log_pi(S[t], A[t])  # REINFORCE update\n",
    "        pi.inc(Delta)  # Apply update\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot learning process:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Moving average for smoothing plot\n",
    "def running_mean(x, N):\n",
    "    cumsum = np.cumsum(np.insert(x, 0, x[0]*np.ones(N)))\n",
    "    return (cumsum[N:] - cumsum[:-N]) / N\n",
    "\n",
    "eps, rews = np.array(total_reward_list).T\n",
    "smoothed_rews = running_mean(rews, 10)\n",
    "plt.plot(eps, smoothed_rews)\n",
    "plt.plot(eps, rews, color='grey', alpha=0.3)\n",
    "plt.xlabel('Episode')\n",
    "plt.ylabel('Accumulated Reward');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize policy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = env_visual.reset()[0]  # Environment starts in a random state, cart and pole are moving\n",
    "R = 0\n",
    "while True:  # Environment sets \"truncated\" to true after 500 steps \n",
    "        env_visual.render()\n",
    "        state, reward, terminated, truncated, _ = env_visual.step( pi.sample_action(state) ) #  Take a  action\n",
    "        R += reward  # Accumulate reward\n",
    "        if terminated or truncated:\n",
    "            print(\"return: \", R)\n",
    "            break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}