vamp_learn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "vamp_learn.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "vq5QJi3hj-r4",
        "1klcBEb2kDHD",
        "uNxPbLl-jVc4",
        "Qe16CY46lwaf"
      ],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMROi/xJuUWabnCpOzwd5XL"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S6nuEj4TFyAZ"
      },
      "source": [
        "# Log\n",
        "\n",
        "* start time oct 24 - 21:10 - 01:06\n",
        "* read dqn paper nov 19 - 22:23 - 22:55\n",
        "* add fog nov 25 - 23:15 - 23:33\n",
        "* more on moves nov 26 - 8:00 - 9:00 | 9:49 - 12:21\n",
        "* reading pp0 paper nov 26 - 12:30 - 12:57\n",
        "* conceptualizing project search space/ reward systems nov 26 - 13:50 - 15:32\n",
        "* more conceptualizing nov 26 - 15:57 - 17:24\n",
        "* preparing for PPO using simple rewards 17:30 - 18:54 minus 30 min distracted\n",
        "* connect PPO with game nov 26 - 19:33 - 20:44\n",
        "* still connecting nov 26 - 21:13 - 22:12\n",
        "* finally starting to run nov 26 - 22:12 -"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GCg_FnJsjimQ"
      },
      "source": [
        "#Game"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlxHrzQqwe0_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "14245405-75fb-46ac-f33e-9a2a201be0a8"
      },
      "source": [
        "#python3\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class Map_thing():\n",
        "  id = 0\n",
        "  def __init__(self, t_char, mohs):\n",
        "    self.t_char = t_char\n",
        "    self.mohs = mohs\n",
        "    self.id = Map_thing.id\n",
        "    self.row = None\n",
        "    self.col = None\n",
        "    Map_thing.id += 1\n",
        "\n",
        "  def print_self(self):\n",
        "    return self.t_char\n",
        "\n",
        "  def get_mohs(self):\n",
        "    return self.mohs\n",
        "\n",
        "  def __repr__(self):\n",
        "    return self.t_char\n",
        "\n",
        "  def __eq__(self, other):\n",
        "    return other != None and self.t_char == other.t_char and self.id == other.id\n",
        "\n",
        "  def set_location(self, row, col):\n",
        "    self.row = row\n",
        "    self.col = col\n",
        "\n",
        "class Controller():\n",
        "  def __init__(self, move_func=None, responder=None, creature=None, looker=None):\n",
        "    self.move_func = move_func if move_func is not None else self._random\n",
        "    self.responder = responder if responder is not None else print\n",
        "    self.looker = looker if looker is not None else print\n",
        "    self.creature = creature\n",
        "\n",
        "  def _random(self):\n",
        "    x = np.random.randint(0, 4)\n",
        "    if x == 0:\n",
        "      return 'north'\n",
        "    elif x == 1:\n",
        "      return 'south'\n",
        "    elif x == 2:\n",
        "      return 'east'\n",
        "    elif x == 3:\n",
        "      return 'west'\n",
        "    return x\n",
        "\n",
        "  def look(self, look_info):\n",
        "    self.looker(look_info)\n",
        "\n",
        "  def canMove(self):\n",
        "    if self.creature != None:\n",
        "      return self.creature.canMove()\n",
        "    return False\n",
        "  \n",
        "  def getMove(self):\n",
        "    return self.move_func()\n",
        "  \n",
        "  def sendResponse(self, message):\n",
        "    self.responder(message)\n",
        "\n",
        "  def setCreature(self, creature):\n",
        "    self.creature = creature\n",
        "\n",
        "  def getCreature(self):\n",
        "    return self.creature\n",
        "\n",
        "class SimpleController(Controller):\n",
        "  def __init__(self, creature=None, prey=None, responder=None, looker=None):\n",
        "    super().__init__(creature=creature, move_func=self._getMove, responder=responder, looker=looker)\n",
        "    self.prey = prey\n",
        "    if (self.creature == None or self.prey == None):\n",
        "      self.move_func = super()._random\n",
        "    self.last = ['', '']\n",
        "\n",
        "  def _try_row(self, row_diff):\n",
        "    result = ''\n",
        "    if row_diff > 0:\n",
        "        result = 'north'\n",
        "    elif row_diff < 0:\n",
        "        result = 'south'\n",
        "    return result\n",
        "\n",
        "  def _try_col(self, col_diff):\n",
        "    result = ''\n",
        "    if col_diff > 0:\n",
        "        result = 'west'\n",
        "    elif col_diff < 0:\n",
        "        result = 'east'\n",
        "    return result\n",
        "\n",
        "  def sendResponse(self, message):\n",
        "    self.last[1] = message\n",
        "    super().sendResponse(message)\n",
        "\n",
        "  def _getMove(self, fudge=0):\n",
        "    row_diff = (self.creature.row - self.prey.row) + fudge\n",
        "    col_diff = self.creature.col - self.prey.col\n",
        "\n",
        "    result = ''\n",
        "    # print(self.creature.id, row_diff, col_diff)\n",
        "    if abs(row_diff) > abs(col_diff):\n",
        "      result = self._try_row(row_diff)\n",
        "      if result == '':\n",
        "        result = self._try_col(col_diff)\n",
        "    elif abs(row_diff) < abs(col_diff):\n",
        "      result = self._try_col(col_diff)\n",
        "      if result == '':\n",
        "        result = self._try_row(row_diff)\n",
        "    else:\n",
        "      result = self._getMove(np.random.rand() - .5) # make even chance\n",
        "\n",
        "    if result == '':\n",
        "      result = self._random()      \n",
        "\n",
        "    # super().sendResponse(str(result))\n",
        "    self.last = [result, '']\n",
        "    return result\n",
        "\n",
        "class ConsoleController(Controller):\n",
        "  def __init__(self, creature=None, responder=None):\n",
        "    super().__init__(creature=creature, move_func=self._getMove, responder=responder)\n",
        "  \n",
        "  def _getMove(self):\n",
        "    result = input(\"Your move >\")\n",
        "    return result\n",
        "\n",
        "class Creature(Map_thing):\n",
        "  def __init__(self, t_char, mohs, num_moves=1, wait_moves=0, drop_thing=None):\n",
        "    super().__init__(t_char, mohs)\n",
        "    self.num_moves = num_moves\n",
        "    self.max_wait = wait_moves\n",
        "    self.wait_count = self.max_wait\n",
        "    self.drop_thing = drop_thing\n",
        "\n",
        "  def canMove(self):\n",
        "    if self.wait_count <= 0:\n",
        "      self.wait_count = self.max_wait\n",
        "      return True\n",
        "    self.wait_count -= 1\n",
        "    return False\n",
        "\n",
        "  def getNumMoves(self):\n",
        "    return self.num_moves\n",
        "\n",
        "  def getMove(self):\n",
        "    return self.controller.getMove()\n",
        "\n",
        "  def sendResponse(self, message):\n",
        "    self.controller.sendResponse(message)\n",
        "\n",
        "  def get_drop(self):\n",
        "    return self.drop_thing\n",
        "\n",
        "class Map:\n",
        "  def __init__(self, width, height, density=20):\n",
        "    self.width = width\n",
        "    self.height = height\n",
        "    self.density = density\n",
        "\n",
        "    self._wall = Map_thing(\"█\",10)\n",
        "    self._space = Map_thing(' ',0)\n",
        "    self._fog = Map_thing('*', 10)\n",
        "\n",
        "    self._map = []\n",
        "    self.re_init_map(self.density)\n",
        "\n",
        "  def re_init_map(self, density=None):\n",
        "    den = density if density is not None else self.density\n",
        "    self._map = []\n",
        "    for row in range(self.height):\n",
        "      the_row = []\n",
        "      if row == 0 or row == self.height - 1:\n",
        "        the_row = [[self._wall] for _i in range(self.width)]\n",
        "      else:\n",
        "        the_row = [[self._wall]]\n",
        "        the_row.extend([[self._wall] if np.random.randint(100) <= den else [self._space] for i in range(self.height - 2)])\n",
        "        the_row.append([self._wall])\n",
        "      self._map.append(the_row)\n",
        "\n",
        "  def place(self, map_thing, to_row, to_col):\n",
        "    worked = True\n",
        "    if to_row <= 0 or to_row >= self.height:\n",
        "      worked = False\n",
        "    if to_col <= 0 or to_col >= self.width:\n",
        "      worked = False\n",
        "    if map_thing.mohs <= self._map[to_row][to_col][-1].mohs:\n",
        "      worked = False\n",
        "    if not worked:\n",
        "      return 'Bad Move'\n",
        "    self._map[to_row][to_col].append(map_thing)\n",
        "    map_thing.set_location(to_row, to_col)\n",
        "    return 'Moved'\n",
        "\n",
        "  def rand_place(self, map_thing, tries=10):\n",
        "    result = ''\n",
        "    for _x in range(tries):\n",
        "      result = self.place(map_thing, np.random.randint(1, self.height -1), np.random.randint(1, self.width - 1))\n",
        "      if result == 'Moved':\n",
        "        return result\n",
        "    return result\n",
        "\n",
        "\n",
        "  def move(self, creature, from_row, from_col, move):\n",
        "    to_row = from_row\n",
        "    to_col = from_col\n",
        "\n",
        "    if move == 'north' or move == 0:\n",
        "      to_row -= 1\n",
        "    elif move == 'south' or move == 1:\n",
        "      to_row += 1\n",
        "    elif move == 'west' or move == 2:\n",
        "      to_col -= 1\n",
        "    elif move == 'east' or move == 3:\n",
        "      to_col += 1\n",
        "    else:\n",
        "      return 'Unknown Move'\n",
        "    \n",
        "    place_result = self.place(creature, to_row, to_col)\n",
        "    if place_result == 'Moved':\n",
        "      self._map[from_row][from_col].remove(creature)\n",
        "      if creature.get_drop() != None:\n",
        "        self._map[from_row][from_col].append(creature.get_drop())\n",
        "    return place_result\n",
        "\n",
        "  def move_creature(self, map_creature, move):\n",
        "    if map_creature != None and map_creature.row != None and map_creature.col != None:\n",
        "      return self.move(map_creature, map_creature.row, map_creature.col, move)\n",
        "    return 'Location Unknown'\n",
        "\n",
        "  def print(self, map_thing=None, flashlight=5):\n",
        "    map_str = ''\n",
        "    for r in range(self.height):\n",
        "      if (map_thing != None and map_thing.row != None and map_thing.col != None) and (r < map_thing.row - flashlight or r > map_thing.row + flashlight):\n",
        "        map_str += self._fog.t_char * self.width\n",
        "      else:\n",
        "        for c in range(self.width):\n",
        "          if map_thing != None and (c < map_thing.col - flashlight or c > map_thing.col + flashlight):\n",
        "            map_str += self._fog.t_char\n",
        "          else:\n",
        "            map_str += self._map[r][c][-1].t_char\n",
        "      map_str += '\\n'\n",
        "    return map_str\n",
        "\n",
        "  def __len__(self):\n",
        "    return self.width * self.height\n",
        "  \n",
        "  def __repr__(self):\n",
        "    return self.print()\n",
        "\n",
        "class Game:\n",
        "  def __init__(self, size=(10, 10), level=0, player_controller=None, smart_controller=None):\n",
        "    self.level = level\n",
        "    self.map = Map(*size, density=20)\n",
        "    self.resets = 4\n",
        "    self.goal = Map_thing('@', 1)\n",
        "    self.player = Creature('#', 2)\n",
        "    self.player_controller = SimpleController(self.player, self.goal) if player_controller == None else player_controller\n",
        "    # self.player.setController(self.player_controller)\n",
        "    self.player_controller.setCreature(self.player)\n",
        "    self.controllers = [self.player_controller]\n",
        "    self.vampires = []\n",
        "    self.other_creatures = []\n",
        "    self.init_map()\n",
        "\n",
        "\n",
        "  def init_map(self):\n",
        "    self.turns_taken = 0\n",
        "    self.map.re_init_map()\n",
        "\n",
        "    vamp_len = len(self.vampires)\n",
        "\n",
        "    def no_print(message):\n",
        "      pass\n",
        "    \n",
        "    while len(self.vampires) < self.level:\n",
        "      vamp = Creature('V', 5, num_moves=2)\n",
        "      control = SimpleController(vamp, self.player, looker=no_print, responder=no_print)\n",
        "      self.controllers.append(control)\n",
        "      self.vampires.append(vamp)\n",
        "\n",
        "    wrappings = Map_thing('W', 7)\n",
        "    for level in range(vamp_len, self.level):\n",
        "      if self.level != 0 and level != 0:\n",
        "        if level % 5 == 0:\n",
        "          mummy = Creature('M', 8, drop_thing=wrappings)\n",
        "          control = SimpleController(mummy, self.player, looker=no_print, responder=no_print)\n",
        "          self.controllers.append(control)\n",
        "          self.other_creatures.append(mummy)\n",
        "        if level % 10 == 0:\n",
        "          ghost = Creature('G', 11, wait_moves=5)\n",
        "          control = SimpleController(ghost, self.player, looker=no_print, responder=no_print)\n",
        "          self.controllers.append(control)\n",
        "          self.other_creatures.append(ghost)\n",
        "\n",
        "    for v in self.vampires:\n",
        "      self.map.rand_place(v)\n",
        "    for c in self.other_creatures:\n",
        "      self.map.rand_place(c)  \n",
        "    \n",
        "    self.map.rand_place(self.player)\n",
        "    self.map.rand_place(self.goal)\n",
        "\n",
        "  def check_win(self):\n",
        "    win = self.player.row == self.goal.row and self.player.col == self.goal.col\n",
        "    if win:\n",
        "      self.level += 1\n",
        "      self.turns_taken = 0\n",
        "    return win\n",
        "\n",
        "  def check_loss(self):\n",
        "    for c in self.vampires:\n",
        "      if self.player.row == c.row and self.player.col == c.col:\n",
        "        return True\n",
        "    for other in self.other_creatures:\n",
        "      if self.player.row == other.row and self.player.col == other.col:\n",
        "        return True\n",
        "    return False\n",
        "\n",
        "  def broadcastMessage(self, message):\n",
        "    for controller in self.controllers:\n",
        "      controller.sendResponse(message)\n",
        "\n",
        "  def run(self, turns=None, max_level=50):\n",
        "    turns = len(self.map) * 2 if turns == None else turns\n",
        "    player_wins = False\n",
        "    while self.turns_taken < turns and self.level <= max_level:\n",
        "      self.turns_taken += 1\n",
        "      for controller in self.controllers:\n",
        "        map_info = self.map.print(map_thing=controller.getCreature(), flashlight=len(self.map))\n",
        "        controller.look(map_info)\n",
        "        if controller.canMove():\n",
        "          move = controller.getMove()\n",
        "          if (move == 'reset' or move == 4) and self.resets > 0:\n",
        "            self.resets -= 1\n",
        "            # controller.sendResponse(str(self.resets))\n",
        "            self.broadcastMessage('Reset ' + str(self.level))\n",
        "            self.init_map()\n",
        "            break\n",
        "          if (move == 'quit' or move == 5):\n",
        "            self.turns_taken = turns + 1\n",
        "            self.broadcastMessage('Quitting! ' + str(self.level))\n",
        "            return 'Quitting'\n",
        "          move_result = self.map.move_creature(controller.getCreature(), move)\n",
        "          player_wins = self.check_win()\n",
        "          if player_wins:\n",
        "            self.broadcastMessage('Player leveled! ' + str(self.level) + ' ' + self.goal.t_char)\n",
        "            self.init_map()\n",
        "          if self.check_loss():\n",
        "            self.broadcastMessage('Player lost! ' + str(self.level) + ' ' + controller.getCreature().t_char)\n",
        "            return 'Loss'\n",
        "        else:\n",
        "          controller.sendResponse(\"Can't move \" + str(self.level))\n",
        "\n",
        "    if self.turns_taken >= turns:\n",
        "      self.broadcastMessage('Too long! ' + str(self.level))\n",
        "      return 'Too Long'\n",
        "    self.broadcastMessage('Max level reached! ' + str(self.level))\n",
        "    return 'Max level'\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "  # g = Game(level=11, size=(20, 20), player_controller=ConsoleController())\n",
        "  g = Game(level=1, size=(20, 20), player_controller=None)\n",
        "\n",
        "  g.run(max_level=3, turns=10)\n",
        "  print(g.map)\n",
        "\n",
        "  \n",
        "\n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███#         █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██         █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █       V █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█      # █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██         █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █  V     █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█      #  ██ █ █   █\n",
            "█                  █\n",
            "██         █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █  V     █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█      #           █\n",
            "██         █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █ V      █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█       #          █\n",
            "██         █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  V█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██      #  █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█           V   ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██      #  █       █\n",
            "█       █    █    ██\n",
            "█    █         █████\n",
            "█          V    ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██      #  █       █\n",
            "█       █    █    ██\n",
            "█    █     V   █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██      #  █       █\n",
            "█       █    █    ██\n",
            "█    █    V    █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██       # █       █\n",
            "█       █ V  █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n",
            "Player lost! 1 V\n",
            "████████████████████\n",
            "██ ███   █  ██     █\n",
            "█ █ █       █  █   █\n",
            "█   ███          █ █\n",
            "█        █         █\n",
            "█         ██ █ █   █\n",
            "█                  █\n",
            "██         █       █\n",
            "█       █V   █    ██\n",
            "█    █         █████\n",
            "█               ██ █\n",
            "█  █ █ █ █  @█     █\n",
            "█  █      █        █\n",
            "██   █         █  ██\n",
            "█ █         █      █\n",
            "█ █    █ █         █\n",
            "█   █        █    ██\n",
            "█     ███    █ █   █\n",
            "██      █          █\n",
            "████████████████████\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vq5QJi3hj-r4"
      },
      "source": [
        "#Planning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNF5MuAYGxDT"
      },
      "source": [
        "Okay, so the game works in general now.  I need to find a couple parameters to search over.  \n",
        "\n",
        "In my proposal, I proposed iterating over different:\n",
        "\n",
        "\n",
        "1.   Map sizes\n",
        "2.   Map densities\n",
        "3.   Level progressions\n",
        "4.   Flashlight y (distance?) /n\n",
        "\n",
        "\n",
        "As well, there is the question of when to introduce the Vampire Lord.  The possibilities include:\n",
        "*    Never introduce vampire lord\n",
        "      *   Train player with no vampire lord\n",
        "\n",
        "*    Independant training:\n",
        "      1.   Train player with no vampire lord\n",
        "      2.   Train vampire lord with dumb player\n",
        "      3.   Train them together\n",
        "\n",
        "*    In-course training:\n",
        "      *   Train player\n",
        "      *   Introduce vampire lord at level X\n",
        "    \n",
        "\n",
        "I should have enough responses built in to build reward schemes.\n",
        "\n",
        "Reward for speed completion?\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1klcBEb2kDHD"
      },
      "source": [
        "#Events"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YI87ejLeb8De"
      },
      "source": [
        "\n",
        "*   Player completed a level\n",
        "*   Player caught (specific monster?) - terminates\n",
        "*   too long (bad for everyone) - terminates\n",
        "*   quit (very bad for player) - terminates\n",
        "*   Player reset map\n",
        "\n",
        "\n",
        "The following reward schemes are being developed in paralell and iterated upon.\n",
        "\n",
        "## Simple:\n",
        "\n",
        "The simplest reward scheme for the player would be:\n",
        "\n",
        "*   +1 for every level progressed\n",
        "*   -1 for caught by monster\n",
        "*   -1 for too long\n",
        "*   -L with L being the level achieved on quit\n",
        "*   +0 otherwise\n",
        "\n",
        "The simplest reward scheme for the Vampire Lord would be:\n",
        "\n",
        "*   +1 for the player being caught\n",
        "*   -1 for the player progressing a level\n",
        "*   -1 for too long\n",
        "*   +0 otherwise\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n-hhEH5Ikyj6"
      },
      "source": [
        "\n",
        "### Discussion:\n",
        "This will allow the player an increasing reward for progressing through levels and penalize the Vampire Lord for letting that happen.  It will also let the player quit, but then it will lose all of its points.\n",
        "\n",
        "With the simulation:\n",
        "```\n",
        "score = 0\n",
        "for l in range(1, 8):\n",
        "    print('level', l, 'for complete', score + 1, 'caught', score-1, 'too long', score -1, 'on quit', score - l)\n",
        "    score += 1\n",
        "```\n",
        "the results:\n",
        "```\n",
        "level 1 for complete 1 caught -1 too long -1 on quit -1\n",
        "level 2 for complete 2 caught 0 too long 0 on quit -1\n",
        "level 3 for complete 3 caught 1 too long 1 on quit -1\n",
        "level 4 for complete 4 caught 2 too long 2 on quit -1\n",
        "level 5 for complete 5 caught 3 too long 3 on quit -1\n",
        "level 6 for complete 6 caught 4 too long 4 on quit -1\n",
        "level 7 for complete 7 caught 5 too long 5 on quit -1\n",
        "```\n",
        "I like how for the first two levels, the only way to get any score is to progress.  And if the player is caught, at least it will have some reward for what progress it has made.  However, I'm not sure if I like the balance between the quitting and the too long penalty.\n",
        "\n",
        "Perhaps I may change the penalty for quitting to `-2`.  That way I can still assert that I don't like it to quit, but it would still get some reward.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2gi3_oRNksO7"
      },
      "source": [
        "## Complex:\n",
        "\n",
        "A more complex reward scheme for player:\n",
        "\n",
        "*   +L with L being the level completed\n",
        "*   -T/L with T being the sum of all levels cleared when caught by a monster\n",
        "*   -T/L when too long\n",
        "*   -1 for each reset\n",
        "*   -T on quit\n",
        "*   +0 otherwise\n",
        "\n",
        "A more complex reward scheme for the Vampire Lord:\n",
        "\n",
        "*   +1 for if the player is caught\n",
        "*   +2 for if the player is caught by the Vampire Lord\n",
        "*   -1 for if the player progresses a level\n",
        "*   -2 for if goes too long\n",
        "*   +0 otherwise\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wc5y1pMjknrX"
      },
      "source": [
        "### Discussion:\n",
        "The player scheme is way to complex.  For a simulation of:\n",
        "```\n",
        "score = 0\n",
        "for l in range(1, 8):\n",
        "    summer = sum([x+1 for x in range(l)])\n",
        "    avg = summer / l\n",
        "    print('level', l, 'for complete', score + l, 'caught', score-avg, 'too long', score -avg, 'on quit', score - summer)\n",
        "    score += l\n",
        "```\n",
        "The result of:\n",
        "```\n",
        "level 1 for complete 1 caught -1.0 too long -1.0 on quit -1\n",
        "level 2 for complete 3 caught -0.5 too long -0.5 on quit -2\n",
        "level 3 for complete 6 caught 1.0 too long 1.0 on quit -3\n",
        "level 4 for complete 10 caught 3.5 too long 3.5 on quit -4\n",
        "level 5 for complete 15 caught 7.0 too long 7.0 on quit -5\n",
        "level 6 for complete 21 caught 11.5 too long 11.5 on quit -6\n",
        "level 7 for complete 28 caught 17.0 too long 17.0 on quit -7\n",
        "```\n",
        "This has the same benefit on the first two levels as the simple reward scheme.  Notice the MUCH bigger penalty for quitting than for being caught.  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BaLXJ2eakcjk"
      },
      "source": [
        "## Middle Ground\n",
        "\n",
        "The reward scheme for the player would be:\n",
        "\n",
        "*   +2 for every level completed\n",
        "*   -1 for caught by monster\n",
        "*   -2 for too long  ?? debate??\n",
        "*   -L with L being the current level on quit ?? debate??\n",
        "*   +0 otherwise\n",
        "\n",
        "The reward scheme for the Vampire Lord would be:\n",
        "\n",
        "*   +1 for if the player is caught\n",
        "*   +2 for if the player is caught by the Vampire Lord\n",
        "*   -1 for if the player progresses a level\n",
        "*   -2 for if goes too long\n",
        "*   +0 otherwise\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NpOAZsEhkSlO"
      },
      "source": [
        "### Debate\n",
        "For the player I need to prioritize penalties between being caught, going for too long, or quitting.  Alternatively, I could just make it so that the playerAI does not have the option of quitting.\n",
        "\n",
        "The top prioritiy for the player is to clear levels, and progress to the next one.  In order to do that, it has to avoid getting caught.  Stalling, or going for too long, does not help anyone play well.  For normal play, a human may quit while they are ahead.\n",
        "\n",
        "Using the simulation:\n",
        "```\n",
        "score = 0\n",
        "for l in range(1, 8):\n",
        "    print('level', l, 'for complete', score + 2, 'caught', score-1, 'too long', score -2, 'on quit', score - l)\n",
        "    score += 2\n",
        "```\n",
        "The result of:\n",
        "```\n",
        "level 1 for complete 2 caught -1 too long -2 on quit -1\n",
        "level 2 for complete 4 caught 1 too long 0 on quit 0\n",
        "level 3 for complete 6 caught 3 too long 2 on quit 1\n",
        "level 4 for complete 8 caught 5 too long 4 on quit 2\n",
        "level 5 for complete 10 caught 7 too long 6 on quit 3\n",
        "level 6 for complete 12 caught 9 too long 8 on quit 4\n",
        "level 7 for complete 14 caught 11 too long 10 on quit 5\n",
        "```\n",
        "Shows that the player will get more of a reward for any level if it gets caught compared to if it quits.\n",
        "\n",
        "I think I may want to encourage this, because this will *let* the player allow the Vampire Lord to learn, rather than quitting too soon and 'starving' the Vampire Lord of material."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TerSDelpjsnn"
      },
      "source": [
        "#Policy Translator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I6QGCVmmPtzN"
      },
      "source": [
        "def translate_player_policy(result_str, which='simple'):\n",
        "  results = result_str.split()\n",
        "  results[1] = int(results[1])\n",
        "  if which is 'simple':\n",
        "    policy = {'Quitting!': -1 * results[1], 'Player leveled!': 1, 'Player lost!': -1, 'Too long!': -1}\n",
        "    return policy[results[0]] if results[0] in policy.keys else 0\n",
        "  if which is 'complex':\n",
        "    Total = -1 * sum([x for x in range(results[1])])\n",
        "    T_over_L = Total/results[1]\n",
        "    policy = {'Quitting!': Total, 'Player leveled!': int(results[1]), 'Player lost!': T_over_L, 'Too long!': T_over_L, 'Reset': -1}\n",
        "    return policy[results[0]] if results[0] in policy.keys else 0\n",
        "  if which is 'middle':\n",
        "    policy = {'Quitting!': results[1], 'Player leveled!': +2, 'Player lost!': -1, 'Too long!': -2}\n",
        "    return policy[results[0]] if results[0] in policy.keys else 0\n",
        "  # default simple\n",
        "  policy = {'Quitting!': -1 * results[1], 'Player leveled!': 1, 'Player lost!': -1, 'Too long!': -1}\n",
        "  return policy[results[0]] if results[0] in policy.keys else 0\n",
        "\n"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hc1fPo24GS4v",
        "outputId": "84e6904d-dc17-4d27-f4ea-4e44e9148396"
      },
      "source": [
        "import numpy as np\n",
        "import gc\n",
        "\n",
        "def map_to_numpy(map_str):\n",
        "  one_dim = map_str.split('\\n')[:-1]\n",
        "  two_dim = [[ord(y) for y in x] for x in one_dim]\n",
        "  two_dim = np.array(two_dim)\n",
        "  # print(two_dim)\n",
        "  return two_dim\n",
        "\n",
        "mmm = Map(10, 10)\n",
        "\n",
        "map_to_numpy(mmm.print())"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608],\n",
              "       [9608,   32,   32,   32,   32,   32,   32,   32,   32, 9608],\n",
              "       [9608, 9608,   32,   32,   32,   32,   32,   32,   32, 9608],\n",
              "       [9608,   32,   32,   32,   32,   32,   32,   32,   32, 9608],\n",
              "       [9608, 9608, 9608,   32,   32,   32,   32,   32,   32, 9608],\n",
              "       [9608, 9608,   32, 9608, 9608, 9608,   32,   32, 9608, 9608],\n",
              "       [9608,   32, 9608, 9608,   32,   32,   32,   32,   32, 9608],\n",
              "       [9608,   32,   32,   32,   32,   32, 9608,   32,   32, 9608],\n",
              "       [9608,   32, 9608, 9608, 9608, 9608, 9608, 9608,   32, 9608],\n",
              "       [9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608, 9608]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v5-W3pbTMTct"
      },
      "source": [
        "I wonder, is it going to be bad that I am using such estoric characters to represent the map...or is there an encoder that I can use...?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uNxPbLl-jVc4"
      },
      "source": [
        "#Networks\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fe4g6POWNf6X"
      },
      "source": [
        "import gym\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "\n",
        "# Dataset that wraps memory for a dataloader\n",
        "class RLDataset(Dataset):\n",
        "  def __init__(self, data):\n",
        "    super().__init__()\n",
        "    self.data = []\n",
        "    for d in data:\n",
        "      self.data.append(d)\n",
        "  \n",
        "  def __getitem__(self, index):\n",
        "    return self.data[index]\n",
        " \n",
        "  def __len__(self):\n",
        "    return len(self.data)\n",
        "\n",
        "# Policy Network from lab9\n",
        "class PolicyNetwork(nn.Module):\n",
        "  def __init__(self, state_size, action_size):\n",
        "    super().__init__()\n",
        "    hidden_size = 8\n",
        "    \n",
        "    self.net = nn.Sequential(nn.Linear(state_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, action_size),\n",
        "                             nn.Softmax(dim=1))\n",
        "  \n",
        "  def forward(self, x):\n",
        "    \"\"\"Get policy from state\n",
        "\n",
        "      Args:\n",
        "          state (tensor): current state, size (batch x state_size)\n",
        "\n",
        "      Returns:\n",
        "          action_dist (tensor): probability distribution over actions (batch x action_size)\n",
        "    \"\"\"\n",
        "    print(x, x.size())\n",
        "    print(self.net[0].weight.size())\n",
        "    return self.net(x)\n",
        "\n",
        "\n",
        "# Value Network from lab9\n",
        "class ValueNetwork(nn.Module):\n",
        "  def __init__(self, state_size):\n",
        "    super().__init__()\n",
        "    hidden_size = 8\n",
        "  \n",
        "    self.net = nn.Sequential(nn.Linear(state_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, hidden_size),\n",
        "                             nn.ReLU(),\n",
        "                             nn.Linear(hidden_size, 1))\n",
        "    \n",
        "  def forward(self, x):\n",
        "    \"\"\"Estimate value given state\n",
        "\n",
        "      Args:\n",
        "          state (tensor): current state, size (batch x state_size)\n",
        "\n",
        "      Returns:\n",
        "          value (tensor): estimated value, size (batch)\n",
        "    \"\"\"\n",
        "    return self.net(x)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qe16CY46lwaf"
      },
      "source": [
        "#Helper Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wik_FggMOvXq"
      },
      "source": [
        "def calculate_return(memory, rollout, gamma):\n",
        "  \"\"\"Return memory with calculated return in experience tuple\n",
        "\n",
        "    Args:\n",
        "        memory (list): (state, action, action_dist, return) tuples\n",
        "        rollout (list): (state, action, action_dist, reward) tuples from last rollout\n",
        "        gamma (float): discount factor\n",
        "\n",
        "    Returns:\n",
        "        list: memory updated with (state, action, action_dist, return) tuples from rollout\n",
        "  \"\"\"\n",
        "  reversed_return = []\n",
        "  for item in reversed(rollout):\n",
        "    the_return = 0\n",
        "    if len(reversed_return) > 0:\n",
        "      the_return = item[-1] + gamma*reversed_return[-1][-1]\n",
        "    else:\n",
        "      the_return = item[-1]\n",
        "    reversed_return.append(item[:-1] + (the_return,))\n",
        "\n",
        "  reversed_return.reverse()\n",
        "  return memory + reversed_return\n",
        "\n",
        "\n",
        "def get_action_ppo(network, state):\n",
        "  \"\"\"Sample action from the distribution obtained from the policy network\n",
        "\n",
        "    Args:\n",
        "        network (PolicyNetwork): Policy Network\n",
        "        state (np-array): current state, size (state_size)\n",
        "\n",
        "    Returns:\n",
        "        int: action sampled from output distribution of policy network\n",
        "        array: output distribution of policy network\n",
        "  \"\"\"\n",
        "  with torch.no_grad():\n",
        "    torch_state = torch.as_tensor(state, device='cuda').float().unsqueeze(0)\n",
        "    distribution = network(torch_state).detach()\n",
        "    action = torch.multinomial(distribution, 1).item()\n",
        "    return action, distribution\n",
        "  \n",
        "\n",
        "def learn_ppo(optim, policy, value, memory_dataloader, epsilon, policy_epochs):\n",
        "  \"\"\"Implement PPO policy and value network updates. Iterate over your entire \n",
        "     memory the number of times indicated by policy_epochs.    \n",
        "\n",
        "    Args:\n",
        "        optim (Adam): value and policy optimizer\n",
        "        policy (PolicyNetwork): Policy Network\n",
        "        value (ValueNetwork): Value Network\n",
        "        memory_dataloader (DataLoader): dataloader with (state, action, action_dist, return, discounted_sum_rew) tensors\n",
        "        epsilon (float): trust region\n",
        "        policy_epochs (int): number of times to iterate over all memory\n",
        "  \"\"\"\n",
        "  optim.zero_grad()\n",
        "\n",
        "  for count in range(policy_epochs):\n",
        "    optim.zero_grad()\n",
        "    for batch in memory_dataloader:\n",
        "      state, action, action_dist, theReturn = batch\n",
        "\n",
        "      torch_state = torch.as_tensor(state, device='cuda').float()\n",
        "      actions = torch.as_tensor(action.view(-1, 1), device='cuda')\n",
        "      torch_return = torch.as_tensor(theReturn, device='cuda')\n",
        "\n",
        "      advantages = torch_return - value(torch_state)\n",
        "      value_loss = (advantages**2).mean()\n",
        "      advantages = advantages.detach()\n",
        "\n",
        "      pi = torch.gather(action_dist.squeeze(1).cuda(), 1, actions).squeeze()\n",
        "      piprime = torch.gather(policy(torch_state), 1, actions).squeeze()\n",
        "      ratio = piprime / pi\n",
        "      policy_loss = -1 * torch.mean(torch.min(ratio * advantages, torch.clamp(ratio, 1-epsilon, 1+epsilon) * advantages))\n",
        "\n",
        "      total_loss = value_loss + policy_loss\n",
        "      \n",
        "      total_loss.backward()\n",
        "      optim.step()\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tyeiIR6cl0ho"
      },
      "source": [
        "#NetController"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xSMQV0r4Z3u-"
      },
      "source": [
        "class NetController(Controller):\n",
        "  \n",
        "  def __init__(self, net):\n",
        "    super().__init__(move_func=self._getMove, responder=self._responder, looker=self._looker)\n",
        "    self.net = net\n",
        "    self.memory = []\n",
        "    self.rewards = []\n",
        "    self.cumulative_reward = 0\n",
        "    self.rollout = []\n",
        "    self.cur_state = None\n",
        "    # self.move_func = self._getMove\n",
        "    # self.looker = self._looker\n",
        "    # self.responder = self._responder\n",
        "\n",
        "  def epoch_reset(self):\n",
        "    self.memory = []\n",
        "    self.rewards = []\n",
        "    self.rollout = []\n",
        "    self.cumulative_reward = 0\n",
        "\n",
        "  def flush_episode(self, gamma):\n",
        "    self.memory = calculate_return(self.memory, self.rollout, gamma)\n",
        "    self.rewards.append(self.cumulative_reward)\n",
        "    self.cumulative_reward = 0\n",
        "\n",
        "  def _looker(self, map_str):\n",
        "    self.cur_state = map_to_numpy(map_str)\n",
        "\n",
        "  def _getMove(self):\n",
        "    print(self.cur_state)\n",
        "    self.action, self.action_dist = get_action_ppo(self.net, self.cur_state)\n",
        "    return self.action\n",
        "\n",
        "  def _responder(self, message):\n",
        "    reward = translate_player_policy(message)\n",
        "    self.cumulative_reward += reward\n",
        "    rollout.append((self.cur_state, self.action, self.action_dist, reward))"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hMGb-c-Bl3cU"
      },
      "source": [
        "#Main PPO Algorithm"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 608
        },
        "id": "WkhAD7bkT8Mw",
        "outputId": "be93642c-8dc7-460b-f145-3395330b4d76"
      },
      "source": [
        "from itertools import chain\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "\n",
        "\n",
        "def ppo_main():\n",
        "  # Hyper parameters\n",
        "  lr = 1e-3\n",
        "  epochs = 20\n",
        "  env_samples = 100\n",
        "  gamma = 0.9\n",
        "  batch_size = 256\n",
        "  epsilon = 0.2\n",
        "  policy_epochs = 5\n",
        "\n",
        "  # Init environment varables\n",
        "  map_width = 40\n",
        "  map_height = 40\n",
        "  player_actions = 6\n",
        "\n",
        "  # Init networks\n",
        "  player_policy_network = PolicyNetwork(map_width * map_height, player_actions).cuda()\n",
        "  player_value_network = ValueNetwork(map_width * map_height).cuda()\n",
        "\n",
        "  # init controllers\n",
        "  player_controller = NetController(player_policy_network)\n",
        "\n",
        "  # Init optimizer\n",
        "  optim = torch.optim.Adam(chain(player_policy_network.parameters(), player_value_network.parameters()), lr=lr)\n",
        "\n",
        "  # Start main loop\n",
        "  results_ppo = []\n",
        "  loop = tqdm(total=epochs, position=0, leave=False)\n",
        "  for epoch in range(epochs):\n",
        "    \n",
        "    # memory = []  # Reset memory every epoch\n",
        "    # rewards = []  # Calculate average episodic reward per epoch\n",
        "    player_controller.epoch_reset()\n",
        "\n",
        "    # Begin experience loop\n",
        "    for episode in range(env_samples):\n",
        "\n",
        "      game = Game(size=(map_width, map_height), player_controller=player_controller)\n",
        "      game.run(max_level=16)\n",
        "      player_controller.flush_episode(gamma)\n",
        "      \n",
        "      \n",
        "    # Train\n",
        "    dataset = RLDataset(player_controller.memory)\n",
        "    loader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n",
        "    learn_ppo(optim, player_policy_network, player_value_network, loader, epsilon, policy_epochs)\n",
        "    \n",
        "    # Print results\n",
        "    results_ppo.extend(player_controller.rewards)  # Store rewards for this epoch\n",
        "    loop.update(1)\n",
        "    loop.set_description(\"Epochs: {} Reward: {}\".format(epoch, results_ppo[-1]))\n",
        "\n",
        "  return results_ppo\n",
        "\n",
        "results_ppo = ppo_main()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/20 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[[9608 9608 9608 ... 9608 9608 9608]\n",
            " [9608   32 9608 ...   32   32 9608]\n",
            " [9608   32 9608 ...   32   32 9608]\n",
            " ...\n",
            " [9608   32 9608 ...   32 9608 9608]\n",
            " [9608   32   32 ...   32   32 9608]\n",
            " [9608 9608 9608 ... 9608 9608 9608]]\n",
            "tensor([[[9608., 9608., 9608.,  ..., 9608., 9608., 9608.],\n",
            "         [9608.,   32., 9608.,  ...,   32.,   32., 9608.],\n",
            "         [9608.,   32., 9608.,  ...,   32.,   32., 9608.],\n",
            "         ...,\n",
            "         [9608.,   32., 9608.,  ...,   32., 9608., 9608.],\n",
            "         [9608.,   32.,   32.,  ...,   32.,   32., 9608.],\n",
            "         [9608., 9608., 9608.,  ..., 9608., 9608., 9608.]]], device='cuda:0') torch.Size([1, 40, 40])\n",
            "torch.Size([8, 1600])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-705ae2271cee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mresults_ppo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0mresults_ppo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mppo_main\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-29-705ae2271cee>\u001b[0m in \u001b[0;36mppo_main\u001b[0;34m()\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m       \u001b[0mgame\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap_width\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_height\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplayer_controller\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplayer_controller\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m       \u001b[0mgame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m       \u001b[0mplayer_controller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush_episode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgamma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-13-3aeeb7adc053>\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, turns, max_level)\u001b[0m\n\u001b[1;32m    335\u001b[0m         \u001b[0mcontroller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcontroller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanMove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 337\u001b[0;31m           \u001b[0mmove\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcontroller\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetMove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    338\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmove\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'reset'\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmove\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresets\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresets\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-13-3aeeb7adc053>\u001b[0m in \u001b[0;36mgetMove\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mgetMove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmove_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0msendResponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-22-7443f7f5f569>\u001b[0m in \u001b[0;36m_getMove\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     29\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_getMove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcur_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_dist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_action_ppo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcur_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-17-dde88bd0dd0a>\u001b[0m in \u001b[0;36mget_action_ppo\u001b[0;34m(network, state)\u001b[0m\n\u001b[1;32m     36\u001b[0m   \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0mtorch_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'cuda'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m     \u001b[0mdistribution\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnetwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistribution\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdistribution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-28-bb449813c732>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1690\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1691\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1692\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1693\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1694\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: mat1 dim 1 must match mat2 dim 0"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGENYmBV_k6p"
      },
      "source": [
        "plt.plot(results_ppo)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}