Battleship -- RL .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow Reinforcement Learning -- Battleship\n",
    "##### JSL September 2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 1.1 Import tensorflow and other libraries.\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import pylab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 1.2 Define the nn variable network.\n",
    "# Input is array of BOARD_SIZE values.\n",
    "# ---------------------------------------\n",
    "#  -1 value -> Not yet checked\n",
    "#   0 value -> Checked, no ship\n",
    "#   1 value -> Checked, is ship location.\n",
    "# ---------------------------------------\n",
    "BOARD_SIZE = 10\n",
    "SHIP_SIZE = 3\n",
    "\n",
    "hidden_units = BOARD_SIZE\n",
    "output_units = BOARD_SIZE\n",
    "\n",
    "input_positions = tf.placeholder(tf.float32, shape=(1, BOARD_SIZE))\n",
    "labels =          tf.placeholder(tf.int64)\n",
    "learning_rate =   tf.placeholder(tf.float32, shape=[])\n",
    "# Generate hidden layer\n",
    "W1 = tf.Variable(tf.truncated_normal([BOARD_SIZE, hidden_units],\n",
    "             stddev=0.1 / np.sqrt(float(BOARD_SIZE))))\n",
    "b1 = tf.Variable(tf.zeros([1, hidden_units]))\n",
    "h1 = tf.tanh(tf.matmul(input_positions, W1) + b1)\n",
    "# Second layer -- linear classifier for action logits\n",
    "W2 = tf.Variable(tf.truncated_normal([hidden_units, output_units],\n",
    "             stddev=0.1 / np.sqrt(float(hidden_units))))\n",
    "b2 = tf.Variable(tf.zeros([1, output_units]))\n",
    "logits = tf.matmul(h1, W2) + b2 \n",
    "probabilities = tf.nn.softmax(logits) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 1.3 Define the operations we will use\n",
    "init = tf.initialize_all_variables()\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "        logits, labels, name='xentropy')\n",
    "train_step = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=learning_rate).minimize(cross_entropy)\n",
    "# Start TF session\n",
    "sess = tf.Session()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([[[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1]],\n",
       "  [[-1, -1, 0, -1, -1, -1, -1, -1, -1, -1]],\n",
       "  [[-1, -1, 0, -1, -1, 0, -1, -1, -1, -1]],\n",
       "  [[-1, -1, 0, -1, -1, 0, 1, -1, -1, -1]],\n",
       "  [[-1, -1, 0, -1, -1, 0, 1, 1, -1, -1]]],\n",
       " [2, 5, 6, 7, 8],\n",
       " [0, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.4 Game play definition.\n",
    "TRAINING = True\n",
    "def play_game(training=TRAINING):\n",
    "    \"\"\" Play game of battleship using network.\"\"\"\n",
    "    # Select random location for ship\n",
    "    ship_left = np.random.randint(BOARD_SIZE - SHIP_SIZE + 1)\n",
    "    ship_positions = set(range(ship_left, ship_left + SHIP_SIZE))\n",
    "    # Initialize logs for game\n",
    "    board_position_log = []\n",
    "    action_log = []\n",
    "    hit_log = []\n",
    "    # Play through game\n",
    "    current_board = [[-1 for i in range(BOARD_SIZE)]]\n",
    "    while (sum(hit_log) < SHIP_SIZE) and (len(action_log) < BOARD_SIZE):\n",
    "        board_position_log.append([[i for i in current_board[0]]])\n",
    "        probs = sess.run([probabilities], feed_dict={input_positions:current_board})[0][0]\n",
    "        probs = [p * (index not in action_log) for index, p in enumerate(probs)]\n",
    "        probs = [p / sum(probs) for p in probs]\n",
    "        if training == True:\n",
    "            bomb_index = np.random.choice(BOARD_SIZE, p=probs)            \n",
    "        else:\n",
    "            bomb_index = np.argmax(probs)\n",
    "        # update board, logs\n",
    "        hit_log.append(1 * (bomb_index in ship_positions))\n",
    "        current_board[0][bomb_index] = 1 * (bomb_index in ship_positions)\n",
    "        action_log.append(bomb_index)\n",
    "    return board_position_log, action_log, hit_log\n",
    "# Example:\n",
    "play_game()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.16904761904761903,\n",
       " 0.2619047619047619,\n",
       " 1.1904761904761905,\n",
       " 1.130952380952381,\n",
       " 0.8333333333333334]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.5 Reward function definition\n",
    "def rewards_calculator(hit_log, gamma=0.5):\n",
    "    \"\"\" Discounted sum of future hits over trajectory\"\"\"            \n",
    "    hit_log_weighted = [(item -  \n",
    "                         float(SHIP_SIZE - sum(hit_log[:index])) / float(BOARD_SIZE - index)) * (\n",
    "            gamma ** index) for index, item in enumerate(hit_log)]\n",
    "    return [((gamma) ** (-i)) * sum(hit_log_weighted[i:]) for i in range(len(hit_log))]\n",
    "\n",
    "# Example\n",
    "rewards_calculator([0,0,1,1,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 1.6 Training loop: Play and learn\n",
    "game_lengths = []\n",
    "TRAINING = True   # Boolean specifies training mode\n",
    "ALPHA = 0.06      # step size\n",
    "\n",
    "for game in range(10000):\n",
    "    board_position_log, action_log, hit_log = play_game(training=TRAINING)\n",
    "    game_lengths.append(len(action_log))\n",
    "    rewards_log = rewards_calculator(hit_log)\n",
    "    for reward, current_board, action in zip(rewards_log, board_position_log, action_log):\n",
    "        # Take step along gradient\n",
    "        if TRAINING:\n",
    "            sess.run([train_step], \n",
    "                feed_dict={input_positions:current_board, labels:[action], learning_rate:ALPHA * reward})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x10cb806d0>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg8AAAFkCAYAAACn/timAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XecVNX9//HXUgIIAooKaoxYqCqRXTUYI2IUG/ZYssaO\niVGTiOarxq+xxq/GjjVRMZZgVlIkRgXFhoKdXREQsQUsoBQBQTrs/f1xdn8zW9nZnd075fV8POZx\n79y5d+az15V577nnngOSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJElSreYA5bU87qrn\nmCFAGbAa+Ag4rVkrlCRJGaUbsFXS4wBCeBhcx/47ACuAm4A+wHnAOuCgZq9UkiRlpJHAh/W8fgMw\nrdq2EmB8s1UkSZIy1neARcDv6tnnFeDWatvOAJY2V1GSJKn5tWnkcUcDXYCH6tmnOzC/2rb5QGeg\nHbCmjuO2rnhIkqTUfFnxaFaNDQ/DgXHAV2msBWDrbbbZZt68efPS/LaSJOWFucCeNHOAaEx42J7Q\nWfKYjez3FdCj2rbuwDLqaXWYN28eo0ePpl+/fo0oTY0xYsQIRo4cGXcZecVz3vI85y3Pc96y3n//\nfU4++eRtCa33GRceziBcfnh6I/u9DhxWbdtQ4LWNfUC/fv0oLCxsRGlqjK5du3q+W5jnvOV5zlue\n5zx3tWrE/mcADxNu00x2fcX2Sn8GdiTcddEXOBc4HritUZVKkqSMkGp4OBD4LvCXWl7rAWyX9HwO\nMIzQ2jAVuIDQV+K5lKuUJEkZI9XLFhOA1nW8dkYt214GbLOSJCmHpNryoBxUXFwcdwl5x3Pe8jzn\nLc9znrsK4i6gmkKgtLS01E42kiSloKysjKKiIoAiwrxSzcaWB0mSlBLDgyRJSonhQZIkpcTwIEmS\nUmJ4kCRJKTE8SJKklBgeJElSSgwPkiQpJYYHSZKUEsODJElKieFBkiSlxPAgSZJSYniQJEkpMTxI\nkqSUGB4kSVJKDA+SJCklhgdJkpQSw4MkSUqJ4UGSJKUkI8NDFMVdgSRJqktGhocZM+KuQJIk1SUj\nw8MLL8RdgSRJqktGhoeZM+OuQJIk1SUjw8NHH9nvQZKkTJWR4WHZMpg7N+4qJElSbTIyPABMmxZ3\nBZIkqTYZGR46dYJ33427CkmSVJuMDA+9ehkeJEnKVIYHSZKUkowMD717w4cfwqpVcVciSZKqy9jw\nUF7uSJOSJGWijAwPO+4YliUl8dYhSZJqysjw0KEDbLMN3H03bNgQdzWSJClZRoYHgAcfhLVr4YMP\n4q5EkiQly9jwMGgQFBTAm2/GXYkkSUqWseGhc2fo2hXOPNN5LiRJyiQZGx4ATjghLB3zQZKkzJHR\n4eHGG8PyvffirUOSJCVkdHjo3Bl69AgDRkmSpMyQ0eEBwoBR3nEhSVLmyPjw0LMnjBkD8+bFXYkk\nSYIsCA99+4blo4/GW4ckSQoyPjz89rfQvbudJiVJyhQZHx6+8x04+mgoK4u7EkmSBFkQHgCKimDm\nTKfoliQpE2RNeNiwAUaPjrsSSZKUFeFh4EBo2xbeeSfuSiRJUlaEh4ICOP54h6mWJCkTZEV4ABgw\nAKZPh/LyuCuRJCm/pRoetgVGA4uAlcA0oKie/YcA5dUeG4CtUi30+9+H5cvh009TPVKSJKVTmxT2\n3Qx4FXgBOARYCPQCljTg2F7A8qTnC1P4XCC0PEC4dLHDDqkeLUmS0iWV8HAJ8CkwPGlbQ9sBFgHf\npPBZNWy9NXTrBtOmhXEfJElSPFK5bHEkUAr8A5gPlAFnNfDYqcA8YALww1QKrFRQEFof7DQpSVK8\nUgkPOwLnAB8ABwF/Au4ATq3nmHnA2cCxwE+Az4GJwMBG1ErfvvDxx405UpIkpUtBCvuuBd4CfpS0\n7XZgT1JrTZgIfEbtoaMQKN13333p2rVrlReKi4uZP7+YCy6Ajz6CnXdO4RMlScohJSUllJSUVNm2\ndOlSJk2aBOFGhmad1CGV8DCHcNnhF0nbzgEuA76bwvvcBOxD7YGjECgtLS2lsLCwxosLFoRJsn76\nU6h2ziRJymtlZWUUFRVBC4SHVC5bvAr0rbatNyFUpGJ3wuWMlG21FRxxBDzxBKxf35h3kCRJTZVK\neLgNGARcCuwMnAT8HLg7aZ/rgYeTno8gdLTcGdgVGEkY+yH5mJSMGBEmyPrkk8a+gyRJaopUwsMU\n4BigGJhOuFxxPpB8AaEHsF3S87bALYTBpCYCuwEHAi81tuD+/cNy5szGvoMkSWqKVMZ5AHi64lGX\nM6o9v6nikTbdu8Pmm8N778Exx6TznSVJUkNkzdwWlQoKYJddQniQJEktL+vCA8Buuzk9tyRJccnK\n8LD33vDBB7B0adyVSJKUf7IyPAysGJ+yrFnvYpUkSbXJyvDQpw9stx2cf37clUiSlH+yMjy0aQNn\nngkzZsCiRXFXI0lSfsnK8ABw4olhOWNGvHVIkpRvsjY87LwztG3rLZuSJLW0rA0PbduGKbpteZAk\nqWVlbXiAMN7DtGlxVyFJUn7J+vAwfTosXx53JZIk5Y+sDg8DBoTg0LkzrFwZdzWSJOWHrA4P++yT\nWP/3v+OrQ5KkfJLV4aFLF/jqK9hpJ3ip0ZN8S5KkVGR1eIAwRffgwfDWW3FXIklSfsj68AAwZEi4\n62LSpLgrkSQp9+VEeCgsDMvBg+OtQ5KkfJAT4WGXXcJEWRCm6pYkSc0nJ8JDQQG89lpYP+MMKC+P\ntx5JknJZToQHgK23DsvXX4dnn423FkmSclnOhIfWrWHmzLD+0Ufx1iJJUi7LmfAA0K8f9O4Nb78d\ndyWSJOWunAoPAJttBqNHw2efxV2JJEm5KefCw/XXh+V998VbhyRJuSrnwsP++8NPfgIvvwxRFHc1\nkiTlnpwLDxBGnJw8GYqL465EkqTck5Ph4fDDw3LMGPjii3hrkSQp1+RkeOjZE2bPDuvTp8daiiRJ\nOScnwwPA974HnTrBjBlxVyJJUm7J2fDQqlWY8+L++2H16rirkSQpd+RseAAYNCiMNvmjH8VdiSRJ\nuSOnw8Mf/xhGnSwthaVL465GkqTckNPhoX17KCkJ63/8Y7y1SJKUK3I6PEBoeQC44Qan6pYkKR1y\nPjx85zvwz3+G9TfeiLcWSZJyQc6HB4CjjgqXMO65J+5KJEnKfnkRHtq0CbdrPvoojB/vnBeSJDVF\nXoQHgHPOCcvDDoOLLoq3FkmSslnehId77oFevcL6Lbd466YkSY2VN+EB4MMP4dVXw/orr8RbiyRJ\n2SqvwgPA3nuH5VFHxVuHJEnZKu/CQ0FBYv3vf4+vDkmSslXehQeAb78Ny3/9K946JEnKRnkZHjp2\nhPPOCy0PBghJklKTl+EBYLfdwvK44+Dzz+OtRZKkbJK34eHkkxPrzz4bXx2SJGWbvA0PHTvCggVh\nfdmyeGuRJCmb5G14ANhyS/j+92Hy5LgrkSQpe+R1eIDQ52HsWHjmmbgrkSQpO+R9eLjkEujTB+68\nM+5KJEnKDnkfHtq2hXPPDZ0m582LuxpJkjJfquFhW2A0sAhYCUwDijZyzBCgDFgNfAScluJnNrtT\nToHWreEf/4i7EkmSMl8q4WEz4FVgDXAI0A+4EFhSzzE7AE8DLwDfB0YCo4CDGlNsc9lsMzjwQBgx\nAhYvjrsaSZIyWyrh4RLgU2A4MKVi/Xngv/Uc80vgE+Ai4APgbuCfwAWNKbY5HX98WB56aLx1SJKU\n6VIJD0cCpcA/gPmESxFnbeSYvQkBI9mEiu0Z5ZRToHdvWLgw7kokScpsqYSHHYFzCC0IBwF/Au4A\nTq3nmO6EoJFsPtAZaJfCZze71q3hoovg009hzZq4q5EkKXO1SWHfVsBbwO8rnr8L7Eq4NPFIOosa\nMWIEXbt2rbKtuLiY4uLidH5MDb16QXk57LUXTJ1adfpuSZIyRUlJCSUlJVW2LV26tMU+P5XwMA+Y\nWW3bLOAn9RzzFdCj2rbuwDJCx8tajRw5ksLCwhRKS49+/cJy2jR4/nkYOrTFS5AkaaNq+4O6rKyM\noqKN3QCZHqlctngV6FttW29gTj3HvA4cUG3bUOC1FD63xWy1FXz9dVg/5hiIonjrkSQpE6USHm4D\nBgGXAjsDJwE/J9xBUel64OGk538m9JW4gRA8zgWOr3ivjLT55uHOixUrYNasuKuRJCnzpBIepgDH\nAMXAdOAy4Hwg+aJLD2C7pOdzgGGE1oaphFs0hwPPNbriFvDQQ2F58MGxliFJUkZKpc8DhAGfnq7n\n9TNq2fYy0PIdGJpgk03C8vPPw6ULO05KkpSQ93Nb1OW888Jy+vR465AkKdMYHupw/fXQvj38619x\nVyJJUmYxPNRh003hgAMMD5IkVWd4qEffvvDee2HQqHXr4q5GkqTMYHiox5VXhuXbb4cRJyVJkuGh\nXptuCmPGhPWZ1cfWlCQpTxkeNuKEE6BrVzj9dFiyJO5qJEmKn+GhAa66KiwvvjjWMiRJygiGhwY4\n/3wYPhxGjfLuC0mSDA8NdFvFbBy33x5vHZIkxc3w0ECbbhr6PaxeHXclkiTFy/CQgn79wkybTtUt\nScpnhocUDBgAy5fDlClxVyJJUnwMDyk48EDo1g3uuSfuSiRJio/hIQVt2sAFF8BDD8G778ZdjSRJ\n8TA8pOjQQ8PyzDPjrUOSpLgYHlJUWBhm21y2LO5KJEmKh+GhEYYNg48/hhkz4q5EkqSWZ3hohLPP\nDsviYm/blCTlH8NDI2yyCey7b2h5GD8+7mokSWpZhodGGj8ett02zHchSVI+MTw0UseO4bbNsWPh\ns8/irkaSpJZjeGiC448Py+23h/LyeGuRJKmlGB6a4HvfS6y/+WZ8dUiS1JIMD0309tthec458dYh\nSVJLMTw00R57QOfOYbhqb9uUJOUDw0MaPPZYWM6ZE2sZkiS1CMNDGuy+e1jut1+8dUiS1BIMD2nQ\no0dYfv55vHVIktQSDA9pUFAAd98NbdvC6tVxVyNJUvMyPKTJoEGwbh1Mnhx3JZIkNS/DQ5oMHAi9\ne8Mjj8RdiSRJzcvwkCYFBWGWzaeeirsSSZKal+EhjXr1giVLYMqUuCuRJKn5GB7SaK+9wvLKK+Ot\nQ5Kk5mR4SKNeveCXv4Rx4+DZZ+OuRpKk5mF4SLPrroMuXeDCC+OuRJKk5mF4SLPNNoNf/QpmzoS3\n3oq7GkmS0s/w0AzOOy8s77033jokSWoOhodmsPXWcP758MILcVciSVL6GR6aye67w6efwqpVcVci\nSVJ6GR6ayU47hWXHjvDSS/HWIklSOhkemkn//mEZRTByZLy1SJKUToaHZtKtG8yeDddcA//5D3To\nAOXlcVclSVLTGR6aUc+eifEeVq+GadNiLUeSpLQwPDSzjh1DCwTALbfEW4skSelgeGgBPXtCv36w\neHHclUiS1HSGhxZy3HHw9tuhA6UkSdnM8NBC9tgDFi6EL76IuxJJkprG8NBCiorCsrQ03jokSWoq\nw0ML2WYb6NEDRo2CNWvirkaSpMZLJTxcBZRXe8ysZ/8htey/AdiqEXVmvYICGD4cnn4azj477mok\nSWq8VFseZgA9kh4/asAxvZL23xpYmOJn5oyrrgrLhx+GyZNjLUWSpEZLNTxsABYkPRpy8+Giasfk\n7f0GbdokZtq86654a5EkqbFSDQ+9gLnAJ8BoYLsGHDMVmAdMAH6Y4uflnB//GH79a5g6Ne5KJElq\nnFTCwxvAacDBwDnADsAkoFMd+88DzgaOBX4CfA5MBAY2stacsfvu8OGHsGJF3JVIkpS6giYc2wX4\nFLgQ+EsDj5kIfAacWsfrhUDpvvvuS9euXau8UFxcTHFxceMqzTAzZ8Iuu8C4cXDooXFXI0nKNiUl\nJZSUlFTZtnTpUiZNmgRQBJQ15+c3JTwAvAU8B1zWwP1vAvah7ssXhUBpaWkphYWFTSwtc0URdO0K\nF1yQ6EQpSVJTlJWVURQGFWr28NCUcR46EfpAfJnCMbsTLmfktYIC2HdfuPpqZ9qUJGWfVMLDzcBg\noCeh5WAssBaobDe5Hng4af8RwJHAzsCuwEjC2A93N6XgXHHZZVWXkiRli1TCw7aEoDALGEMYr2EQ\n8HXF6z2oevdFW+AWYBqhr8NuwIHAS02qOEfsvXdYPvUU/OIX8dYiSVIqUgkPxYQA0Z4QEk4CZie9\nfgbw46TnNxEua2wCbAEcALzclGJzTWWrw/33w4wZ8dYiSVJDObdFjC67DCZMCOtjx8ZbiyRJDWV4\niFGHDjB0KBxzDIwZE3c1kiQ1jOEhAxx3HLz3HixaFHclkiRtnOEhA/ywYtSL55+Ptw5JkhrC8JAB\nevYMyxwZQFOSlOMMDxni+OPD0vkuJEmZzvCQIUaMCMuZM+OtQ5KkjTE8ZIg994SOHeFlR8KQJGU4\nw0OGaNsWdtsNLroI3nkn7mokSaqb4SGDbFcxuHdhIZSXx1uLJEl1MTxkkBtvTKwfcUR8dUiSVB/D\nQwbp2TMMFgUwbhzccUes5UiSVCvDQ4bp3x+++iqsn38+vPFGvPVIklSd4SEDde8Ot90W1vfeG0aN\nirceSZKSGR4y1G9+A08+GdZ//vNwGUOSpExgeMhQrVrB4YfDnDnh+bBhYdpu78KQJMXN8JDhtt8+\nsX7ssfDoo/HVIkkSGB6yQnKnyccfj68OSZLA8JAVfvADmDsX/vAHePFFWLcu7ookSfnM8JAlttkG\nDjoIli2DTp3irkaSlM8MD1mkqCgs166FggI7T0qS4mF4yCKtW8P77yeeP/tsfLVIkvKX4SHL9O0L\nK1dC+/Zw/fVxVyNJykeGhyzUoQM88ABMmhRu5aycD0OSpJZgeMhSxcUwfDh89hmcemrc1UiS8onh\nIUsVFIQ5L9q1g7IyeOWVuCuSJOULw0OWmz8/LPfbD2bMiLcWSVJ+MDxkuS5d4JxzwrodKCVJLcHw\nkAPuvBP23BP+9jeYNy/uaiRJuc7wkANat05MmDVmTLy1SJJyn+EhR/TqBfvsA6+9FnclkqRcZ3jI\nIcOGwVNPwddfx12JJCmXGR5yyPDhYb6LBx+MuxJJUi4zPOSQrbaCoUPhuedgzZq4q5Ek5SrDQ47Z\nay+YMAF22gmiKO5qJEm5yPCQY3772xAg5s6FAQNgxYq4K5Ik5RrDQ47p2BEeeyysz5gBzz8fbz2S\npNxjeMhBPXvCeeeF9bfeirUUSVIOahN3AUq/ggK46y744gsnzJIkpZ8tDzls111h8mQDhCQpvQwP\nOezXvw7LkSPjrUOSlFsMDzmse/cwVXdZWdyVSJJyieEhxw0fDp9+CtddF3clkqRcYXjIcb16heVl\nl8Hq1fHWIknKDYaHHNe/f2L9ySfjq0OSlDsMDzmuc+cwTPUee8Cjj8ZdjSQpFxge8kRxMTzxBHz5\nZdyVSJKyneEhTxx1VFja+iBJairDQ57YaSc4+OAwXbckSU1heMgjRxwBL74I8+bFXYkkKZsZHvLI\nySdDu3Zw331xVyJJymaphIergPJqj5kbOWYIUAasBj4CTku5QqVNly7h1s2rr4by8rirkSRlq1Rb\nHmYAPZIeP6pn3x2Ap4EXgO8DI4FRwEGpl6l0GTQoLJ98Ej78ELbcEiZMiLcmSVJ2STU8bAAWJD0W\n17PvL4FPgIuAD4C7gX8CF6ReptLlhhvC8uijYf/9YdEiuPPOeGuSJGWXVMNDL2AuIRSMBrarZ9+9\ngeerbZtQsV0x6dABHn44rFd2nJw4Edati60kSVKWSSU8vEHos3AwcA7hssQkoFMd+3cH5lfbNh/o\nDLRLrUyl04knhuVOO8HkyfDtt2H67vnV/2tJklSLNins+0zS+gzgTeBT4ATgL+ksasSIEXTt2rXK\ntuLiYoqLi9P5MXmrXbswZDXA+vVhee+94fHOO7D77vHVJknauJKSEkpKSqpsW7p0aYt9fkETj38L\neA64rJbXXibcaZHcx+EM4Dagay37AxQCpaWlpRQWFjaxNDXUlVfCNdeE9eHDYdSoeOuRJKWurKyM\noqIigCLC92+zaco4D50IfSDqmi3hdeCAatuGAq814TPVDK6+GpYvhyuugAcegEmT4q5IkpTJUgkP\nNwODgZ7AD4GxwFqgst3keuDhpP3/DOwI3AD0Bc4Fjie0PCjDdOoE55wT1gcPhttvj7ceSVLmSiU8\nbEsICrOAMcBCYBDwdcXrPah698UcYBihtWEq4fLFcMJlDmWgHj1g9uywPmIEtODlM0lSFkmlw+TG\neiueUcu2lwn9GJQlevaE6dNht93gjTfgkEPirkiSlGmc20I17LILbLEFXH+9w1hLkmoyPKiGgoIw\nEuUrr8Cll8Ktt8LatYnbOiVJ+c3woFpVDqlx443w29+GsSF22CGECElSfjM8qFYdOsAJJ4TQUOmL\nL+CZZ+o+RpKUHwwPqtOYMbB6NcyYAQsXwvbbhxaJW25JjFApSco/hgdtVGUHysJCWLkS/ud/wpTe\nkqT8ZHhQg40eDXffHdbPPNMOlJKUrwwParBNNoFzz4UnnoCvv4Zhw+xAKUn5yPCglB15JDz4IEyY\nAE8/HXc1kqSWZnhQo5x+elgeeyxMnhxrKZKkFmZ4UKPtuWdY3nRTvHVIklqW4UGN9tZboeVh+fK4\nK5EktSTDg5pk4ECYOtVxHyQpnxge1CS77w5LlkCrVnD11XFXI0lqCYYHNUlRUWL9qqvg3XdjK0WS\n1EIMD2qSrbeG2bNh7tzw/IUX4q1HktT8DA9qsp49YZttYI89wgyc3/0uLF0ad1WSpOZieFDaXHBB\nWM6dC337GiAkKVcZHpQ2J50U7rp4+OHQiXLkyLgrkiQ1hzZxF6Dcc+qpMGVKuPvi3Xdh7Ni4K5Ik\npZMtD2oWlcNX//vf8NFHsZYiSUozw4OaxcCBcP/9YfyHa68NLRHLlsGHH8ZdmSSpqQwPahYFBXDW\nWXDjjfDII2EejC5doE8fmD497urCkNodO4Y627WDGTOqvv7mm6H+BQsa/xlRBHfdBf37hwC1alXT\napakTGF4ULM677wwCmWyAw6AWbPiqQfCnBydO8PKleH52rWw225QVhaeL14MgwbBAw/AxRc37D3X\nrw/HVw7TXVYGxx0Hv/41vP8+XH45bLIJ/O1v6fkZnn02tOLceSd8/XXYFkVQUgI77wwdOsBtt8HR\nR3vZSFL6GR7UrNq3h3fegTVr4Ntv4dZbYeFC6Nev+efD+Oyz0LJQUBDGnwB4/HH4wQ/C+umnhxaI\nRx8Nz4uKwr7duiXeY+3a0PqwZEn4kq78ok4WRTBkSDh+l13C3SZFReGzIBx7661h/Wc/g3nz4K9/\nDR1Kx48PrTOTJoUA0hCvvgqHHBJacX7zG9hiixBUWrUKd7y0agWrV8OFF8ITT0Dv3nDvvaHl4557\n4Iorwvn/y19SPaOSlJkKgai0tDRSbpo6NYrC120UjRhR8/Xy8qZ/xrJlUXTkkYnPqe0xZUrVz5wz\nJ4puvTWKNt88vH7MMVF09dVVj2nfvurznj2jaN26KHr88fC8Y8eqrw8ZEkXTpyc+p6Sk/pp+85so\nGjUqiubNSxyzalUUPfVUFJ14YhQtXBhFK1ZE0Z57Jo7p16/qe/zhD6GmSZOiqHfvKBo0qP7PnDo1\n/PzpOO+S4lVaWhoBUcV3aV4xPOSB5cvDFytEUWFh2LZqVeILbfjwqvvPnRtFgwdH0SefbPy9v/oq\nfPEXFETRfvtF0V13RdHMmVW/cB94oP73+Prr8AW8dGmizroe22wTRV27RtF224Wf4b33oujyy8Oy\nNscfH44rKIii1q3rf++nn46inXZKPD/llMT6+PFRtHJleM+FC6PoxRdDsKjNlCmJ4047LYpeeSWK\n3nmn5ufdf//Gz6+kzGV4MDzkvIULE19a770XRT/9adUvshtvjKLDDouiO++su8Ug2ZQp4cuvcr/b\nb6+5T3l5+OLcsCH1eh9/PASd11+PouefD60D++yT+LyJExv2PuXlUfTss1G0aFEUrV4dRd9+G4JK\nWVkUbbllaPlo1ar+UHHddanXv2pVqDm5heG++2q+98iRUXTppVF0zTVR9NpriZptmZAyn+HB8JAX\n5syp+eV11121f2FuumlY9upV833Ky6Noxx0T+/bpE0Xr1zd//WvWRNGhh0bRL36RnvdbtSrxJT17\ndhSdc04IChs2hJaCDh1qtso01erVUTRrVhTddFP9gQWiaNiwKPryy/R+fhSFIHXzzVE0f37631vK\nJ4YHw0PeSG4tqGzqHzMmin71qyjabLOwfcKE8KVa2QehS5fwZTN0aBTtv3/i+P79Q/+AXP0refny\n5v3Zpk+PonPPjaIttgitD7UFiLPOiqIZM9JXR3l51X4Zjz+e2P7uu6GVRlLDtGR4KGjuD0hRIVBa\nWlpKYWHeBae8VVoKO+4Im21W/34LFsCBB9Y+TsRhh8FTT4W7JZQe69fDHXfAUUeFO1DuuAOuvDK8\nNmoUDB/e9M94/nkYOhS+851wZ0ttPv0Uvve9pn9WOq1bF85B27bw059Cp05h++TJ4WcpL4fzz4cb\nbgh34kgtoaysjKKiIoAioCzmclqULQ/aqJNPDn+l7rprFP3+91E0eXLLXKbIdwsWJDp57rpr01of\nvvwyin74w/Beu+wSRd98E0WPPFJ7a8cNNySOmzcvHHfFFYkOoy2tvDyKfvSjqjX+5CdRdPrptdf/\n2GPx1Kn842ULw4PqsXx56LSoeDz9dPhSPPHEKFq7NoqeeSb8N4mi0A8kuUPqcceFy0/TplUNeJVf\ntIMGhcsTle6+O/TB+OqrEBr23Tfs9+MfR9FVV9X8Yv722/C+114bOp22hJEjw2dfe21iPfnS2Qkn\nhFtoK+vddtsoeuGFRNhavz70b3nuuSjq1CmEocWLw2srViTOx9Kl4Vz8+c9RdO+9UXTqqeGumssu\nC2HrP/8Jn1VXB+A1a3L3Ep5qZ3gwPEgZa/36xJgXu+2W+OL8618T6z16RNGBB9b8st955xAQIIp+\n+9uNf7mtXh1FfftWfY/nn4+iAw4I69Vvd129uuZ7rFsX7sap7bMqv3grg82TT4Yv6/rqGjAg3Ppb\nad680Gke1CRtAAANFUlEQVT297+v+flvv52obfvto+iQQ2pvnTj99Ch6+eUQkiCKDj984x1YKx+d\nOiXqveuuKPrvf0NrR8eOUXT00fWfX+UWw4PhQcpoixaF8S029sV2xhlRNHZszQG2KlsNGmL9+sSX\n6s03J7afdFLivXbdNRFmxo5NDJS19941P3fChHB8eXnV8JP8ePHF2mv58MPw+iOPNPxc/fOftX9G\nt241b0VOfrRpE0LYqaeGyyQXXRTOQ1FRYp/99gvLAQOi6K23an+fX/winA/lPsOD4UHKeBMnRtFe\ne4Uv1A8+CF9kt94aXps8OYquvDI0vVf673/D+BFFReEyRqrKyqpe+tiwIYruuSdx++gf/tDwv9a7\ndYuin/2s5vbLLw8tBCedFC4lrFwZLiWsWVN1YK05c1Krfd26cGnnhRfCpZ5kye974YVhDJT77gvn\nqzbr14cWjUqDB9f+M86enRgxtXXr0DKi3ObdFt5tIakR5s6F7343rJ91VphH5OGHw/wfixeHeVaG\nDg1frwBt2oSJ0gYMCM9bt4brroPLLqv9/bfcMtxR0bt3euteuzbcpdEYURTu+Pj738MMtgMGhDlW\n2rQJ73vjjTB2bJis7ZFH4JRT0lu7MkdL3m1heJCUU6Ko/lt2y8vD5GKDB8PEibDfflVfX7y46uRo\nlb7/fRgzJkxIlmm+/RY+/rjmDLaVli8PM8kCfPUVdO/ecrWlaskS6NrV264boyXDg7NqSsopG/vS\nadUK9t03hIzqwQFg881rXgRYtw6mTs3M4ABhnIm6ggPAppvC22+H9R49EtPRt5TFi2HpUnjsMfjk\nk9r3Wbs2jCmy+ebhv1FBAbzwQsvWqYYzPEjSRrRpE3cFTbfHHvDHP4b1oUNhzZqa+6xZAxs2wO23\nwzff1P4+tR2XbOHCsFyyBJ58MrQidOsWBoErLoaddw7B4LLLwlT0EC6ptGsH//lPeN62bVheeGFq\nP6NajuFBkvLEJZfAzTfDa69B+/Zw5pkwYQLMmgXHHhu2tWkDI0aEL/3//V9YvTocG0Vw3nlhn4IC\nuOmmsO3ll0NLxqpVIZRstVV4ffPN4cgjQwjp2jW8R3FxopbrrguXjrp1g9DSDhdfHD5v7doQdD7+\nOPH5yiyZdlXJPg+S1MxOPhkefbRh+954I2y/PZx4Yv37nXBC6LRZ3f/9Xwgh5eXhcgTA55/DihWh\njtLSsO3FF2H//RPHzZoF/frBD34QWigqWyNUN/s8SJKazejRMGNG+FIuKoJeveDSS+Hrr8McMuXl\n4fJEly6hNaAyODz2WHht3bpwB0uyv/8d9twT/vvf0L/hT38KrQb/+7/h9VZJ3zbbbQd9+8KUKaEV\n5JFHqgYHCK/37w9vvhlqS9WTT4afc9Gi1I/VxtnyIEmq1bRp4S4TCP0SBg5MvLZ8eegf0blzaCWY\nNAlOPz30XUiXtWvhiivCBGPvvx8CRbIlS8Jllk03rbo9iqqGlY4dQ73pvINjwYLwMx97bObcGWLL\ngyQpdgMGwOzZoaUhOThA+MLu2jV8SffvD2efnd7gAGHsiyuuCONv3Hpr1dfGjQv9Kjp3TtxJkvxa\nshUr4PXX01fX+vVw9NFw3HHw0kvpe18IrT+XX177nSYbNoTLO488AmecES7lxNWyYniQJNWpZ894\n7zbZZBM44AC4/354/PGwrX9/GDYssc9ee8G118KyZWGa98MPD9vffjvcJgrwr38l9p8/PwSf6oGk\nuuXLQ2vKwQeHKegr7xTZaadEGLn44pphpbpXXkl0JB04EK6/Pjw2bKi637hx4XLQtdfCgQfCNdck\nBjQDOOaYcC5OOw0eeiiEmDPPrP+z84XDU0uSqpgxo/YhuD/6KMxlUttrV1+dOP7yy8O2ESOiaPz4\nqvsNHBiGOD/77DBZ28KFieOuuabuIc6HDImihx9OPO/RI4rmzw8zni5ZEkU33hhFw4dH0bBh9Q+V\nvsUWYWbaf/wjse2ss6KoT5/E8/79E+sHHBDmOfnrX8Osqttum5gYzeGp7fMgSUry0UdhmO1LLoGf\n/QxGjQq3jUIYhvzJJ8PQ4ePGhb/QR41KHLtiRRhIK9nvfgfvvgvjx9f8rKuvDpcD7rwTTj0VBg0K\n42Rsu23oh/Hqq2FI8CiCe++FX/2q/toHDoQnngiXdVasCLe2Tp9e9dbVSrNmhcHIysvDrayvvlr1\n9TVrEkOZjxsXWmB23RXuugs23bTl+jxkGlseJElp9+aboTUAqk5VfvPNYVufPlFUUhJFvXpVbRlo\nyCRo5eVR9NBDUdSzZ9VjJ06Moqeeqn2q+OS6Lrkk7H/77TXfd9asKFq1KkxAV90330TRZpslf6Yt\nD7Y8SJJiMWpUaMUoLg79HVI1blzoh1H9dta6RBF8+GGYcC3VOzfWrAkDfR15JLRvX8bq1d5toRZS\nUlISdwl5x3Pe8jznLS9bz/lZZ4UOiY0JDgCHHdbw4AAhMPTp07hbPtu1gyOOCAGk+iWO5tSU8PA7\noBy4rZ59hlTsk/zYAGzVhM9VmmXr/+DZzHPe8jznLc9znrsaewPOnsAvgGmE6ysb0wtYnvR8YSM/\nV5IkxawxLQ+dgNHAWcCSBh6zCFiQ9GhI4JAkSRmoMeHhbuAp4EUa3uFyKjAPmAD8sBGfKUmSMkSq\nly1+CuxOuGwBG29BmAecDUwB2hNaKyYCPwDeqeug999/P8Wy1BRLly6lrCyvbgmOnee85XnOW57n\nvGW15HdnKn07tyOEgAOB6RXbJhJCwAUpvM9E4DPg1Fpe2xp4G9g2hfeTJEnBXMIf+F8254ekEh6O\nBh4n3C1RqTWh9WED0I6G9WW4CdiHui9fbF3xkCRJqfmSZg4OkNpli+eBXZOeFwAPAu8DN9DwTpC7\nEy5n1KVFfnBJktQ4qYSHb4GZ1batBBYnbb8e2AY4reL5COC/Fa9X9nkYAhzUuHIlSVLcmjrRauU4\n2pV6EPpGVGoL3ELow7ASeJfQZ+LlJn6uJEmSJEmSJEmSJElSpjgPmAOsAt4gMRCVUnMVNScjq97R\n9RrCHS8rgeeAnau93p4wkugiwpwk/8TJzJINBp4k3E9dDhxVyz7pOMebA48C3xCGgh8FdEzLT5B9\nNnbOH6Lm7/24avt4zlNzKWHcnWXAfGAs0LuW/fxdT5+GnPOH8Hf9/zsRWE24S6MvcC/hLo4t4ywq\nS11FmLBsq6TH5kmvX0L4RTkC2A34N/AJYZyOSn8CPiXcGVMIvAZMbt6ys8ohhH8wjyb8j3tktdfT\ndY7HA2WEIL0P8CHhf/Z8tLFz/iDwNFV/77tU28dznprxhMH8+gEDCNMSzAE2SdrH3/X0asg593c9\nyZvAHUnPC4AvCL+YSs1V1D30dwFhDI0Lk7Z1JrT2nFjxvAuwBjg2aZ8+hH+wf5DOQnNE9S+ydJ3j\nfhXPC5P2OZgwIFuPNNWerWoLDw8R/kqri+e86bYgnJ8fVTz3d735VT/nkCG/642ZGCvdvkP4AZ5P\n2hZVPN87loqyXy9C8+4nhBlQK2+f3QHoTtVzvYwQ3irPdRHhFtvkfT4gDCnuf4+Na+o5HlTxfG9g\nKeEvg0ovYIirS0T4K2s+MAu4h6otbp7zputasVxcsfR3vflVP+eQIb/rmRAetiAMcz2/2vYFmDob\n4w3C5Z+DgXMI/4NPIkylXnk+q5/r+YR/BKjYZy3hH4G69lHdmnqOeyTts6Da6+sJ/4j4/0VNzwCn\nAD8mtFjuR2iWrfw3znPeNK2AkYSm78o+VP6uN6/azjlkyO96UweJUuZ5Jml9BuGvgE+BEwgptTap\nzHGixvEcN68xSevvEfr9fEL4h/WlWCrKLXcD/anafF4Xf9fTo65znhG/65nQ8rCIcJ2l+l+13XGO\ni3T4htARZicS57O2c/1VxfpXhEtJnevZR3WrPEdNPcdfUbN3dBtC86T/HTZuNuHflsqe/57zxrsL\nOAzYn6rzEvm73nzqOue1yevf9Teo2mGyFaHD5MXxlJNTOhF6Q/+q4vk8au/gdELF8/o62+zVrJVm\np9o6TKbjHNfWoekg7EQGtXeYrO67hHN1eMVzz3nqCghfYp8T/vio7XV/19NrY+e8Nnn9u34C4Reu\n8haVe4Gv8VbNxriZcE98T8K0588RrnV1q3j9YsJ1reRbqz4mJNVK9xBuDxpC6HzjrZpVdSTMDrs7\n4X/AERXrlR1T03WOxwGlVL2VanSaf5ZsUd857wjcROjo1RM4gHDeZhE6jlXynKfmHsIfHoMJXyiV\nj/ZJ+/i7nl4bO+f+rteicpCo1cDrOEhUY5UQ7rRYTUivfyN0mkx2NeESxipgAjUHdWlHSL9fE2ZT\ndZCoqoaQGJxlQ9L6X5L2Scc53oxw3/UyQs/oUVS93zufDKHuc96e0NdnPuEvrtnAn6n5x4fnPDXV\nz3Xl49Rq+/m7nj4bO+f+rkuSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJEmSJCnj/T+r\nCr+LqlwiEAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10c668f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1.7 Plot running average game lengths\n",
    "window_size = 500\n",
    "running_average_length = [np.mean(game_lengths[i:i+window_size]) for i in range(len(game_lengths)- window_size)]\n",
    "pylab.plot(running_average_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 0.36417413,  0.21842285, -1.05438769,  0.92670012, -0.971443  ,\n",
       "         -0.42181543, -0.12549876, -0.1459792 ,  1.37149823,  0.88486165],\n",
       "        [ 0.40269327, -2.3138361 , -1.52807772, -1.89444983,  1.6621691 ,\n",
       "          1.21897411, -0.59128791,  0.32018194,  0.12851822, -1.5733639 ],\n",
       "        [ 0.75337362,  1.18250155,  1.49976432,  2.45021749, -2.0455699 ,\n",
       "         -1.00975513, -3.52385998, -2.81726122, -1.13073385,  0.09371365],\n",
       "        [-2.73572493, -2.58016634, -2.54336786,  0.78801006, -1.01746631,\n",
       "          0.83182812, -0.28737754,  0.25912744, -0.71347362,  5.37003899],\n",
       "        [-0.85589498,  1.38819838,  3.10451198, -3.52761793, -2.22771525,\n",
       "         -0.35671455,  2.07410955, -1.88416684, -1.69944739, -0.94166577],\n",
       "        [ 1.26362193, -1.74665356, -0.12891544,  0.56791085,  2.06429219,\n",
       "         -1.82320964, -1.02321613,  2.33585477,  0.26630208, -0.07931795],\n",
       "        [-0.70836729,  1.72130585, -2.96304846, -0.78141487, -3.5275557 ,\n",
       "          1.48491037,  0.20496099, -2.35341787,  0.51234746, -0.44399992],\n",
       "        [-3.01618671, -2.97795844,  1.63824856,  2.3610189 , -1.56688738,\n",
       "         -3.25071836,  0.62363726,  0.54381591,  0.38046595, -2.24974942],\n",
       "        [-0.28313148,  1.94622719, -1.26995134,  0.29591283,  2.50572872,\n",
       "         -0.94764274,  0.5393216 , -1.34083796, -3.52424574,  0.24192466],\n",
       "        [ 0.41004246, -0.53354394,  1.02343953, -0.28061312, -0.14645265,\n",
       "          0.08760639, -0.39470139,  1.62556291,  0.77635783,  0.37842411]], dtype=float32)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1.8 Example showing how to print current coefficient values\n",
    "sess.run([W1], \n",
    "             feed_dict={input_positions:board_position_log[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}