ep53_RI_RoboTrader.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ep53_RI_RoboTrader.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Episode 53. Reinforcement Learning: Robo Trader"
      ],
      "metadata": {
        "id": "vuvteE1ZAzJU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install yfinance\n",
        "import yfinance as yf"
      ],
      "metadata": {
        "id": "07fzm8IyBKA3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "import math\n",
        "from scipy.stats import norm\n",
        "#import tensorflow as tf\n",
        "import tensorflow.compat.v1 as tf\n",
        "tf.disable_v2_behavior()\n",
        "\n",
        "from tensorflow.keras.layers import Input, Dense, Lambda, Reshape\n",
        "from tensorflow.keras.models import Sequential, Model\n",
        "from tensorflow.keras.layers import Dropout, BatchNormalization, LeakyReLU\n",
        "from tensorflow.keras.layers import Reshape, Flatten, Conv2D, Conv2DTranspose\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras import metrics\n",
        "from tensorflow.keras import regularizers\n",
        "from tensorflow.keras.optimizers import RMSprop\n",
        "from tensorflow.keras.initializers import RandomNormal\n",
        "from tensorflow.keras.constraints import Constraint\n",
        "#from tensorflow.keras.layers.merge import concatenate\n",
        "from tensorflow.keras.models import load_model\n",
        "\n",
        "\n",
        "# changed to yfinance\n",
        "import yfinance as yf\n",
        "import random\n",
        "import pandas_datareader.data as web"
      ],
      "metadata": {
        "id": "fYCheruoA390"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Yahoo Finance API"
      ],
      "metadata": {
        "id": "2_9F3DMfBRJl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_prices(share_symbol, start_date, end_date,\n",
        "               cache_filename='stock_prices.npy', force=False):\n",
        "    try:\n",
        "        if force:\n",
        "            raise IOError\n",
        "        else:\n",
        "            stock_prices = np.load(cache_filename)\n",
        "    except IOError:\n",
        "        stock_hist = yf.download(share_symbol, start=start_date, end=end_date)\n",
        "        stock_prices = []\n",
        "        stock_hist = stock_hist.sort_values(by=['Date'])\n",
        "        for index, day in stock_hist.iterrows():\n",
        "            stock_val = day['Open'].astype(float)\n",
        "            stock_prices.append(stock_val)\n",
        "            \n",
        "        stock_prices = np.asarray(stock_prices)\n",
        "        np.save(cache_filename, stock_prices)\n",
        "        \n",
        "    return stock_prices.astype(float)"
      ],
      "metadata": {
        "id": "RhVRcGQRA4Bl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_prices(prices):\n",
        "    plt.title('Opening stock prices')\n",
        "    plt.xlabel('day')\n",
        "    plt.ylabel('price ($)')\n",
        "    plt.plot(prices)\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "itOAhJy4A4E-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Deep Q-Network"
      ],
      "metadata": {
        "id": "ld5zon5qBb-U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# set neural network structure\n",
        "hist = 30\n",
        "Na = 3 # action choices: 0=hold,1=sell,2=buy\n",
        "Ns = 2+hist # # state: budget,shares,past Nlag sp500 indices\n",
        "\n",
        "input_dim = 2+hist # budget,shares,past Nlag sp500 indices\n",
        "intermediate_dim1 = 10\n",
        "intermediate_dim2 = 10\n",
        "intermediate_dim3 = 10\n",
        "output_dim = Na # Q-values for 0=hold,1=sell,2=buy"
      ],
      "metadata": {
        "id": "wsRSOoPyBhjC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# deep Q network\n",
        "# input: num shares, portfolio amount, past Nlag sp500 index values\n",
        "# output: action choice(0=hold,1=sell,2=buy)\n",
        "def build_dqn():\n",
        "    # define deep q-network\n",
        "    dqn_in = Input(shape=(input_dim,), name=\"dqn_input\")\n",
        "    # intermediate layer\n",
        "    h1 = Dense(intermediate_dim1, name=\"hidden1\")(dqn_in)\n",
        "    #h1 = BatchNormalization()(h1)\n",
        "    h1 = LeakyReLU(alpha=0.1)(h1)\n",
        "    #h1 = Dropout(rate=0.01)(h1)\n",
        "    \n",
        "    h2 = Dense(intermediate_dim2, name=\"hidden2\")(h1)\n",
        "    #h2 = BatchNormalization()(h2)\n",
        "    h2 = LeakyReLU(alpha=0.1)(h2)\n",
        "    #h2 = Dropout(rate=0.01)(h2)\n",
        "    \n",
        "    #h3 = Dense(intermediate_dim3, name=\"hidden3\")(h1)\n",
        "    #h3 = BatchNormalization()(h3)\n",
        "    #h3 = LeakyReLU(alpha=0.1)(h3)\n",
        "    #h3 = Dropout(rate=0.01)(h3)\n",
        "    #dqn_out = Dense(output_dim, activation='linear', name=\"dqn_output\")(h3)\n",
        "    \n",
        "    dqn_out = Dense(output_dim, activation='linear', name=\"dqn_output\")(h2)\n",
        "    \n",
        "    dqn = Model(dqn_in, dqn_out, name=\"DQN\")\n",
        "    dqn.compile(loss='mean_squared_error', optimizer='adam')\n",
        "    return dqn"
      ],
      "metadata": {
        "id": "GWJ0qt8NA4IH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# agent's action choice (0=hold,1=sell,2=buy)\n",
        "def chooseaction(q,explore):\n",
        "    # choose action of largest q\n",
        "    a = np.argmax(q)\n",
        "    # exploration\n",
        "    if np.random.rand(1) < explore:\n",
        "        amax = a\n",
        "        while a==amax:\n",
        "            a = np.random.randint(0,Na)\n",
        "    return a"
      ],
      "metadata": {
        "id": "geDFCSDoA4LP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## simulation: multiple episodes"
      ],
      "metadata": {
        "id": "Xa_4he5OBtUI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# run simulation for one episode\n",
        "def run_episode(dqn, initial_budget, initial_num_stocks, prices, hist):\n",
        "    budget = initial_budget # The state is a hist + 2 dimensional vector. You’ll force it to be a NumPy matrix.\n",
        "    # Calculates the portfolio value\n",
        "    num_stocks = initial_num_stocks\n",
        "    share_value = 0\n",
        "    for i in range(len(prices) - hist - 1):\n",
        "        # Initializes values that depend on computing the net worth of a portfolio\n",
        "        if i % 1000 == 0:\n",
        "            print('progress {:.2f}%'.format(float(100*i) / (len(prices) - hist - 1)))\n",
        "        # current state: busget, share, current & lag prices\n",
        "        current_state = np.asmatrix(np.hstack((budget, num_stocks, prices[i:i+hist])))\n",
        "        current_portfolio = budget + num_stocks * share_value\n",
        "        \n",
        "        # get Q-values for a=0 (continue) and a=1 (stop)\n",
        "        Q1 = dqn.predict(current_state)\n",
        "        # choose action\n",
        "        a = chooseaction(Q1,explore)  \n",
        "        # update share value\n",
        "        share_value = prices[i+hist]\n",
        "        \n",
        "        # update share and num_stocks based on action choice\n",
        "        if a == 2 and budget >= share_value: # buy\n",
        "            budget -= share_value\n",
        "            num_stocks += 1\n",
        "        elif a == 1 and num_stocks > 0: # sell\n",
        "            budget += share_value\n",
        "            num_stocks -= 1\n",
        "        else:\n",
        "            action = 0\n",
        "        \n",
        "        # next state\n",
        "        next_state = np.asmatrix(np.hstack((budget, num_stocks, prices[i+1:i+hist+1])))\n",
        "        next_portfolio = budget + num_stocks * share_value\n",
        "        \n",
        "        # reward\n",
        "        r = next_portfolio-current_portfolio\n",
        "        \n",
        "        # get max of Q(s(t+1))  \n",
        "        Q2 = dqn.predict(next_state)\n",
        "        maxQ2 = np.max(Q2)\n",
        "        # get target Q1 (only change value of Q1 for chosen action a)\n",
        "        targetQ1 = Q1\n",
        "        targetQ1[0,a] = r+gamma*maxQ2\n",
        "        # update Q0 (Q-learning)\n",
        "        dqn_hist = dqn.fit(current_state,targetQ1,verbose=0)\n",
        "    portfolio = budget + num_stocks * share_value\n",
        "    return portfolio"
      ],
      "metadata": {
        "id": "JKw98FC6A4Ol"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def test_episode(dqn, initial_budget, initial_num_stocks, prices, hist):\n",
        "    budget = initial_budget # The state is a hist + 2 dimensional vector. You’ll force it to be a NumPy matrix.\n",
        "    # Calculates the portfolio value\n",
        "    num_stocks = initial_num_stocks\n",
        "    share_value = 0\n",
        "    for i in range(len(prices) - hist - 1):\n",
        "        # Initializes values that depend on computing the net worth of a portfolio\n",
        "        if i % 1000 == 0:\n",
        "            print('progress {:.2f}%'.format(float(100*i) / (len(prices) - hist - 1)))\n",
        "        # current state: busget, share, current & lag prices\n",
        "        current_state = np.asmatrix(np.hstack((budget, num_stocks, prices[i:i+hist])))\n",
        "        current_portfolio = budget + num_stocks * share_value\n",
        "        \n",
        "        # get Q-values for a=0 (continue) and a=1 (stop)\n",
        "        Q1 = dqn.predict(current_state)\n",
        "        # choose action\n",
        "        a = chooseaction(Q1,explore)\n",
        "        # update share value\n",
        "        share_value = float(prices[i + hist])\n",
        "        \n",
        "        # update share and num_stocks based on action choice\n",
        "        if a == 2 and budget >= share_value: # buy\n",
        "            budget -= share_value\n",
        "            num_stocks += 1\n",
        "        elif a == 1 and num_stocks > 0: # sell\n",
        "            budget += share_value\n",
        "            num_stocks -= 1\n",
        "        else:\n",
        "            action = 0\n",
        "\n",
        "    portfolio = budget + num_stocks * share_value\n",
        "    return portfolio"
      ],
      "metadata": {
        "id": "d7w8pol6A4SK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## set initial parameters"
      ],
      "metadata": {
        "id": "vbT7-9jHB1fD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# learning rate: slow learning makes Q-curve smoother (<0.005)\n",
        "alpha = 0.01\n",
        "# discount rate: option value becomes more distinct for large discount\n",
        "gamma = 0.99\n",
        "# epsilon-greedy search: need sufficiently large exploration (>0.2)\n",
        "explore = 0.3"
      ],
      "metadata": {
        "id": "clOsW8HRA4Vk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Build Deep Q-network"
      ],
      "metadata": {
        "id": "UReN9NgAB5QV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dqn = build_dqn()"
      ],
      "metadata": {
        "id": "5lRyrXheB4Qz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run simulation (July, 1992 ~ July, 2016) "
      ],
      "metadata": {
        "id": "otbg0Q-xCAgi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#prices = get_prices('MSFT', '1992-07-22', '2016-07-22')\n",
        "stock_hist = yf.download('^GSPC', start='1992-07-22', end='2016-07-22')\n",
        "prices = stock_hist['Close']\n",
        "plot_prices(prices)\n",
        "actions = ['Buy', 'Sell', 'Hold']\n",
        "budget = 10000\n",
        "num_stocks = 0\n",
        "\n",
        "num_episodes = 20\n",
        "final_portfolios = list()\n",
        "final_portfolios.append(budget)\n",
        "for i in range(num_episodes):\n",
        "    # reset budget & num_shares for each episode\n",
        "    budget = 10000\n",
        "    num_stocks = 0\n",
        "    # Stores the portfolio worth of each run in this array\n",
        "    # Runs this simulation\n",
        "    final_portfolio = run_episode(dqn, budget, num_stocks, prices, hist)\n",
        "    final_portfolios.append(final_portfolio)\n",
        "    print('Final portfolio: ${}'.format(final_portfolio))\n",
        "    plt.title('Final Portfolio Value')\n",
        "    plt.xlabel('Simulation #')\n",
        "    plt.ylabel('Net worth')\n",
        "    plt.plot(final_portfolios)\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "rYoVWSnzB4Uc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Out-Of-Sample Test (July, 2016 ~ July, 2019)"
      ],
      "metadata": {
        "id": "mMTV3dmwCV51"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# reset budget & num_shares for each episode\n",
        "budget = 10000\n",
        "num_stocks = 0\n",
        "stock_hist = yf.download('^GSPC', start='2016-07-23', end='2019-07-22')\n",
        "prices = stock_hist['Close']\n",
        "test_portfolio = test_episode(dqn, budget, num_stocks, prices, hist)\n",
        "print(\"Return(%): \", 100*test_portfolio/budget)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sK-0jQu9B4XS",
        "outputId": "989572b7-443a-4d98-d2d7-e3515cb16216"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n",
            "progress 0.00%\n",
            "Return(%):  128.0311083984375\n"
          ]
        }
      ]
    }
  ]
}