kernel1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "182a3810-aab7-4d65-8ff7-0ad50183d769",
    "_uuid": "c8c2cca5ed3dcf9ff50ba2da4c9db56329bfc2cb"
   },
   "source": [
    "  \n",
    "**Description:** \n",
    "\n",
    "This notebook demonstrates the future price prediction for different stocks using recurrent neural networks in tensorflow. Recurrent neural networks with basic, LSTM or GRU cells are implemented. \n",
    "\n",
    "**Outline:**\n",
    "\n",
    "1. [Libraries and settings](#1-bullet)\n",
    "2. [Analyze data](#2-bullet)\n",
    "3. [Manipulate data](#3-bullet)\n",
    "4. [Model and validate data](#4-bullet)\n",
    "5. [Predictions](#5-bullet)\n",
    "\n",
    "**Reference:**  \n",
    "\n",
    "[LSTM_Stock_prediction-20170507 by BenF](https://www.kaggle.com/benjibb/lstm-stock-prediction-20170507/notebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "1426ca5e-ed0c-4766-ac74-cd1dd38c680c",
    "_uuid": "8ec772a02eb88c3446fe0f689b2e1ee8ec9deb6b"
   },
   "source": [
    "# 1. Libraries and settings <a class=\"anchor\" id=\"1-bullet\"></a> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "5a142d3f-96b5-4378-bb01-903367721e08",
    "_uuid": "78e6c0015907947ff5fade7ad76317633d27177a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import sklearn\n",
    "import sklearn.preprocessing\n",
    "import datetime\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "\n",
    "# split data in 80%/10%/10% train/validation/test sets\n",
    "valid_set_size_percentage = 10 \n",
    "test_set_size_percentage = 10 \n",
    "\n",
    "#display parent directory and working directory\n",
    "print(os.path.dirname(os.getcwd())+':', os.listdir(os.path.dirname(os.getcwd())));\n",
    "print(os.getcwd()+':', os.listdir(os.getcwd()));\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "5416ce8a-24ff-4903-a09b-b2941eae71d9",
    "_uuid": "9422490f417bc1fcac8e6a0a392faa7451173dac"
   },
   "source": [
    "# 2. Analyze data <a class=\"anchor\" id=\"2-bullet\"></a> \n",
    "- load stock prices from prices-split-adjusted.csv\n",
    "- analyze data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "60dbc696-dda8-476e-8227-010a79df3aa2",
    "_uuid": "aef1074dd4f86a9fe5a380c83994123ce56cce9d",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import all stock prices \n",
    "df = pd.read_csv(\"../input/prices-split-adjusted.csv\", index_col = 0)\n",
    "df.info()\n",
    "df.head()\n",
    "\n",
    "# number of different stocks\n",
    "print('\\nnumber of different stocks: ', len(list(set(df.symbol))))\n",
    "print(list(set(df.symbol))[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "962ea6e1-51ff-4bba-ad30-77ca5c765d59",
    "_uuid": "c06e709a1bfecb963213943dc7ce8cd559780d9b",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "d1fc9207-adcd-44f4-85a1-065d34fd82f2",
    "_uuid": "60ee56dd22397b9c545ea7dcc318f5c733d419ba",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "8e1e0b13-c1da-4fe1-b971-bb2ebcad453b",
    "_uuid": "ef59e62b988d2b0bfd00f834a7ecc17cb927dfc9",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "ce84999a-306f-4586-b8d6-dc3a0ab9ab67",
    "_uuid": "1235c99db35b562f09d6fb5721368bcab68e307d",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 5));\n",
    "plt.subplot(1,2,1);\n",
    "plt.plot(df[df.symbol == 'EQIX'].open.values, color='red', label='open')\n",
    "plt.plot(df[df.symbol == 'EQIX'].close.values, color='green', label='close')\n",
    "plt.plot(df[df.symbol == 'EQIX'].low.values, color='blue', label='low')\n",
    "plt.plot(df[df.symbol == 'EQIX'].high.values, color='black', label='high')\n",
    "plt.title('stock price')\n",
    "plt.xlabel('time [days]')\n",
    "plt.ylabel('price')\n",
    "plt.legend(loc='best')\n",
    "#plt.show()\n",
    "\n",
    "plt.subplot(1,2,2);\n",
    "plt.plot(df[df.symbol == 'EQIX'].volume.values, color='black', label='volume')\n",
    "plt.title('stock volume')\n",
    "plt.xlabel('time [days]')\n",
    "plt.ylabel('volume')\n",
    "plt.legend(loc='best');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "73ad0def-f162-42b9-a1d6-aac45be27e81",
    "_uuid": "777e047511205f929abf0867c7c8a2edee2531ca"
   },
   "source": [
    "# 3. Manipulate data <a class=\"anchor\" id=\"3-bullet\"></a> \n",
    "- choose a specific stock\n",
    "- drop feature: volume\n",
    "- normalize stock data\n",
    "- create train, validation and test data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "e577ab98-344e-4a8e-88c2-09a7d45834b9",
    "_uuid": "3333a7ef79ad756a4b0190e8aef2ed8b3624d7d8",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function for min-max normalization of stock\n",
    "def normalize_data(df):\n",
    "    min_max_scaler = sklearn.preprocessing.MinMaxScaler()\n",
    "    df['open'] = min_max_scaler.fit_transform(df.open.values.reshape(-1,1))\n",
    "    df['high'] = min_max_scaler.fit_transform(df.high.values.reshape(-1,1))\n",
    "    df['low'] = min_max_scaler.fit_transform(df.low.values.reshape(-1,1))\n",
    "    df['close'] = min_max_scaler.fit_transform(df['close'].values.reshape(-1,1))\n",
    "    return df\n",
    "\n",
    "# function to create train, validation, test data given stock data and sequence length\n",
    "def load_data(stock, seq_len):\n",
    "    data_raw = stock.as_matrix() # convert to numpy array\n",
    "    data = []\n",
    "    \n",
    "    # create all possible sequences of length seq_len\n",
    "    for index in range(len(data_raw) - seq_len): \n",
    "        data.append(data_raw[index: index + seq_len])\n",
    "    \n",
    "    data = np.array(data);\n",
    "    valid_set_size = int(np.round(valid_set_size_percentage/100*data.shape[0]));  \n",
    "    test_set_size = int(np.round(test_set_size_percentage/100*data.shape[0]));\n",
    "    train_set_size = data.shape[0] - (valid_set_size + test_set_size);\n",
    "    \n",
    "    x_train = data[:train_set_size,:-1,:]\n",
    "    y_train = data[:train_set_size,-1,:]\n",
    "    \n",
    "    x_valid = data[train_set_size:train_set_size+valid_set_size,:-1,:]\n",
    "    y_valid = data[train_set_size:train_set_size+valid_set_size,-1,:]\n",
    "    \n",
    "    x_test = data[train_set_size+valid_set_size:,:-1,:]\n",
    "    y_test = data[train_set_size+valid_set_size:,-1,:]\n",
    "    \n",
    "    return [x_train, y_train, x_valid, y_valid, x_test, y_test]\n",
    "\n",
    "# choose one stock\n",
    "df_stock = df[df.symbol == 'EQIX'].copy()\n",
    "df_stock.drop(['symbol'],1,inplace=True)\n",
    "df_stock.drop(['volume'],1,inplace=True)\n",
    "\n",
    "cols = list(df_stock.columns.values)\n",
    "print('df_stock.columns.values = ', cols)\n",
    "\n",
    "# normalize stock\n",
    "df_stock_norm = df_stock.copy()\n",
    "df_stock_norm = normalize_data(df_stock_norm)\n",
    "\n",
    "# create train, test data\n",
    "seq_len = 20 # choose sequence length\n",
    "x_train, y_train, x_valid, y_valid, x_test, y_test = load_data(df_stock_norm, seq_len)\n",
    "print('x_train.shape = ',x_train.shape)\n",
    "print('y_train.shape = ', y_train.shape)\n",
    "print('x_valid.shape = ',x_valid.shape)\n",
    "print('y_valid.shape = ', y_valid.shape)\n",
    "print('x_test.shape = ', x_test.shape)\n",
    "print('y_test.shape = ',y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "27e07a3b-89ea-48cc-bd86-023785634d1e",
    "_uuid": "c5ce171e4160f5e0175e4e5ab8e242f181262440",
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 5));\n",
    "plt.plot(df_stock_norm.open.values, color='red', label='open')\n",
    "plt.plot(df_stock_norm.close.values, color='green', label='low')\n",
    "plt.plot(df_stock_norm.low.values, color='blue', label='low')\n",
    "plt.plot(df_stock_norm.high.values, color='black', label='high')\n",
    "#plt.plot(df_stock_norm.volume.values, color='gray', label='volume')\n",
    "plt.title('stock')\n",
    "plt.xlabel('time [days]')\n",
    "plt.ylabel('normalized price/volume')\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "4375362b-7a7c-4572-947c-91f46c4fd77c",
    "_uuid": "a495f69f3a265f419cd5aa0a4119a8b0392365fd"
   },
   "source": [
    "# 4. Model and validate data <a class=\"anchor\" id=\"4-bullet\"></a> \n",
    "- RNNs with basic, LSTM, GRU cells\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "dde96de3-fef3-45cf-959c-22171a28a3b6",
    "_uuid": "b5015613883c52563da5f8e5be8d6adeb274e420",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Basic Cell RNN in tensorflow\n",
    "\n",
    "index_in_epoch = 0;\n",
    "perm_array  = np.arange(x_train.shape[0])\n",
    "np.random.shuffle(perm_array)\n",
    "\n",
    "# function to get the next batch\n",
    "def get_next_batch(batch_size):\n",
    "    global index_in_epoch, x_train, perm_array   \n",
    "    start = index_in_epoch\n",
    "    index_in_epoch += batch_size\n",
    "    \n",
    "    if index_in_epoch > x_train.shape[0]:\n",
    "        np.random.shuffle(perm_array) # shuffle permutation array\n",
    "        start = 0 # start next epoch\n",
    "        index_in_epoch = batch_size\n",
    "        \n",
    "    end = index_in_epoch\n",
    "    return x_train[perm_array[start:end]], y_train[perm_array[start:end]]\n",
    "\n",
    "# parameters\n",
    "n_steps = seq_len-1 \n",
    "n_inputs = 4 \n",
    "n_neurons = 200 \n",
    "n_outputs = 4\n",
    "n_layers = 2\n",
    "learning_rate = 0.001\n",
    "batch_size = 50\n",
    "n_epochs = 100 \n",
    "train_set_size = x_train.shape[0]\n",
    "test_set_size = x_test.shape[0]\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.float32, [None, n_outputs])\n",
    "\n",
    "# use Basic RNN Cell\n",
    "layers = [tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.elu)\n",
    "          for layer in range(n_layers)]\n",
    "\n",
    "# use Basic LSTM Cell \n",
    "#layers = [tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons, activation=tf.nn.elu)\n",
    "#          for layer in range(n_layers)]\n",
    "\n",
    "# use LSTM Cell with peephole connections\n",
    "#layers = [tf.contrib.rnn.LSTMCell(num_units=n_neurons, \n",
    "#                                  activation=tf.nn.leaky_relu, use_peepholes = True)\n",
    "#          for layer in range(n_layers)]\n",
    "\n",
    "# use GRU cell\n",
    "#layers = [tf.contrib.rnn.GRUCell(num_units=n_neurons, activation=tf.nn.leaky_relu)\n",
    "#          for layer in range(n_layers)]\n",
    "                                                                     \n",
    "multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers)\n",
    "rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)\n",
    "\n",
    "stacked_rnn_outputs = tf.reshape(rnn_outputs, [-1, n_neurons]) \n",
    "stacked_outputs = tf.layers.dense(stacked_rnn_outputs, n_outputs)\n",
    "outputs = tf.reshape(stacked_outputs, [-1, n_steps, n_outputs])\n",
    "outputs = outputs[:,n_steps-1,:] # keep only last output of sequence\n",
    "                                              \n",
    "loss = tf.reduce_mean(tf.square(outputs - y)) # loss function = mean squared error \n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) \n",
    "training_op = optimizer.minimize(loss)\n",
    "                                              \n",
    "# run graph\n",
    "with tf.Session() as sess: \n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for iteration in range(int(n_epochs*train_set_size/batch_size)):\n",
    "        x_batch, y_batch = get_next_batch(batch_size) # fetch the next training batch \n",
    "        sess.run(training_op, feed_dict={X: x_batch, y: y_batch}) \n",
    "        if iteration % int(5*train_set_size/batch_size) == 0:\n",
    "            mse_train = loss.eval(feed_dict={X: x_train, y: y_train}) \n",
    "            mse_valid = loss.eval(feed_dict={X: x_valid, y: y_valid}) \n",
    "            print('%.2f epochs: MSE train/valid = %.6f/%.6f'%(\n",
    "                iteration*batch_size/train_set_size, mse_train, mse_valid))\n",
    "\n",
    "    y_train_pred = sess.run(outputs, feed_dict={X: x_train})\n",
    "    y_valid_pred = sess.run(outputs, feed_dict={X: x_valid})\n",
    "    y_test_pred = sess.run(outputs, feed_dict={X: x_test})\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "9035de79-0f5c-4b6b-bc86-0d8d4e2f392e",
    "_uuid": "a88583a9effb6f1f84e13b8853adca62578c2992"
   },
   "source": [
    "# 5. Predictions <a class=\"anchor\" id=\"5-bullet\"></a> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "412b33a3-1e02-4ed5-9846-9984022492bd",
    "_uuid": "2923055320991790fe0c8c0ffa3e75558e5c313a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "1bb898b9-943e-4e6f-b43d-8bdb6331cc4e",
    "_uuid": "6fa5eb60c8e8a5d3542547629c3e0df14e5c33a3",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ft = 0 # 0 = open, 1 = close, 2 = highest, 3 = lowest\n",
    "\n",
    "## show predictions\n",
    "plt.figure(figsize=(15, 5));\n",
    "plt.subplot(1,2,1);\n",
    "\n",
    "plt.plot(np.arange(y_train.shape[0]), y_train[:,ft], color='blue', label='train target')\n",
    "\n",
    "plt.plot(np.arange(y_train.shape[0], y_train.shape[0]+y_valid.shape[0]), y_valid[:,ft],\n",
    "         color='gray', label='valid target')\n",
    "\n",
    "plt.plot(np.arange(y_train.shape[0]+y_valid.shape[0],\n",
    "                   y_train.shape[0]+y_test.shape[0]+y_test.shape[0]),\n",
    "         y_test[:,ft], color='black', label='test target')\n",
    "\n",
    "plt.plot(np.arange(y_train_pred.shape[0]),y_train_pred[:,ft], color='red',\n",
    "         label='train prediction')\n",
    "\n",
    "plt.plot(np.arange(y_train_pred.shape[0], y_train_pred.shape[0]+y_valid_pred.shape[0]),\n",
    "         y_valid_pred[:,ft], color='orange', label='valid prediction')\n",
    "\n",
    "plt.plot(np.arange(y_train_pred.shape[0]+y_valid_pred.shape[0],\n",
    "                   y_train_pred.shape[0]+y_valid_pred.shape[0]+y_test_pred.shape[0]),\n",
    "         y_test_pred[:,ft], color='green', label='test prediction')\n",
    "\n",
    "plt.title('past and future stock prices')\n",
    "plt.xlabel('time [days]')\n",
    "plt.ylabel('normalized price')\n",
    "plt.legend(loc='best');\n",
    "\n",
    "plt.subplot(1,2,2);\n",
    "\n",
    "plt.plot(np.arange(y_train.shape[0], y_train.shape[0]+y_test.shape[0]),\n",
    "         y_test[:,ft], color='black', label='test target')\n",
    "\n",
    "plt.plot(np.arange(y_train_pred.shape[0], y_train_pred.shape[0]+y_test_pred.shape[0]),\n",
    "         y_test_pred[:,ft], color='green', label='test prediction')\n",
    "\n",
    "plt.title('future stock prices')\n",
    "plt.xlabel('time [days]')\n",
    "plt.ylabel('normalized price')\n",
    "plt.legend(loc='best');\n",
    "\n",
    "corr_price_development_train = np.sum(np.equal(np.sign(y_train[:,1]-y_train[:,0]),\n",
    "            np.sign(y_train_pred[:,1]-y_train_pred[:,0])).astype(int)) / y_train.shape[0]\n",
    "corr_price_development_valid = np.sum(np.equal(np.sign(y_valid[:,1]-y_valid[:,0]),\n",
    "            np.sign(y_valid_pred[:,1]-y_valid_pred[:,0])).astype(int)) / y_valid.shape[0]\n",
    "corr_price_development_test = np.sum(np.equal(np.sign(y_test[:,1]-y_test[:,0]),\n",
    "            np.sign(y_test_pred[:,1]-y_test_pred[:,0])).astype(int)) / y_test.shape[0]\n",
    "\n",
    "print('correct sign prediction for close - open price for train/valid/test: %.2f/%.2f/%.2f'%(\n",
    "    corr_price_development_train, corr_price_development_valid, corr_price_development_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "581be505-fdc8-4aec-a3a3-19cd28f4e324",
    "_uuid": "2dc96fc968aa367482f1b7dc2018848af84bdb75",
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}