Final_DLOB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Sequential, models\n",
    "from tensorflow.keras.layers import Conv2D, LeakyReLU, Input, MaxPooling2D, concatenate, Reshape, LSTM, Dense\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_x(data):\n",
    "    df1 = data[:40, :].T\n",
    "    return np.array(df1)\n",
    "\n",
    "def get_label(data):\n",
    "    lob = data[-5:, :].T\n",
    "    return lob\n",
    "\n",
    "def data_classification(X, Y, T):\n",
    "    [N, D] = X.shape\n",
    "    df = np.array(X)\n",
    "\n",
    "    dY = np.array(Y)\n",
    "\n",
    "    dataY = dY[T - 1:N]\n",
    "\n",
    "    dataX = np.zeros((N - T + 1, T, D))\n",
    "    for i in range(T, N + 1):\n",
    "        dataX[i - T] = df[i - T:i, :]\n",
    "\n",
    "    return dataX.reshape(dataX.shape + (1,)), dataY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dec_train = np.loadtxt('Train_Dst_NoAuction_DecPre_CF_7.txt')\n",
    "dec_test1 = np.loadtxt('Test_Dst_NoAuction_DecPre_CF_7.txt')\n",
    "dec_test2 = np.loadtxt('Test_Dst_NoAuction_DecPre_CF_8.txt')\n",
    "dec_test3 = np.loadtxt('Test_Dst_NoAuction_DecPre_CF_9.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(149, 254750)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dec_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_lob = prepare_x(dec_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.2615 , 0.00353, 0.2606 , ..., 0.00311, 0.2579 , 0.00128],\n",
       "       [0.2615 , 0.00211, 0.2606 , ..., 0.00138, 0.2588 , 0.00123],\n",
       "       [0.2614 , 0.00122, 0.2606 , ..., 0.00311, 0.2588 , 0.00123],\n",
       "       ...,\n",
       "       [0.3531 , 0.00775, 0.3527 , ..., 0.00044, 0.3515 , 0.005  ],\n",
       "       [0.3531 , 0.00775, 0.3527 , ..., 0.00496, 0.3514 , 0.01091],\n",
       "       [0.3532 , 0.002  , 0.3527 , ..., 0.02366, 0.3514 , 0.01091]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_lob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(254750, 40)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_lob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_label = get_label(dec_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2., 2., 2., 2., 2.],\n",
       "       [2., 2., 2., 2., 2.],\n",
       "       [3., 3., 2., 2., 2.],\n",
       "       ...,\n",
       "       [2., 2., 2., 2., 1.],\n",
       "       [2., 1., 1., 1., 1.],\n",
       "       [1., 1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(254750, 5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dec_test = np.hstack((dec_test1, dec_test2, dec_test3))\n",
    "\n",
    "# extract limit order book data from the FI-2010 dataset\n",
    "train_lob = prepare_x(dec_train)\n",
    "test_lob = prepare_x(dec_test)\n",
    "\n",
    "# extract label from the FI-2010 dataset\n",
    "train_label = get_label(dec_train)\n",
    "test_label = get_label(dec_test)\n",
    "\n",
    "# prepare training data. We feed past 100 observations into our algorithms and choose the prediction horizon. \n",
    "trainX_CNN, trainY_CNN = data_classification(train_lob, train_label, T=100)\n",
    "trainY_CNN = trainY_CNN[:,3] - 1\n",
    "trainY_CNN = to_categorical(trainY_CNN, 3)\n",
    "\n",
    "# prepare test data.\n",
    "testX_CNN, testY_CNN = data_classification(test_lob, test_label, T=100)\n",
    "testY_CNN = testY_CNN[:,3] - 1\n",
    "testY_CNN = to_categorical(testY_CNN, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = Input(shape=(100, 40, 1), name = \"input_shape\")\n",
    "layer1 = Conv2D(16, (1,2), strides = (1,2))(input_shape)\n",
    "layer1_act = LeakyReLU(alpha = 0.01)(layer1)\n",
    "layer2 = Conv2D(16, (4,1), padding = 'same')(layer1_act)\n",
    "layer2_act = LeakyReLU(alpha = 0.01)(layer2)\n",
    "layer3 = Conv2D(16, (4,1), padding = 'same')(layer2_act)\n",
    "layer3_act = LeakyReLU(alpha = 0.01)(layer3)\n",
    "\n",
    "layer1 = Conv2D(16, (1,2), strides = (1,2))(input_shape)\n",
    "layer1_act = LeakyReLU(alpha = 0.01)(layer1)\n",
    "layer2 = Conv2D(16, (4,1), padding = 'same')(layer1_act)\n",
    "layer2_act = LeakyReLU(alpha = 0.01)(layer2)\n",
    "layer2 = Conv2D(16, (4,1), padding = 'same')(layer2_act)\n",
    "layer2_act = LeakyReLU(alpha = 0.01)(layer2)\n",
    "\n",
    "layer3 = Conv2D(16, (1,2), strides = (1,2))(layer2_act)\n",
    "layer3_act = LeakyReLU(alpha = 0.01)(layer3)\n",
    "layer4 = Conv2D(16, (4,1), padding = 'same')(layer3_act)\n",
    "layer4_act = LeakyReLU(alpha = 0.01)(layer4)\n",
    "layer5 = Conv2D(16, (4,1), padding = 'same')(layer4_act)\n",
    "layer5_act = LeakyReLU(alpha = 0.01)(layer5)\n",
    "\n",
    "layer6 = Conv2D(16, (1,10))(layer5_act)\n",
    "layer6_act = LeakyReLU(alpha = 0.01)(layer6)\n",
    "layer7 = Conv2D(16, (4,1), padding = 'same')(layer6_act)\n",
    "layer7_act = LeakyReLU(alpha = 0.01)(layer7)\n",
    "layer8 = Conv2D(16, (4,1), padding = 'same')(layer7_act)\n",
    "layer8_act = LeakyReLU(alpha = 0.01)(layer8)\n",
    "\n",
    "#inception\n",
    "\n",
    "inception_layer1a = Conv2D(32, (1,1), padding = 'same')(layer8_act)\n",
    "inception_layer1a_act = LeakyReLU(alpha = 0.01) (inception_layer1a)\n",
    "inception_layer1b = Conv2D(32, (3,1), padding = 'same')(inception_layer1a_act)\n",
    "inception_layer1b_act = LeakyReLU(alpha = 0.01) (inception_layer1b)\n",
    "\n",
    "inception_layer2a = Conv2D(32, (1,1), padding = 'same')(layer8_act)\n",
    "inception_layer2a_act = LeakyReLU(alpha = 0.01) (inception_layer2a)\n",
    "inception_layer2b = Conv2D(32, (5,1), padding = 'same')(inception_layer2a_act)\n",
    "inception_layer2b_act = LeakyReLU(alpha = 0.01) (inception_layer2b)\n",
    "\n",
    "inception_layer3a = MaxPooling2D((3,1), strides = (1,1), padding = 'same')(layer8_act)\n",
    "inception_layer3a_act = LeakyReLU(alpha = 0.01) (inception_layer2a)\n",
    "inception_layer3b = Conv2D(32, (1,1), padding = 'same')(inception_layer3a_act)\n",
    "inception_layer3b_act = LeakyReLU(alpha = 0.01) (inception_layer3b)\n",
    "\n",
    "inception_concatenate = concatenate([inception_layer1b_act, inception_layer2b_act, inception_layer3b_act])\n",
    "\n",
    "inception_reshape = Reshape((int(inception_concatenate.shape[1]),int(inception_concatenate.shape[3])))(inception_concatenate)\n",
    "\n",
    "layer9 = LSTM(64)(inception_reshape)\n",
    "outputs = Dense(3, activation = 'softmax')(layer9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Model(inputs = input_shape, outputs = outputs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon = 1\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "model.compile(optimizer = optimizers.Adam(learning_rate = learning_rate, epsilon = epsilon),\n",
    "             loss = 'categorical_crossentropy',metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_shape (InputLayer)        [(None, 100, 40, 1)] 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 100, 20, 16)  48          input_shape[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)       (None, 100, 20, 16)  0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 100, 20, 16)  1040        leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)       (None, 100, 20, 16)  0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 100, 20, 16)  1040        leaky_re_lu_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)       (None, 100, 20, 16)  0           conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 100, 10, 16)  528         leaky_re_lu_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)       (None, 100, 10, 16)  0           conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 100, 10, 16)  1040        leaky_re_lu_6[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_7 (LeakyReLU)       (None, 100, 10, 16)  0           conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 100, 10, 16)  1040        leaky_re_lu_7[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_8 (LeakyReLU)       (None, 100, 10, 16)  0           conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 100, 1, 16)   2576        leaky_re_lu_8[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_9 (LeakyReLU)       (None, 100, 1, 16)   0           conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 100, 1, 16)   1040        leaky_re_lu_9[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_10 (LeakyReLU)      (None, 100, 1, 16)   0           conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 100, 1, 16)   1040        leaky_re_lu_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_11 (LeakyReLU)      (None, 100, 1, 16)   0           conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 100, 1, 32)   544         leaky_re_lu_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 100, 1, 32)   544         leaky_re_lu_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_12 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_14 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_16 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 100, 1, 32)   3104        leaky_re_lu_12[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 100, 1, 32)   5152        leaky_re_lu_14[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 100, 1, 32)   1056        leaky_re_lu_16[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_13 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_15 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_17 (LeakyReLU)      (None, 100, 1, 32)   0           conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 100, 1, 96)   0           leaky_re_lu_13[0][0]             \n",
      "                                                                 leaky_re_lu_15[0][0]             \n",
      "                                                                 leaky_re_lu_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "reshape (Reshape)               (None, 100, 96)      0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (None, 64)           41216       reshape[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 3)            195         lstm[0][0]                       \n",
      "==================================================================================================\n",
      "Total params: 61,203\n",
      "Trainable params: 61,203\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 254651 samples, validate on 139488 samples\n",
      "Epoch 1/200\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-ec86727766e2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainX_CNN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainY_CNN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtestX_CNN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtestY_CNN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    729\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m                 \u001b[0mtraining_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m                 total_epochs=epochs)\n\u001b[0m\u001b[1;32m    325\u001b[0m             \u001b[0mcbks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_logs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py\u001b[0m in \u001b[0;36mrun_one_epoch\u001b[0;34m(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)\u001b[0m\n\u001b[1;32m    121\u001b[0m         step=step, mode=mode, size=current_batch_size) as batch_logs:\n\u001b[1;32m    122\u001b[0m       \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m       \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mStopIteration\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOutOfRangeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0;31m# TODO(kaftan): File bug about tf function and errors.OutOfRangeError?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mexecution_function\u001b[0;34m(input_fn)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[0;31m# `numpy` translates Tensors to values in Eager mode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     return nest.map_structure(_non_none_constant_value,\n\u001b[0;32m---> 86\u001b[0;31m                               distributed_function(input_fn))\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mexecution_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_counter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcalled_without_tracing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    485\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 487\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    488\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    489\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1821\u001b[0m     \u001b[0;34m\"\"\"Calls a graph function specialized to the inputs.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1822\u001b[0m     \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1823\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1824\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1825\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1139\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1140\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1141\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1143\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1222\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1224\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1225\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(trainX_CNN, trainY_CNN, epochs=200, batch_size=64, verbose=2, validation_data=(testX_CNN, testY_CNN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}