Prediction_original_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspirational Notebooks\n",
    "### Generating new festures according to these notebooks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* https://www.kaggle.com/nuhsikander/lgbm-new-features-corrected\n",
    "* https://www.kaggle.com/rteja1113/lightgbm-with-count-features\n",
    "* https://www.kaggle.com/aharless/swetha-s-xgboost-revised\n",
    "* https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pandas import read_csv\n",
    "import sklearn\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "plt.rc(\"font\", size=14)\n",
    "sns.set(style=\"white\")\n",
    "sns.set(style=\"whitegrid\", color_codes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import Adagrad\n",
    "from keras.optimizers import Adadelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200000 entries, 0 to 199999\n",
      "Data columns (total 8 columns):\n",
      "ip                 200000 non-null int64\n",
      "app                200000 non-null int64\n",
      "device             200000 non-null int64\n",
      "os                 200000 non-null int64\n",
      "channel            200000 non-null int64\n",
      "click_time         200000 non-null object\n",
      "attributed_time    348 non-null object\n",
      "is_attributed      200000 non-null int64\n",
      "dtypes: int64(6), object(2)\n",
      "memory usage: 12.2+ MB\n"
     ]
    }
   ],
   "source": [
    "# df_train= pd.read_csv('data/train.csv',nrows=200000, parse_dates=['click_time'])\n",
    "df_train = pd.read_csv('data/train.csv',nrows=200000) #train data subset, original too large\n",
    "df_train.dropna()\n",
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>83230</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:32:21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17357</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:33:34</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35810</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:34:12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45745</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>478</td>\n",
       "      <td>2017-11-06 14:34:52</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>161007</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:35:08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18787</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:36:26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>103022</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:37:44</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>114221</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:37:59</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>165970</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>379</td>\n",
       "      <td>2017-11-06 14:38:10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>74544</td>\n",
       "      <td>64</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>459</td>\n",
       "      <td>2017-11-06 14:38:23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel           click_time attributed_time  \\\n",
       "0   83230    3       1  13      379  2017-11-06 14:32:21             NaN   \n",
       "1   17357    3       1  19      379  2017-11-06 14:33:34             NaN   \n",
       "2   35810    3       1  13      379  2017-11-06 14:34:12             NaN   \n",
       "3   45745   14       1  13      478  2017-11-06 14:34:52             NaN   \n",
       "4  161007    3       1  13      379  2017-11-06 14:35:08             NaN   \n",
       "5   18787    3       1  16      379  2017-11-06 14:36:26             NaN   \n",
       "6  103022    3       1  23      379  2017-11-06 14:37:44             NaN   \n",
       "7  114221    3       1  19      379  2017-11-06 14:37:59             NaN   \n",
       "8  165970    3       1  13      379  2017-11-06 14:38:10             NaN   \n",
       "9   74544   64       1  22      459  2017-11-06 14:38:23             NaN   \n",
       "\n",
       "   is_attributed  \n",
       "0              0  \n",
       "1              0  \n",
       "2              0  \n",
       "3              0  \n",
       "4              0  \n",
       "5              0  \n",
       "6              0  \n",
       "7              0  \n",
       "8              0  \n",
       "9              0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    199652\n",
       "1       348\n",
       "Name: is_attributed, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['is_attributed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEFCAYAAAAmIwo/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHI1JREFUeJzt3XtsVGX+x/H3dFoQOq21EggNFItC\nlLvtCO5auqvYDGxE5FpgLa6oCMvFuguCFVr4tVzMsiWx3IvZkCrLRVgl0XihWWxasMhowRZhWRcq\nt0URlXYQ2s6c3x8bJ9u11NGnM4X280pIOg/fc/o9zcl85pxnzjk2y7IsREREDIS1dAMiInLjU5iI\niIgxhYmIiBhTmIiIiDGFiYiIGAtv6QZagtvtbukWRERuSElJSY2Ot8kwgWv/QUREpHFNfRDXaS4R\nETGmMBEREWMKExERMaYwERERYwoTERExpjARERFjQftqcF1dHZmZmZw5c4ba2lpmzJjBHXfcwYIF\nC7DZbPTq1Yvs7GzCwsJYvXo1e/fuJTw8nMzMTAYMGEBVVZVxrYiIhEbQ3nF3795NTEwMW7ZsYdOm\nTeTk5LB8+XIyMjLYsmULlmVRVFREZWUlBw4cYMeOHeTl5bFkyRIA41oREQmdoB2ZDB8+HJfLBYBl\nWdjtdiorKxk8eDAAKSkplJaWkpCQQHJyMjabjbi4OLxeLxcvXjSuTU1NbbI/XQUvItJ8ghYmkZGR\nANTU1DBnzhwyMjJ48cUXsdls/v+vrq6mpqaGmJiYBstVV1djWZZR7Y8xvQL+4JzpRstL6+R8aX1L\ntyASNC12Bfy5c+eYMmUKo0aNYuTIkQ3mMTweD9HR0TgcDjweT4PxqKgo41oREQmdoIXJhQsXmDp1\nKvPmzWPcuHEA9OnTh7KyMgCKi4txOp0kJiZSUlKCz+fj7Nmz+Hw+YmNjjWtFRCR0gnaaa/369Vy6\ndIm1a9eydu1aAF544QVyc3PJy8ujZ8+euFwu7HY7TqeTtLQ0fD4fWVlZAMyfP59Fixb97FoREQkd\nm2VZVks3EWput1tzJhIUmjOR1qyp905djCEiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJM\nYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEi\nIiLGFCYiImIsaM+ABzh06BArV66ksLCQZ599lgsXLgBw5swZBg4cyKpVq5gxYwZff/01ERERtG/f\nnk2bNlFVVcWCBQuw2Wz06tWL7OxswsLCWL16NXv37iU8PJzMzEwGDBhwzVoREQmdoIVJQUEBu3fv\npkOHDgCsWrUKgG+//ZYpU6bw/PPPA1BVVcWbb76JzWbzL7t8+XIyMjIYMmQIWVlZFBUVERcXx4ED\nB9ixYwfnzp1j9uzZ7Ny5s9Ha1NTUYG2WiIg0ImhhEh8fT35+Ps8991yD8fz8fB599FE6d+7MhQsX\nuHTpEtOnT+fSpUtMmzaN+++/n8rKSgYPHgxASkoKpaWlJCQkkJycjM1mIy4uDq/Xy8WLFxutDSRM\n3G5382+0tHnar6StClqYuFwuTp8+3WDsq6++Yv/+/f6jkrq6OqZOncqUKVP49ttvmTRpEgMGDMCy\nLP+RSmRkJNXV1dTU1BATE+Nf1/fjjdUGIikpyWj7Dm4uMFpeWifT/UrketbUh6WQTi68/fbbPPTQ\nQ9jtdgA6derExIkTCQ8P59Zbb+Wuu+7ixIkTDeY8PB4P0dHROBwOPB5Pg/GoqKhGa0VEJLRCGib7\n9+8nJSXF/3rfvn0888wzwH+C4Pjx4/Ts2ZM+ffpQVlYGQHFxMU6nk8TEREpKSvD5fJw9exafz0ds\nbGyjtSIiElpB/TbX/zpx4gTdu3f3v/7Vr35FSUkJEyZMICwsjD/84Q/ExsYyf/58Fi1aRF5eHj17\n9sTlcmG323E6naSlpeHz+cjKygJotFZERELLZlmW1dJNhJrb7TafM5kzvZm6kdbE+dL6lm5BJGia\neu/UBRkiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIi\nxhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLGghomhw4dIj09HYAj\nR44wdOhQ0tPTSU9P56233gJg9erVjBs3jokTJ3L48GEAqqqqmDRpEpMnTyY7Oxufz/eTa0VEJHTC\ng7XigoICdu/eTYcOHQCorKzk8ccfZ+rUqf6ayspKDhw4wI4dOzh37hyzZ89m586dLF++nIyMDIYM\nGUJWVhZFRUXExcUFXJuamhqszRIRkUYELUzi4+PJz8/nueeeA6CiooITJ05QVFREjx49yMzMxO12\nk5ycjM1mIy4uDq/Xy8WLF6msrGTw4MEApKSkUFpaSkJCQsC1gYSJ2+0O1qZLG6b9StqqoIWJy+Xi\n9OnT/tcDBgxg/Pjx9OvXj3Xr1rFmzRqioqKIiYnx10RGRlJdXY1lWdhstgZjNTU1AdcGIikpyWj7\nDm4uMFpeWifT/UrketbUh6WQTcCnpqbSr18//89HjhzB4XDg8Xj8NR6Ph6ioKMLCwhqMRUdH/6Ra\nEREJrZCFyRNPPOGfNN+/fz99+/YlMTGRkpISfD4fZ8+exefzERsbS58+fSgrKwOguLgYp9P5k2pF\nRCS0gnaa638tXryYnJwcIiIi6NSpEzk5OTgcDpxOJ2lpafh8PrKysgCYP38+ixYtIi8vj549e+Jy\nubDb7QHXiohIaNksy7JauolQc7vd5nMmc6Y3UzfSmjhfWt/SLYgETVPvnbpoUUREjClMRETEmMJE\nRESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCRERE\njClMRETEmMJERESMKUxERMSYwkRERIwF9bG9hw4dYuXKlRQWFvLpp5+Sk5OD3W6nXbt2vPjii3Tq\n1Inc3Fw++ugjIiMjAVi7di11dXXMnTuXK1eu0LlzZ5YvX06HDh3Yvn07W7duJTw8nBkzZnD//fdz\n8eLFRmtFRCR0gnZkUlBQwMKFC7l69SoAS5cuZdGiRRQWFpKamkpBQQEAlZWVbNq0icLCQgoLC4mK\nimLt2rU89NBDbNmyhT59+rBt2za+/PJLCgsL2bp1Ky+//DJ5eXnU1tY2WisiIqEVtDCJj48nPz/f\n/zovL4+77roLAK/XS/v27fH5fFRVVZGVlcXEiRN57bXXgP88Z3jo0KEApKSksG/fPg4fPszdd99N\nu3btiIqKIj4+nqNHjzZaKyIioRW001wul4vTp0/7X3fu3BmAjz76iFdeeYVXX32Vy5cv8+ijj/L4\n44/j9XqZMmUK/fr1o6amhqioKAAiIyOprq5uMPb9eE1NTaO1gXC73c21qSJ+2q+krQrqnMn/euut\nt1i3bh0bN24kNjbWHyDfz3Hce++9HD16FIfDgcfj4aabbsLj8RAdHe0f+57H4yEqKqrR2kAkJSUZ\nbcvBzQVGy0vrZLpfiVzPmvqwFLJvc73xxhu88sorFBYW0r17dwBOnjzJpEmT8Hq91NXV8dFHH9G3\nb18SExN5//33ASguLiYpKYkBAwbgdru5evUq1dXVfPbZZ/Tu3bvRWhERCa2QHJl4vV6WLl1K165d\nmT17NgD33HMPc+bMYdSoUUyYMIGIiAhGjRpFr169mDFjBvPnz2f79u3ccsst/PnPf6Zjx46kp6cz\nefJkLMvi2WefpX379o3WiohIaNksy7JauolQc7vd5qe55kxvpm6kNXG+tL6lWxAJmqbeO3XRooiI\nGFOYiIiIMYWJiIgYU5iIiIgxhYmIiBgLKExycnJ+MDZ//vxmb0ZERG5MTV5n8sILL3Dq1CkqKio4\nfvy4f7y+vj7g25aIiEjr12SYzJgxgzNnzrB06VJmzZrlH7fb7dx+++1Bb05ERG4MTYZJt27d6Nat\nG7t376ampobq6mq+v8bx8uXLxMTEhKRJERG5vgV0O5UNGzawYcOGBuFhs9koKioKWmMiInLjCChM\nduzYwZ49e4iNjQ12PyIicgMK6NtcXbt25eabbw52LyIicoMK6MjktttuY/LkyQwZMoR27dr5x/97\nUl5ERNqugMKkS5cudOnSJdi9iIjIDSqgMNERiIiINCWgMLnzzjux2WwNxjp37ux/wqGIiLRtAYXJ\n0aNH/T/X1dWxZ88eysvLg9aUiIjcWH7yjR4jIiIYMWIEH3zwQTD6ERGRG1BARyavv/66/2fLsjh+\n/DgRERE/utyhQ4dYuXIlhYWFVFVVsWDBAmw2G7169SI7O5uwsDBWr17N3r17CQ8PJzMzkwEDBjRL\nrYiIhE5A77plZWX+fwcOHABg1apVTS5TUFDAwoULuXr1KgDLly8nIyODLVu2YFkWRUVFVFZWcuDA\nAXbs2EFeXh5LlixplloREQmtgI5Mli9fTl1dHSdOnMDr9dKrVy/Cw5teND4+nvz8fJ577jkAKisr\nGTx4MAApKSmUlpaSkJBAcnIyNpuNuLg4vF4vFy9eNK5NTU392X8QERH56QIKk4qKCubMmUNMTAw+\nn48LFy6wZs0aBg4ceM1lXC4Xp0+f9r+2LMv/jbDIyEiqq6upqalpcL+v78dNawPhdrsDqhP5KbRf\nSVsVUJjk5uayatUqf3iUl5eTk5PDa6+9FvAv+u95DI/HQ3R0NA6HA4/H02A8KirKuDYQSUlJAffe\nmIObC4yWl9bJdL8SuZ419WEpoDmTy5cvNzgKGTRokH8uJFB9+vShrKwMgOLiYpxOJ4mJiZSUlODz\n+Th79iw+n4/Y2FjjWhERCa2Ajkxuvvlm9uzZw4MPPgjAnj17fvKzTObPn8+iRYvIy8ujZ8+euFwu\n7HY7TqeTtLQ0fD4fWVlZzVIrIiKhZbO+f9pVE06ePMnTTz/NN9984x/bunUrCQkJQW0uWNxut/lp\nrjnTm6kbaU2cL61v6RZEgqap986ATnMVFxfToUMH/v73v7N582ZiY2P9XxEWEREJKEy2b9/OX//6\nVzp27Midd97Jrl27eOWVV4Ldm4iI3CACCpO6uroGV7wHcvW7iIi0HQFNwD/44IM89thjjBgxAoB3\n332XYcOGBbUxERG5cQQUJvPmzePtt9/mww8/JDw8nClTpvi/2SUiIhJQmAAMHz6c4cOHB7MXERG5\nQen2uiIiYkxhIiIixhQmIiJiTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhIiIixhQmIiJiTGEiIiLG\nFCYiImJMYSIiIsYCvtFjc9i1axd/+9vfALh69SqffvopeXl5vPjii3Tt2hWA2bNn43Q6Wbx4MceO\nHaNdu3bk5ubSo0cPysvLWbp0KXa7neTkZGbNmoXP52u0VkREQiekYTJmzBjGjBkDwJIlSxg7diwV\nFRXMmzcPl8vlr3v33Xepra1l27ZtlJeXs2LFCtatW0d2djb5+fl0796dadOmceTIEU6fPt1orYiI\nhE6LnOb65JNP+Oc//0laWhqVlZXs3LmTyZMns2LFCurr63G73QwdOhSAQYMGUVFRQU1NDbW1tcTH\nx2Oz2UhOTmbfvn2N1oqISGiF9Mjkexs2bGDmzJkA3HfffTz44IN069aN7Oxstm7dSk1NDQ6Hw19v\nt9t/MBYZGcmpU6cara2vryc8vOlNc7vdzbxVItqvpO0KeZhcunSJEydOcO+99wIwduxYoqOjARg2\nbBjvvPMOUVFReDwe/zI+nw+Hw9FgzOPxEB0dzZUrV35Q+2NBApCUlGS0HQc3FxgtL62T6X4lcj1r\n6sNSyE9zffjhh/ziF78AwLIsHn74Yf79738DsH//fvr27UtiYiLFxcUAlJeX07t3bxwOBxEREXz+\n+edYlkVJSQlOp7PRWhERCa2QH5mcOHGCbt26AWCz2cjNzWXWrFncdNNN3H777UyYMAG73U5paSkT\nJ07EsiyWLVsG/GfSfu7cuXi9XpKTkxk4cCD9+/dvtFZERELHZlmW1dJNhJrb7TY/zTVnejN1I62J\n86X1Ld2CSNA09d6pixZFRMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwp\nTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMhfwZ\n8KNHj8bhcADQrVs30tLSWLp0KXa7neTkZGbNmoXP52Px4sUcO3aMdu3akZubS48ePSgvLw+4VkRE\nQiekYXL16lUsy6KwsNA/NmrUKPLz8+nevTvTpk3jyJEjnD59mtraWrZt20Z5eTkrVqxg3bp1ZGdn\nB1wrIiKhE9IwOXr0KN999x1Tp06lvr6e2bNnU1tbS3x8PADJycns27ePL7/8kqFDhwIwaNAgKioq\nqKmpCbhWRERCK6RhctNNN/HEE08wfvx4Tp48yVNPPUV0dLT//yMjIzl16hQ1NTX+U2EAdrv9B2NN\n1dbX1xMe3vSmud3uZtwykf/QfiVtVUjDJCEhgR49emCz2UhISCAqKopvvvnG//8ej4fo6GiuXLmC\nx+Pxj/t8PhwOR4Oxpmp/LEgAkpKSjLbl4OYCo+WldTLdr0SuZ019WArpt7lee+01VqxYAcD58+f5\n7rvv6NixI59//jmWZVFSUoLT6SQxMZHi4mIAysvL6d27Nw6Hg4iIiIBqRUQktEJ6ZDJu3Dief/55\nJk2ahM1mY9myZYSFhTF37ly8Xi/JyckMHDiQ/v37U1paysSJE7Esi2XLlgGwZMmSgGtFRCR0bJZl\nWS3dRKi53W7z01xzpjdTN9KaOF9a39ItiARNU++dumhRRESMKUxERMSYwkRERIwpTERExJjCRERE\njClMRETEmMJERESMKUxERMSYwkRERIwpTERExJjCREREjClMRETEmMJERESMKUxERMSYwkRERIwp\nTERExJjCREREjClMRETEWEifAV9XV0dmZiZnzpyhtraWGTNm0LVrV55++mluu+02ACZNmsRvfvMb\nVq9ezd69ewkPDyczM5MBAwZQVVXFggULsNls9OrVi+zsbMLCwhqtFRGR0AlpmOzevZuYmBj+9Kc/\n8c033/DII48wc+ZMHn/8caZOneqvq6ys5MCBA+zYsYNz584xe/Zsdu7cyfLly8nIyGDIkCFkZWVR\nVFREXFxco7UiIhI6IQ2T4cOH43K5ALAsC7vdTkVFBSdOnKCoqIgePXqQmZmJ2+0mOTkZm81GXFwc\nXq+XixcvUllZyeDBgwFISUmhtLSUhISERmtjY2NDuWkiIm1aSMMkMjISgJqaGubMmUNGRga1tbWM\nHz+efv36sW7dOtasWUNUVBQxMTENlquursayLGw2W4OxmpqaRmt/LEzcbncQtlDaOu1X0laFNEwA\nzp07x8yZM5k8eTIjR47k0qVLREdHA5CamkpOTg7Dhg3D4/H4l/F4PERFRREWFtZgLDo6GofD0Wjt\nj0lKSjLajoObC4yWl9bJdL8SuZ419WEppN/munDhAlOnTmXevHmMGzcOgCeeeILDhw8DsH//fvr2\n7UtiYiIlJSX4fD7Onj2Lz+cjNjaWPn36UFZWBkBxcTFOp/OatSIiEjohPTJZv349ly5dYu3ataxd\nuxaABQsWsGzZMiIiIujUqRM5OTk4HA6cTidpaWn4fD6ysrIAmD9/PosWLSIvL4+ePXvicrmw2+2N\n1oqISOjYLMuyWrqJUHO73eanueZMb6ZupDVxvrS+pVsQCZqm3jt10aKIiBhTmIiIiDGFiYiIGFOY\niIiIMYWJiIgYU5iIiIgxhYmIiBhTmIiIiDGFiYiIGFOYiIiIMYWJiIgYU5iIiIgxhYmIiBhTmIiI\niDGFiYiIGFOYiIiIMYWJiIgYU5iIiIixkD4DPlh8Ph+LFy/m2LFjtGvXjtzcXHr06NHSbYmItBmt\n4shkz5491NbWsm3bNv74xz+yYsWKlm5JRKRNaRVHJm63m6FDhwIwaNAgKioqWrgjkZY1fd/Blm5B\nrkPrf+kM2rpbRZjU1NTgcDj8r+12O/X19YSHX3vz3G630e+0PfaU0fLSOpnuV83lqfa2lm5BrkPB\n3D9bRZg4HA48Ho//tc/nazJIkpKSQtGWiEib0SrmTBITEykuLgagvLyc3r17t3BHIiJti82yLKul\nmzD1/be5/vGPf2BZFsuWLeP2229v6bZERNqMVhEmIiLSslrFaS4REWlZChMRETGmMBEREWMKE/nZ\nfD4fWVlZpKWlkZ6eTlVVVUu3JNLAoUOHSE9Pb+k22oRWcZ2JtIz/vo1NeXk5K1asYN26dS3dlggA\nBQUF7N69mw4dOrR0K22CjkzkZ9NtbOR6Fh8fT35+fku30WYoTORnu9ZtbESuBy6Xq8k7YUjzUpjI\nz/ZTb2MjIq2XwkR+Nt3GRkS+p4+R8rOlpqZSWlrKxIkT/bexEZG2SbdTERERYzrNJSIixhQmIiJi\nTGEiIiLGFCYiImJMYSIiIsYUJiIiYkxhInINn3zyCS+88EKzre+/7177/PPPc+bMmR/UnD9/nqee\negqABQsWsGvXroDXX11dze9///uf1NOuXbtYsGDBT1pGpDEKE5Fr6N+/P0uXLm229R04cMD/c1lZ\nGY1d4tWlSxcKCgp+1vq//fZbjh49+rP7EzGhMBG5hrKyMtLT0/nLX/7Cww8/zCOPPEJWVlaTy9TX\n17Nw4ULS0tIYNmwYTz75JFeuXCE3NxeA8ePHs3HjRr744gumTZvG119/zQMPPEBGRgYul4vDhw/z\nwAMP+Ne3d+9exowZw8iRI3nrrbeAHx5NpKenU1ZWRm5uLl988QUzZ84E4PXXX2f06NGMGjWKzMxM\nrl696h93uVyMHTuWvXv3NuefTNowhYlIE+rr69mwYQM7d+5k165d2Gw2zp8/f836jz/+mIiICLZt\n28Z7773H1atXef/991m4cCEAO3bsYNq0aXTu3JmNGzdyyy23AJCSksI777xDbGxsg/V99913bN++\nnU2bNrFs2TK+/PLLa/7uhQsX0rlzZ9asWcPx48fZvn07W7du5Y033uDWW2/l5Zdf5vz586xcuZJX\nX32Vbdu2NbhRp4gJ3ZtLpAnh4eHcfffdjBs3jmHDhvHb3/6WLl26XLP+nnvuISYmhldffZV//etf\nnDx5ksuXL//o7xk4cGCj46NHjyY8PJwuXbowaNAgDh06FFDfZWVlVFVVMWHCBADq6uro06cPH3/8\nMXfffTedOnUCYOTIkXzwwQcBrVOkKQoTkR+xdu1aysvLKS4u5sknn2TlypUMHjy40dqioiJeeukl\npkyZwpgxY/j6668bnRv5X+3bt2903G63+3+2LIuIiAhsNluDddbV1f1gOa/Xy4gRI/xHRB6PB6/X\ny/79+/H5fP46PTJAmotOc4k04eLFi4wYMYLevXvzzDPPcN9993Hs2LFr1u/fv58RI0YwduxYOnXq\nxIcffojX6wUaPjzMbrf7x5vy5ptvYlkWZ86c4ZNPPqF///7ccsstfPbZZ1iWxalTp/z9hIeH+9c/\nZMgQ3nvvPb766issy2Lx4sVs3ryZpKQkDh06xPnz5/H5fP55GBFT+lgi0oTY2FiGDRvGuHHj6NCh\nA127dmX06NHXrB8/fjxz587l7bffpl27dgwaNIjTp08DMGzYMEaNGsWuXbv49a9/zbRp09i0aVOT\nv79jx46MGTOG+vp6/u///o/Y2Fh++ctfsnPnToYPH05CQgJJSUkA3HrrrcTFxZGenk5hYSGzZs3i\nsccew+fzcddddzFt2jTat2/PwoUL+d3vfkeHDh244447mu+PJW2abkEvIiLGdGQi8hMdPHiQnJyc\nRv9v48aNTU7Qi7RWOjIRERFjmoAXERFjChMRETGmMBEREWMKExERMfb/Bht0YSFsvxQAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='is_attributed', data=df_train, palette='hls')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50000 entries, 0 to 49999\n",
      "Data columns (total 8 columns):\n",
      "ip                 50000 non-null int64\n",
      "app                50000 non-null int64\n",
      "device             50000 non-null int64\n",
      "os                 50000 non-null int64\n",
      "channel            50000 non-null int64\n",
      "click_time         50000 non-null object\n",
      "attributed_time    88 non-null object\n",
      "is_attributed      50000 non-null int64\n",
      "dtypes: int64(6), object(2)\n",
      "memory usage: 3.1+ MB\n"
     ]
    }
   ],
   "source": [
    "# The ratio of df_train to df_test is 0.8 to 0.2 or 0.75 to 0.25\n",
    "df_test = pd.read_csv('data/train.csv', nrows=50000,skiprows=range(1, 400000))\n",
    "df_test.dropna()\n",
    "df_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ip</th>\n",
       "      <th>app</th>\n",
       "      <th>device</th>\n",
       "      <th>os</th>\n",
       "      <th>channel</th>\n",
       "      <th>click_time</th>\n",
       "      <th>attributed_time</th>\n",
       "      <th>is_attributed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115115</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>145</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>21633</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>178</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>144498</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>178</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>76919</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>237</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1556</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>67467</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20266</td>\n",
       "      <td>64</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>459</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>31564</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>265</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1732</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>134</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>111114</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>245</td>\n",
       "      <td>2017-11-06 16:07:47</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ip  app  device  os  channel           click_time attributed_time  \\\n",
       "0  115115    8       1  13      145  2017-11-06 16:07:47             NaN   \n",
       "1   21633    1       1  19      178  2017-11-06 16:07:47             NaN   \n",
       "2  144498   12       1  17      178  2017-11-06 16:07:47             NaN   \n",
       "3   76919    2       1   6      237  2017-11-06 16:07:47             NaN   \n",
       "4    1556   15       1  13      245  2017-11-06 16:07:47             NaN   \n",
       "5   67467   15       1  37      245  2017-11-06 16:07:47             NaN   \n",
       "6   20266   64       1  19      459  2017-11-06 16:07:47             NaN   \n",
       "7   31564   15       1  19      265  2017-11-06 16:07:47             NaN   \n",
       "8    1732    9       1  13      134  2017-11-06 16:07:47             NaN   \n",
       "9  111114   15       1  13      245  2017-11-06 16:07:47             NaN   \n",
       "\n",
       "   is_attributed  \n",
       "0              0  \n",
       "1              0  \n",
       "2              0  \n",
       "3              0  \n",
       "4              0  \n",
       "5              0  \n",
       "6              0  \n",
       "7              0  \n",
       "8              0  \n",
       "9              0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    49912\n",
       "1       88\n",
       "Name: is_attributed, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['is_attributed'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEFCAYAAAAfRLtkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFNRJREFUeJzt3X9MVff9x/HX4V60yo/ibaMpcTq0\nGktmpXCHSYesKW7IH9ZqcagN7dJV1q7qSNYORX7oxF9pg4la648ui7F2KoU4ky3tlNQS0UK9KVrZ\ndKZrmYqxP2w3uFoEzvn+sXq/ZRX9UDiA8Hz8hYfPvbyvubnPe+6591zLcRxHAAAYCOvrAQAAtw+i\nAQAwRjQAAMaIBgDAGNEAABjz9vUAbgsEAn09AgDclpKSkr61bcBHQ7rxDQcAdK6zJ9y8PAUAMEY0\nAADGiAYAwBjRAAAYIxoAAGNEAwBgzNW33M6ePVuRkZGSpNGjRysrK0urV6+Wx+NRSkqKFi1aJNu2\ntWLFCp05c0ZDhgxRSUmJxo4dq7q6OuO1AIDe4Vo0Wlpa5DiOdu3aFdo2a9Ysbdq0Sd/73veUk5Oj\nv/3tbzp//ryuXbumvXv3qq6uTuvWrdMrr7yi4uJi47UAgN7hWjROnz6tq1ev6qmnnlJbW5sWL16s\na9euacyYMZKklJQUHT16VJ9++qmmTZsmSUpISNCpU6fU3NxsvNYEnwoHgJ7hWjTuuOMO/eIXv9Dc\nuXP18ccfa+HChYqOjg79PiIiQufOnVNzc3PoJSxJ8ng839p2s7VtbW3yem9+M7r7ifDjS57p1uUx\n8Pg3bu3rEQBXdfZk27VoxMXFaezYsbIsS3FxcYqKitKXX34Z+n0wGFR0dLS++uorBYPB0HbbthUZ\nGdlh283W3ioYAICe49q7p9544w2tW7dOknTp0iVdvXpVw4cP17/+9S85jqMjR47I7/crMTFRVVVV\nkqS6ujpNnDhRkZGRCg8PN1oLAOg9rj1Nz8zM1LJlyzR//nxZlqU1a9YoLCxMzz//vNrb25WSkqIp\nU6Zo8uTJqq6u1rx58+Q4jtasWSNJWrlypfFaAEDvsBzHcfp6CDcFAgGOaaDHcUwDA11nj518uA8A\nYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADG\niAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxo\nAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGDM1Wh8/vnn+vGPf6wPP/xQDQ0Nmj9/\nvhYsWKDi4mLZti1J2rx5szIzMzVv3jydPHlSkrq0FgDQe1yLRmtrq4qKinTHHXdIktauXavc3Fy9\n/vrrchxHlZWVqq+vV21trcrKylRaWqqVK1d2eS0AoPe4Fo3169dr3rx5GjlypCSpvr5eycnJkqTU\n1FQdPXpUgUBAKSkpsixLsbGxam9v1+XLl7u0FgDQe7xuXGlFRYV8Pp+mTZum7du3S5Icx5FlWZKk\niIgINTU1qbm5WTExMaHLXd/elbU+n++W8wQCgZ68eQD3KQxarkSjvLxclmXp2LFj+vvf/668vLwO\newXBYFDR0dGKjIxUMBjssD0qKkphYWHGa00kJSV16/Yc37mjW5fHwNPd+xTQ33X2xMiVl6d2796t\n1157Tbt27dJ9992n9evXKzU1VTU1NZKkqqoq+f1+JSYm6siRI7JtW42NjbJtWz6fT/Hx8cZrAQC9\nx5U9jRvJy8tTYWGhSktLNW7cOKWnp8vj8cjv9ysrK0u2bauoqKjLawEAvcdyHMfp6yHcFAgEuv/y\n1JJnemgaDBT+jVv7egTAVZ09dvLhPgCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxoAACM\nEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADGiAYAwBjR\nAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxoAACMEQ0A\ngDGiAQAw5nXritvb21VQUKCPPvpIlmVp5cqVGjp0qJYuXSrLsjRhwgQVFxcrLCxMmzdv1uHDh+X1\nepWfn6/7779fDQ0NxmsBAL3DtWi8/fbbkqQ9e/aopqZGGzZskOM4ys3N1dSpU1VUVKTKykrFxsaq\ntrZWZWVlunjxohYvXqzy8nKtXbvWeC0AoHe4Fo3p06froYcekiQ1NjYqOjpaR48eVXJysiQpNTVV\n1dXViouLU0pKiizLUmxsrNrb23X58mXV19cbr/X5fDedJRAIuHUzMUhxn8Jg5Vo0JMnr9SovL08H\nDx7Uxo0bVV1dLcuyJEkRERFqampSc3OzYmJiQpe5vt1xHOO1t4pGUlJSt27H8Z07unV5DDzdvU8B\n/V1nT4xcPxC+fv16vfXWWyosLFRLS0toezAYVHR0tCIjIxUMBjtsj4qKUlhYmPFaAEDvcC0a+/fv\n17Zt2yRJw4YNk2VZ+sEPfqCamhpJUlVVlfx+vxITE3XkyBHZtq3GxkbZti2fz6f4+HjjtQCA3uHa\ny1M//elPtWzZMj3++ONqa2tTfn6+xo8fr8LCQpWWlmrcuHFKT0+Xx+OR3+9XVlaWbNtWUVGRJCkv\nL894LQCgd1iO4zi3WrRq1SoVFhZ22JaXl6f169e7NlhPCQQC3T+mseSZHpoGA4V/49a+HgFwVWeP\nnTfd01i+fLnOnTunU6dO6ezZs6HtbW1tampq6vkpAQD92k2j8eyzz+rChQtavXq1Fi1aFNru8Xg0\nfvx414cDAPQvN43G6NGjNXr0aB04cEDNzc2ht8JK0pUrVzq8/RUAMPAZHQjftm2btm3b1iESlmWp\nsrLStcEAAP2PUTTKysp06NAh3t4KAIOc0ec07rnnHt15551uzwIA6OeM9jS+//3va8GCBZo6daqG\nDBkS2v7Ng+MAgIHPKBqjRo3SqFGj3J4FANDPGUWDPQoAgGQYjUmTJoXOOHvdyJEj9c4777gyFACg\nfzKKxunTp0M/t7a26tChQ6qrq3NtKABA/9Tls9yGh4crIyND7777rhvzAAD6MaM9jf3794d+dhxH\nZ8+eVXh4uGtDAQD6J6NoXP9ei+tGjBihDRs2uDIQAKD/MorG2rVr1draqo8++kjt7e2aMGGCvF5X\nvykWANAPGT3ynzp1SkuWLFFMTIxs29Znn32ml19+WVOmTHF7PgBAP2IUjZKSEm3YsCEUibq6Oq1a\ntUpvvPGGq8MBAPoXo3dPXblypcNeRUJCglpaWlwbCgDQPxlF484779ShQ4dC/z506BDfpQEAg5DR\ny1OrVq3SL3/5Sy1fvjy0bc+ePa4NBQDon4z2NKqqqjRs2DC9/fbb2rlzp3w+n2pra92eDQDQzxhF\nY9++ffrjH/+o4cOHa9KkSaqoqNBrr73m9mwAgH7GKBqtra0dPgHOp8EBYHAyOqYxffp0Pfnkk8rI\nyJAk/fWvf1VaWpqrgwEA+h+jaLzwwgt688039d5778nr9eqJJ57Q9OnT3Z4NANDPGJ8LZMaMGZox\nY4abswAA+rkunxodADB4EQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY658Z2tra6vy8/N14cIF\nXbt2Tc8++6zuvfdeLV26VJZlacKECSouLlZYWJg2b96sw4cPy+v1Kj8/X/fff78aGhqM1wIAeo8r\n0Thw4IBiYmL04osv6ssvv9Sjjz6qSZMmKTc3V1OnTlVRUZEqKysVGxur2tpalZWV6eLFi1q8eLHK\ny8u1du1a47UAgN7jSjRmzJih9PR0SZLjOPJ4PKqvr1dycrIkKTU1VdXV1YqLi1NKSoosy1JsbKza\n29t1+fLlLq31+Xxu3AQAwA24Eo2IiAhJUnNzs5YsWaLc3FytX79elmWFft/U1KTm5uYO3wB4fbvj\nOMZrTaIRCAR68uYB3KcwaLkSDUm6ePGinnvuOS1YsEAzZ87Uiy++GPpdMBhUdHS0IiMjFQwGO2yP\niopSWFiY8VoTSUlJ3botx3fu6NblMfB09z4F9HedPTFy5d1Tn332mZ566im98MILyszMlCTFx8er\npqZG0n+/CdDv9ysxMVFHjhyRbdtqbGyUbdvy+XxdWgsA6D2u7Gls3bpV//nPf7RlyxZt2bJFkrR8\n+XKVlJSotLRU48aNU3p6ujwej/x+v7KysmTbtoqKiiRJeXl5KiwsNFoLAOg9luM4Tl8P4aZAIND9\nl6eWPNND02Cg8G/c2tcjAK7q7LGTD/cBAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMA\nYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEAMEY0AADG\niAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMaAABjRAMAYIxo\nAACMEQ0AgDFXo3HixAllZ2dLkhoaGjR//nwtWLBAxcXFsm1bkrR582ZlZmZq3rx5OnnyZJfXAgB6\nj2vR2LFjhwoKCtTS0iJJWrt2rXJzc/X666/LcRxVVlaqvr5etbW1KisrU2lpqVauXNnltQCA3uN1\n64rHjBmjTZs26be//a0kqb6+XsnJyZKk1NRUVVdXKy4uTikpKbIsS7GxsWpvb9fly5e7tNbn891y\nlkAg4NbNxCDFfQqDlWvRSE9P1/nz50P/dhxHlmVJkiIiItTU1KTm5mbFxMSE1lzf3pW1JtFISkrq\n1m05vnNHty6Pgae79ymgv+vsiVGvHQgPC/v/PxUMBhUdHa3IyEgFg8EO26Oiorq0FgDQe3otGvHx\n8aqpqZEkVVVVye/3KzExUUeOHJFt22psbJRt2/L5fF1aCwDoPa69PPW/8vLyVFhYqNLSUo0bN07p\n6enyeDzy+/3KysqSbdsqKirq8loAQO+xHMdx+noINwUCge4f01jyTA9Ng4HCv3FrX48AuKqzx04+\n3AcAMEY0AADGiAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADBGNAAAxogGAMAY0QAAGCMa\nAABjRAMAYIxoAACMEQ0AgDGiAQAwRjQAAMaIBgDAGNEAABgjGgAAY0QDAGCMaAAAjBENAIAxogEA\nMEY0AADGiAYAwBjRAAAYIxoAAGNEAwBgjGgAAIwRDQCAMaIBADDm7esBusq2ba1YsUJnzpzRkCFD\nVFJSorFjx/b1WAAwKNx2exqHDh3StWvXtHfvXv3mN7/RunXr+nokABg0brs9jUAgoGnTpkmSEhIS\ndOrUqT6eCOg7zxw93tcjoB/a+qDfteu+7aLR3NysyMjI0L89Ho/a2trk9XZ+UwKBQLf+pvXkwm5d\nHgNPd+9TPWXhUKuvR0A/5Ob987aLRmRkpILBYOjftm3fNBhJSUm9MRYADAq33TGNxMREVVVVSZLq\n6uo0ceLEPp4IAAYPy3Ecp6+H6Irr7576xz/+IcdxtGbNGo0fP76vxwKAQeG2iwYAoO/cdi9PAQD6\nDtEAABgjGgAAY0QDt2TbtoqKipSVlaXs7Gw1NDT09UhABydOnFB2dnZfjzEo3Haf00Dv++apW+rq\n6rRu3Tq98sorfT0WIEnasWOHDhw4oGHDhvX1KIMCexq4JU7dgv5szJgx2rRpU1+PMWgQDdxSZ6du\nAfqD9PT0m54VAj2LaOCWunrqFgADF9HALXHqFgDX8XQRt/STn/xE1dXVmjdvXujULQAGJ04jAgAw\nxstTAABjRAMAYIxoAACMEQ0AgDGiAQAwRjQAAMaIBiDpgw8+0PLly3vs+r55xtVly5bpwoUL31pz\n6dIlLVy4UJK0dOlSVVRUGF9/U1OTfvWrX3VppoqKCi1durRLlwH+F9EAJE2ePFmrV6/useurra0N\n/VxTU6MbfRxq1KhR2rFjx3e6/n//+986ffr0d54P+K6IBqD/PrBnZ2frD3/4gx555BE9+uijKioq\nuull2traVFBQoKysLKWlpenpp5/WV199pZKSEknS3LlztX37dn3yySfKycnRF198oYcffli5ublK\nT0/XyZMn9fDDD4eu7/Dhw5ozZ45mzpypv/zlL5K+vXeQnZ2tmpoalZSU6JNPPtFzzz0nSdq/f79m\nz56tWbNmKT8/Xy0tLaHt6enpeuyxx3T48OGe/C/DIEU0gK+1tbVp27ZtKi8vV0VFhSzL0qVLlzpd\n//777ys8PFx79+7VwYMH1dLSonfeeUcFBQWSpLKyMuXk5GjkyJHavn27RowYIUlKTU3VW2+9JZ/P\n1+H6rl69qn379unVV1/VmjVr9Omnn3b6twsKCjRy5Ei9/PLLOnv2rPbt26c9e/boT3/6k+666y79\n/ve/16VLl/TSSy9p9+7d2rt3b4eTTgLfFeeeAr7m9Xr1wAMPKDMzU2lpaXr88cc1atSoTtf/8Ic/\nVExMjHbv3q1//vOf+vjjj3XlypVb/p0pU6bccPvs2bPl9Xo1atQoJSQk6MSJE0Zz19TUqKGhQT/7\n2c8kSa2trYqPj9f777+vBx54QHfffbckaebMmXr33XeNrhPoDNEAvmHLli2qq6tTVVWVnn76ab30\n0ktKTk6+4drKykpt3LhRTzzxhObMmaMvvvjihscu/tfQoUNvuN3j8YR+dhxH4eHhsiyrw3W2trZ+\n63Lt7e3KyMgI7eEEg0G1t7fr2LFjsm07tI7T2aMn8PIU8LXLly8rIyNDEydO1K9//Wv96Ec/0pkz\nZzpdf+zYMWVkZOixxx7T3Xffrffee0/t7e2SOn5RlcfjCW2/mT//+c9yHEcXLlzQBx98oMmTJ2vE\niBH68MMP5TiOzp07F5rH6/WGrn/q1Kk6ePCgPv/8czmOoxUrVmjnzp1KSkrSiRMndOnSJdm2HTpO\nAnQHTz2Ar/l8PqWlpSkzM1PDhg3TPffco9mzZ3e6fu7cuXr++ef15ptvasiQIUpISND58+clSWlp\naZo1a5YqKir00EMPKScnR6+++upN//7w4cM1Z84ctbW16Xe/+518Pp8efPBBlZeXa8aMGYqLi1NS\nUpIk6a677lJsbKyys7O1a9cuLVq0SE8++aRs29Z9992nnJwcDR06VAUFBfr5z3+uYcOG6d577+25\n/ywMWpwaHQBgjD0N4CaOHz+uVatW3fB327dvv+mBcmAgYk8DAGCMA+EAAGNEAwBgjGgAAIwRDQCA\nsf8DoQTTGbbtqNgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='is_attributed', data=df_test, palette='hls')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ip', 'app', 'device', 'os', 'channel']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get columes names except the click_time (object), attributed_time (object) and is_attributed\n",
    "train_cols = []\n",
    "for each_value in df_train.columns.values:\n",
    "    if each_value == 'click_time' or each_value == 'attributed_time' or each_value == 'is_attributed':\n",
    "        continue\n",
    "    train_cols.append(each_value)\n",
    "train_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data_X\n",
    "X_train = df_train.loc[:,train_cols]\n",
    "X_test = df_test.loc[:,train_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 5)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data_Y\n",
    "y_train = df_train[['is_attributed']]\n",
    "y_test = df_test[['is_attributed']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 1)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_test_val = y_test.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def randomForest_pre(X_train, y_train, X_test, y_test):\n",
    "    y_train_model = y_train.values.ravel()\n",
    "    RF_model = RandomForestClassifier()\n",
    "    # Train the model\n",
    "    RF_fit = RF_model.fit(X_train, y_train_model)\n",
    "    # Get the prediction of the test data\n",
    "    RF_predict = RF_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    RF_acc = sklearn.metrics.accuracy_score(np.array(RF_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    # Plot the data\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.plot(RF_predict, color='red', label='Prediction')\n",
    "    plt.plot(y_test, label='Y_test')\n",
    "    plt.legend(['Prediction', 'Y_test'])\n",
    "    _ = plt.ylim()\n",
    "    \n",
    "    return RF_fit, RF_predict, RF_acc, RF_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)\n",
    "# RF_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Gradient Boosting & AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gradientBoosting_pre(X_train, y_train, X_test, y_test):\n",
    "    y_train_model = y_train.values.ravel()\n",
    "    GB_model = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)\n",
    "    # Train the model\n",
    "    GB_fit = GB_model.fit(X_train, y_train_model)\n",
    "    # Get the prediction of the test data\n",
    "    GB_predict = GB_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    GB_acc = sklearn.metrics.accuracy_score(np.array(GB_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    \n",
    "    # Plot the data\n",
    "    plt.figure(figsize=(10,5))\n",
    "    plt.plot(GB_predict, color='red', label='Prediction')\n",
    "    plt.plot(y_test, label='Y_test')\n",
    "    plt.legend(['Prediction', 'Y_test'])\n",
    "    _ = plt.ylim()\n",
    "    \n",
    "    return GB_fit, GB_predict, GB_acc, GB_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)\n",
    "# GB_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def logisticRegression_pre(X_train, y_train, X_test, y_test):\n",
    "    y_train_model = y_train.values.ravel()\n",
    "    LG_model = LogisticRegression()\n",
    "    # Train the model\n",
    "    LG_fit = LG_model.fit(X_train, y_train_model)\n",
    "    # Get the prediction of the test data\n",
    "    LG_predict = LG_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    LG_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], \n",
    "                                         np.array(y_test_val)[:])\n",
    "    \n",
    "    return LG_fit, LG_predict, LG_acc, LG_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)\n",
    "# LG_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. SVM - Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svm_pre(X_train, y_train, X_test, y_test):\n",
    "    svm_model = svm.SVC()\n",
    "    uniq = np.unique(y_train[9000:10000])\n",
    "    # Train the model\n",
    "    SVM_fit = svm_model.fit(X_train[9000:10000], y_train[9000:10000])\n",
    "    # Get the prediction of the test data\n",
    "    SVM_predict = svm_model.predict(X_test)\n",
    "    # Compare the prediction with the known values\n",
    "    SVM_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], \n",
    "                                     np.array(y_test_val)[:])\n",
    "    return SVM_fit, SVM_predict, SVM_acc, svm_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)\n",
    "# svm_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. ANN - Artificial Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shallow_net_A(n=55,i=len(train_cols),o=2):\n",
    "    # Create simple one dense layer net\n",
    "    # Default 55 neurons, input 5, output 2\n",
    "    net = Sequential()\n",
    "    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))\n",
    "    net.add(Dense(2, activation='softmax'))\n",
    "    # Compile net\n",
    "    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ann_pre(X_train, df_train, X_test, df_test):\n",
    "    ann_model = shallow_net_A()\n",
    "    ann_summary = ann_model.summary()\n",
    "    # Convert the values\n",
    "    X_train_ann = X_train.values\n",
    "    y_train_ann = df_train['is_attributed'].values\n",
    "    X_test_ann = X_test.values\n",
    "    y_test_ann = df_test['is_attributed'].values\n",
    "    # Conver the matrix, finally we have two classes (n_classes), the original one has oly one class\n",
    "    n_classes = 2\n",
    "    y_train_ann = keras.utils.to_categorical(y_train_ann, n_classes)\n",
    "    y_test_ann = keras.utils.to_categorical(y_test_ann, n_classes)\n",
    "    # Training the model\n",
    "    ann_fit = ann_model.fit(X_train_ann, y_train_ann, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_ann, y_test_ann))\n",
    "    \n",
    "    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy\n",
    "    ann_evaluate = ann_model.evaluate(X_test_ann, y_test_ann)\n",
    "    \n",
    "    # Using prediction\n",
    "    ann_pre = ann_model.predict(X_test)\n",
    "    # Convert value to boolean value\n",
    "    y_pre = (ann_pre > 0.5)\n",
    "    # Counting the boolean value, counting the accuracy by using basic calculation\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    ann_output = confusion_matrix(y_test_ann.argmax(axis=1), y_pre.argmax(axis=1))\n",
    "    ann_prediction_acc = ann_output[0][0]/(ann_output[0][0]+ann_output[1][0])\n",
    "    \n",
    "    return ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, df_train, X_test, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ann_prediction_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. MLP - Multi-layered Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shallow_net_C(n=55,i=5,o=2):\n",
    "    # Create simple one dense layer net\n",
    "    # Default 55 neurons, input 5, output 2, here we have more hidden layers with different activation\n",
    "    net = Sequential()\n",
    "    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='relu', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='tanh', input_shape=(i,)))\n",
    "    net.add(Dense(n, activation='elu', input_shape=(i,)))\n",
    "    net.add(Dense(2, activation='softmax'))\n",
    "    # Compile net\n",
    "    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mlp_pre(X_train, df_train, X_test, df_test):\n",
    "    mlp_model = shallow_net_C()\n",
    "    mlp_summary = mlp_model.summary()\n",
    "    # Convert the values\n",
    "    X_train_mlp = X_train.values\n",
    "    y_train_mlp = df_train['is_attributed'].values\n",
    "    X_test_mlp = X_test.values\n",
    "    y_test_mlp = df_test['is_attributed'].values\n",
    "    # Conver the matrix, finally we have two classes (n_classes)\n",
    "    n_classes = 2\n",
    "    y_train_mlp = keras.utils.to_categorical(y_train_mlp, n_classes)\n",
    "    y_test_mlp = keras.utils.to_categorical(y_test_mlp, n_classes)\n",
    "    # Training the model\n",
    "    mlp_fit = mlp_model.fit(X_train_mlp, y_train_mlp, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_mlp, y_test_mlp))\n",
    "    \n",
    "    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy\n",
    "    mlp_evaluate = mlp_model.evaluate(X_test_mlp, y_test_mlp)\n",
    "    \n",
    "    # Using prediction\n",
    "    mlp_pre = mlp_model.predict(X_test)\n",
    "    # Convert value to boolean value\n",
    "    y_pre = (mlp_pre > 0.5)\n",
    "    # Counting the boolean value, counting the accuracy by using basic calculation\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    mlp_output = confusion_matrix(y_test_mlp.argmax(axis=1), y_pre.argmax(axis=1))\n",
    "    mlp_prediction_acc = mlp_output[0][0]/(mlp_output[0][0]+mlp_output[1][0])\n",
    "    \n",
    "    return mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, df_train, X_test, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# mlp_prediction_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 7. RNN - Recurrent Neural Network (LSTM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rnn_pre(df_train, train_rate=0.75):\n",
    "    # Set the dataset for train and test\n",
    "    df_rnn_train = df_train.loc[:,train_cols + ['is_attributed']]\n",
    "    df_rnn_test = df_test.loc[:,train_cols + ['is_attributed']]\n",
    "    df_rnn = df_rnn_train.append(df_rnn_test)\n",
    "    \n",
    "    pre_col_index = list(df_rnn_train).index('is_attributed')\n",
    "    dataset = df_rnn.values.astype('float32')\n",
    "    \n",
    "    # Normalize the dataset, set all the data of the dataset to be in the range between 0 and 1\n",
    "    scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "    dataset = scaler.fit_transform(dataset)\n",
    "    \n",
    "    # Split into train and test sets\n",
    "    train_size = int(len(dataset) * train_rate)\n",
    "    test_size = len(dataset) - train_size\n",
    "    train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:]\n",
    "    \n",
    "    # Use this function to prepare the train and test datasets for modeling\n",
    "    look_back = 1\n",
    "    trainY = train[:, pre_col_index]\n",
    "    trainX = np.delete(train, pre_col_index, axis = 1) \n",
    "    testY = test[:, pre_col_index]\n",
    "    testX = np.delete(test, pre_col_index, axis = 1) \n",
    "    \n",
    "    # Reshape input to be [samples, time steps, features], here it changes the dimension from 2D to 3D\n",
    "    trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))\n",
    "    testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))\n",
    "    \n",
    "    # Create and fit the LSTM network\n",
    "    RNN_model = Sequential()\n",
    "    RNN_model.add(LSTM(5, input_shape=(1, len(trainX[0][0]))))\n",
    "    RNN_model.add(Dense(1))\n",
    "    RNN_model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    RNN_model.fit(trainX, trainY, epochs=10, batch_size=128, verbose=2)\n",
    "    \n",
    "    # Make predictions, trainPredict should be 1D array\n",
    "    trainPredict = RNN_model.predict(trainX)\n",
    "    testPredict = RNN_model.predict(testX)\n",
    "    \n",
    "    # Change the dimension from 3D to 2D\n",
    "    trainX_2D = trainX.transpose([1,0,2]).reshape(len(trainX),len(trainX[0][0]))\n",
    "    testX_2D = testX.transpose([1,0,2]).reshape(len(testX),len(testX[0][0]))\n",
    "    \n",
    "    # Append prediction back to the model\n",
    "    trainPredict_6cols = np.append(trainX_2D, trainPredict, 1)\n",
    "    testPredict_6cols = np.append(testX_2D, testPredict, 1)\n",
    "\n",
    "    # Invert predictions back to normal values\n",
    "    trainPredict_6cols = scaler.inverse_transform(trainPredict_6cols)\n",
    "    testPredict_6cols = scaler.inverse_transform(testPredict_6cols)\n",
    "    \n",
    "    # Calculating the RMSE\n",
    "    trainScore = math.sqrt(mean_squared_error(trainY, trainPredict_6cols[:, pre_col_index]))\n",
    "    print('Train Score: %.2f RMSE' % (trainScore))\n",
    "    testScore = math.sqrt(mean_squared_error(testY, testPredict_6cols[:, pre_col_index]))\n",
    "    print('Test Score: %.2f RMSE' % (testScore))\n",
    "    \n",
    "    final_prediction_train = np.where(trainPredict_6cols[:, pre_col_index] > 0, 1, 0)\n",
    "    final_prediction_test = np.where(testPredict_6cols[:, pre_col_index] > 0, 1, 0)\n",
    "    \n",
    "    # Change dimension from 2D to 1D\n",
    "    final_prediction_train = np.reshape(final_prediction_train, (-1, 1))\n",
    "    final_prediction_test = np.reshape(final_prediction_test, (-1, 1))\n",
    "    \n",
    "    # Counting the accuracy by using basic calculation\n",
    "    rnn_acc_train = sklearn.metrics.accuracy_score(np.array(final_prediction_train)[:], \n",
    "                                     np.array(trainY)[:])\n",
    "    rnn_acc_test = sklearn.metrics.accuracy_score(np.array(final_prediction_test)[:], \n",
    "                                     np.array(testY)[:])\n",
    "    return rnn_acc_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# rnn_acc = rnn_pre(df_train)\n",
    "# rnn_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Test Data (Call the function)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest accuracy: 99.842%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAEuCAYAAACu4EdXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcE/X9P/BXjr2z7LIce8Euy3IJ\ngrCx5VsrSuVL9Vtr9asV/NLyaL9qay+tR6vVVkVLEWpb/VZba7Via6sFj5/3iUVQQJDAUljOZQ/2\nvmHZ7JFNMr8/dpPNPZPJTDJJXs+/kszM5/Oez3xm8k4y+Xx0giAIICIiIqKw6GMdABEREVE8YhJF\nREREJAOTKCIiIiIZmEQRERERycAkioiIiEgGJlFEREREMhijXaHFYol2lURERESymc3mgK9HPYkC\nggejFIvFonodFD4eF+3hMdEmHhft4THRpmgcl1Bf/vDnPCIiIiIZmEQRERERycAkioiIiEgGJlFE\nREREMjCJIiIiIpKBSRQRERGRDEyiiIiIiGSQlEQdOHAAq1ev9nv9X//6F6655hqsXLkSmzdvVjw4\nIiIiIq0SHWzzqaeewuuvv46MjAyv14eHh/HQQw/hpZdeQkZGBv7nf/4Hl1xyCSZOnKhasERERERa\nYVizZs2aUCt0dnZi9erV+OCDD3Dttde6Xz9x4gSqqqqwYsUKGAwGnDx5Ek6nEzNnzgxZYUtLC4qK\nihQJPpChYQfe/LgaA/sO42zNKUw+Z/rIgg8+ABoagGnT8NHjm6Cz2ZBbUhi0nOrG0zhS142S/OyR\nF2pqgFdeASoqIgvwueeAtDTAJ9m0O5x4Z1cdJo3PwOCQHVstDZhenAu9Tofa5jM49MI7KBWsgEjb\nOe0OvPvrvyNv4jhYGvrhFASMN6UBS5eObDtjRljh1jSdQVVtFyanAe898gKsWTmo7xrElMnZ6Doz\ngNe216C+pRe9p61o+dtm7D9rRGHReKSlGPzKaq2vR+Hrr6NrUjE2vFSFoolZmPjy89j1zh4M9A9h\nwowSSTF19w56tQ8AtP37OHb87W2Uf2E+dKOv4a23gM5OYOpUr+0/+O0/kJluRHbhpMAVPP88ztgE\nfFhjRVlRDg4c70B37yAa2/vQdWYAk/My/TZp7+7Hy1tPoLXLivLinLEYRHz4yPNIMwDjivODrvPI\nr17C22/vx1RHH8ZNK8a7O2uRn5cJw/Aw3v3NP5Bfmo/0nGxJ9bkM2514d2ct4LBiWkmx+/XTZ4fw\n7FuH0XqsHife+AhTzOcgxWiAIAi478md0L/9Nqre3oEpi2YjJTMjRA3BdfQMYN2ze3C4thuzpuYC\nfX24a+0bmF6cgwkTx4Xe2OnEoUc3otGRisKSyX6Lt/7+nzA67MiZWiAplqpXt+LUZ4dRdN4s7PvH\n23hxazWmlhciOysVTqeAd3fVIW9cOjLTU7BlTz3S04zIzkyVsdf+bMMOvLurDoUTs/zOFynXyZMf\n7saRD/agajgDuaY0ZGWkABjpix9XNqF8SvB+OGiz471ddSiaZILlaDtONPSgvqUX04py3H0jPy8T\n6amBP2cfrO5Ec0cfCidmhb3fx0/14ETDaUyZHLjP1rX04t/VHSgtFOkLYRi2DrjPlc4hYO+RNpQV\n5Yhud7SuGzXNZ1A8yYSWlhYUFBTi3V11GJ890idc/l3dgeYOq+T2qG0+g4PVne59rGvpxf1/3oWL\nK4phbWjFh3/6f5j+uXOgN3q0//Aw8OSTQEkJYDIBAD450AS7Q8D4cenu1U42nsaR2m40vbEFg509\nmDBjKpo/O4Tdm7dg+hfmB43Js79X7jmBZ57+F5acm48d37gVzvF5GO+6PgsC8Je/AOPHo9WZil0H\nW1A+JVfSfgtOJzb+6gUcbRvA/HnF4usLAt7dUYOPn30bheNSsW3jmyj+926kZqYDk0fOf7VzCrE6\ndIIgCGIFNDY24vbbb/f6yW7v3r34+9//jkcffRQA8H//938oKirySrQCUXvuvI8O9uKjg73u52tW\nTQEAmM8/HwDwyWvvYMO2Ia9lgax5vhEAcN91xdDrdVh0wQXQ22w4unEjrPODd8RQUpuaMP/KKwEA\nlr17vZZ9dqIPb312GlMmpMLuFNDaM4wrF4/HovIsdywv/n4lqj7dEbKOujd349neYowbPIve9JGL\n1GOZezFt7dqA9Ypx1X1xfzW2ZY4lYGtWTcHv32hF91m73zazitOx6mL/byTzn30WUx5/HN/5/jNo\nzcgDALzxu6twxe2vusuU4o9vt6H99DCu+o/xWDh95KL1m2eOoi/dhO8Vd6Hg4vMAjB1zz33u+uwo\nHjthgtExjF+sLvMr29jVhfMuvRS/uOYBHCg9D19elIP395/xbpMAcf765Wb0DzkBAN/+z0mYNjlN\ndD96j9Thd/uNQcsEAOvJJjy8e+wUvbQiB+/tO4Npk9NQ0VqFV/QzcG5PLb7+wyWi9XnafawP71hO\no2RSKq5fPpaM/OX9djR02tzPvzBhCJdeWo7DDQPY/HGX+/WlZ49h6U3LwqrT5eFXmmEdHGmrorwU\n5DafwuH0kQ80Yn0g56OPcFvzjIDrWuta8PBOh6RyXFz9e82qKe7HrudVp/rx4ifdyDMZce2FeXjy\n3fawyhbjulYFO1+kxg4A4zINuP2qkTZ09cXVX5qI8sL0gNt+UHkGOw6fxbySDFSdGnC/fvtVhThU\n34/3959BWX4avrUs8AcNz3aTG3ewbV3L7/p6ETJSlbltt/JvW/GqcSbmn67BwdyRD9Y3fzUfE8al\nhNzON9YjDQPY9HEXcrMMuPXKwqDrifHdR9fzieOMyDtVg+O5JVihr8Hc6y5ybzPxpZdQun49+ubP\nx7GNG2F3CFi7qcmvXs9+4Vrmeu22+TbkzJ8eMCbP/t7dN3Jdv37bRjxz8f961WHatw+zv/tdOLKy\ncM33/wGHE7jxy5MxZaL4h4vG9/fi6c6RDzg3XTYZhXmhtznVMYRnPujwem3pkY9wxzuPhv1eFinF\n584zmUywWq3u51arFdnZ0j4NqznPzUfHLADGkijfusqnlAA4IR7HaKerMJth0OsA28gby5xJkwC5\n8Xt8qvCt+1DrYQCn0XraDrtj5A3GND4fZvNsdywOvUG07ZrePgAA7gQKAKYNDwetV9Ro3Z3w/oRl\nNpvR/fxrATc5PaAPWE/Hr34FAO4EypfU2NpH6x2XVwCzeRYAoG80znHp2X7leD7fe7gVgB12Q0rg\n+mprAQA1k0cSrNTMPADeSVSg7fo9LlwFxdNgXiD+yehYWz+A7qBlAkCdFQDGyk7NmgDgDDrOOjHo\nTAX0wKmMiWEf18qmQwBOo7Vn2GvbdZvf8FrP6kyH2WxGU/9JAGNJVBcyZZ/HVo+2au0ZRnfKBPdz\n0TL37Am6bv3APgAN0spxGY3FbDa7H7ueN1lPAuhGd58dU0tnAGgPr2wR24/vA9Ab8HyRNB+YR7y9\n/Q73+q6+OLmwBGbz1ICbvndwD4Cz6LN5vwXMnjMXxztqAJxB51ln8Bg82y1cYtuOLp937nyMzw6c\nBIZr1193AgAa08b62vQZczBjqsg3KB6xWiwWjJ9YBKALp60O7/jDbQ/ffXRdZ3vt6DWNJBkOIc27\nvJdeAgCYjh2D2WzG4JAdGE2iAsXi4tm3C/ImYW6QGD37u0tz7lii6K6jrg4AYLBaMfpWheKS6TCf\nE/zbdJfOfx12P55aOgPnzQrya8Co4UMtALyTqFMTSrziidu588rLy1FfX4/Tp0/DZrNh7969WLRo\nkdziiIiIiOJK2N9EvfHGG+jv78fKlSvxs5/9DDfccAMEQcA111yD/HzxTJSIiIgoEUhKoqZMmeK+\nH+qKK65wv37JJZfgkksuUScyIiIiIg3jYJtEREREMjCJIiIiIpKBSRQRERGRDEyiiIiIiGRgEkVE\nREQkA5MoUp74IPjxVA0B0EHaNDYU3xL9lBK8Hif63voTnMm3z2pLuCRK8Uu97zu16u/cY+X7VS1l\nPrZA60icx011UYhD7PBIndPOXZ6sIORsFG4dHv0kkmIkrufbbErtYtjlhDh+4R5bzVDpmiKl1FDr\nxPxDioL1K9YzlO5jIfYxaJIX8wOjDLlJrKCxD3QJl0SJ0eliuMsh3wBiUy/5iGJbSXnT911FF2KZ\nIthXxuiCPE4AIQ9zgu1r3AmWJIW6GChBankRXSPC2zYeumLSJVFERERESmASRURERCQDkygiIiIi\nGZhEEREREcnAJIqIiIhIBiZRRERERDIwiSIiIiKSgUkUKS5aQ8El44jDUTfaxPEwXgtFTkiQgRyD\nG+vJCb+rgSTlTqsr4ZIopUct9utyKndCr+Jl1BVw97UygGJURiwP3WbhhiDnTSUayZ13DRG0q0io\n7sU+DafUqMFhN28CjVjuClet3iKlbUOtE+u3WyXrV6pnRHHA8uDHRoOJkKzrpMzdEDR2midcEiVK\nH8MjEKuLfJy9ucTUaFvponGhkjONj87zofzjGizhiEVX0Wrv9GzfRDuFQvWdeEtGE40uWGrlc1yU\nPk6SrycR1BvupvHQF5MviSIiTdHe52oiImmYRBERERHJwCSKiIiISAYmUUREREQyMIkiIiIikoFJ\nFBEREZEMTKKIiIiIZGASRURERCQDkyhSXNQG1OUAQ1Ew0shxMOYdKUCDg2ErSmujXUdb4k/rE31M\nokT49Tm1p30J8hgABAnvZAFHeFXiHVCBMqTEH3EdYplVmDHIOdrRuE551hHJG4NYqK7lmnnvCXX8\nYjkbgQyuEaLV6y9SCg4170ts33AVfcNXqGso3cNC7WPQJRpMhGRdJ2XXpq3zPOmSqJgOIx+reb/4\nNYJ0rmlfovA1l5Rj7rtOJFO9eJcT3utq0mrv9Gx6pdpdM0LsToLtadwJ2v5+1wKF65VaYETTvoQ7\n74vsqqIm6ZIoIiIiIiUwiSKimNLejxNERNIwiSIiIiKSgUkUERERkQxMooiIiIhkYBJFREREJAOT\nKCIiIiIZmESR8qI0GBz/1RUF7tE242DAFiIRgsfAQ8k4encS7rLqEi6JSvZrvXq7r8DZp4GDE9PB\nVuOYZlotVgPWqiDOwo1rijV1kh20aAw6HO8SLolSnk8nUjuV9yjftypB7qVAkRM/Ti4eSh+e6M5n\nIKsS2f3Cp5xwlvLSqiCVrilSig21Do+xRkXx6yTJ15YoXiejMX1YOJIuidLstC8xqpd8uKZ9icJ1\nSidhrje/aV90gR+HXXeQbWPRVbTaO3VBn8S/kLuTYPsab3TBkqQQ1wJF6pW8YiTTvoS5vuyaoifp\nkigiIiIiJYgmUU6nE/fddx9WrlyJ1atXo76+3mv5M888g6uvvhrXXHMNPvjgA9UCJSIiItISo9gK\nW7Zsgc1mw6ZNm1BZWYn169fjiSeeAAD09vbib3/7G95//30MDAzgqquuwvLly1UPmogSB++9IaJ4\nJfpNlMViwZIlSwAACxcuxKFDh9zLMjIyUFRUhIGBAQwMDMTdv2OIiIiI5BL9Jqqvrw8mk8n93GAw\nwG63w2gc2bSwsBCXX345HA4HbrrpJvUiJSIiItIQ0STKZDLBarW6nzudTncCtX37drS3t+PDDz8E\nANxwww2oqKjAggULQpZpsVgiiTmkrq7ugHWZR58fPXY0rDj27dsPo0Hn3r66uhpnZMaffvIk5gWp\nu6XlDADvnzZaWpphsVi91hOLubunB8B4r9eam5tRJHH7YOx2u+Q4bMPDAZdPFakj3Niam5phsfR5\nvdbe3uF3zD3LbW5sBDApaH0pra3w7L0dHR1hx1lTW4NMZ6to/N21tQCyQpbZW10LIMX9vK2tDQDg\ncDjR398PmKTF5Ku19bT7see2DofTa73BgQFYLBbUn/JuZ4fDodx57PFvJLEyJ9bXA5gccN3eE3Vw\nXdLCjc13fYvFglMNY/t8/NhxyTFK1dk5cq0Kdr5Eug91dXWw6P37LwD09PQAAAYHB71er6qqQnvb\nyDVHyjGOpC3Etv33vw9iXKZBdvmeBgYGgWx49bWjR4+irzNN0vauWE953BOsxDELtI+uCK1Wq1d5\nRS0tKBxdvs9igd0h7bzxXNbQcAq2IOt69vdQ5eRUV2OGz7Lq6moYBptCbg8AnZ2dcF20Tpw4AcHa\nEHL96qYB0Xh8H0ebaBJVUVGBrVu34itf+QoqKysxa9Ys97KcnBykp6cjNTUVOp0O2dnZ6O3tFa3U\nbDaLriPXxyf2ATX9QeuaM3sOsOuweBzPNwIAKioWIcU41slnzJgByI0/PT1oXEfajwBVZ6HD2ElU\nWFgEs3mOOxbRmAG0vncQ6PJ+raioyP047LYfrduVOHuV4xGXp5SUlID1tItUJTm20XoLi4pgNs/2\nem3y5El+5Xg+1x/vAFqHgtfX4H1ST5o0Caj2TmQDbufRFtPLpsO8qFh0N6q7bcCx9uBlAmgYMgAH\nxi7c+fn5wNE+GAx6ZGZmho4phIMtVcCRar9tDS+3AHaH+3l6RgbMZjM6h+uAPWOJl8FgkH8e+/Qb\nz9sARMvctw84GnjdRnsKsL9WWjk+sfj2Z7PZjNaBGmDvyD7Pmj0L+LAjvLJF7KjeD9ScQmqA88Vi\nsYjX49OO7vVHX582bRrM5pKAm26p+gw4NYD09HSgd+zNc+7cuWg4Ww8c6wt9jD3bLVxi244uX7Bg\nPibkZIRffgB7MnYC8B5jaPacOZhTmhd6Q49YLRYLSkpLgc9Ou18LtJ4kvvvocSxdEWZlZXmXV1jo\nXm42mzFsdwCbmoLG4uLZt6dOmYr5QWL07O+BuOto9L/uz5gxA+Z5BUG3denZdhQYzbVmzpyJijmT\nQ67vzGgFtnUFXOaKR9K5EqFQSZpoErV8+XLs2LED1113HQRBwLp167Bx40aUlJRg2bJl2LlzJ1as\nWAG9Xo+Kigp88YtfVDT4cOniYmQJFfG+tJB4355cGmm3BBqxnKJHqa6RbD2MI5aLE02i9Ho9Hnzw\nQa/XysvL3Y9vueUW3HLLLcpHphF+456pPFqs4PVYCLqMAlN6Pix5A/Gqf6SU2k255bAvKkettox4\nxPIYH+RY1x8NstpfgyOWy7nuyr1Oaq1bcLDNZMBP6ERERIpLviRKo9O+eCb8iofIJEo6d1tFY94X\nCcclxNQw6hxV9hU3j+OTaK0S6qfPpL8lQqv8jpnS875ILE+N+aaCrq79vph8SRQRERGRAphEERER\nEcnAJIqIiIhIBiZRRERERDIwiSIiIiKSgUkUKS5qw5gkw0AyscYmTiqJf7g9/u2V+DvrR+lx9IhJ\nFBEREZEsCZdExcGwEqrS9P5rIDgNhBCXNNNuCTTtS7zFG8+UGvsq2Q4Zp30Rl3BJlNL8upDa0754\nz/viTe4ZnERnvtKHR9Z0BlG47nh1ExWPLy+h6lOvv0gpONQ6sT36yfDLU8hpX4K1vxanfZFTtuwp\np7T1fpZ0SZQuxAjQ6lcubZRgxSNMoiQqYqNtpYvChUrKNxG+63g9j2jg4MAbs6uM8WrqBGuYUHuT\nYLsad4J++6PygZFcfARxJGLXSrokioiIiEgJTKKIiIiIZGASRURERCQDkygiIiIiGZhEEREREcnA\nJIqIiIhIBiZRpILojGOSBMPIaABbOakk+OEWPGd9SfB9DYTTviiPSVSCUWpkXlVoYQAaLcQQhzTT\naqGOXyzHgJOBXTEeJddB44jl4phEEREREcnAJEqE39efUR1y3/d5DKd9iZePzRr4ujoaEXjuZiTT\nIIjFGmy51qZeiG/q9Bgpp0LIaUdifCoFnfYkgYTcRw3svuRpX6IZq8bei5IuiYrp9A0hJ0+Vtp7S\n9ZIP17QvUalKxrQvXo8jmH4hyKbsKWMSui0SeufiW9App9Se9kXyipFcd8LbNh7eupIuiSIiIiJS\nApMoIiIiIhmYRBERERHJwCSKiIiISAYmUUREREQyJF0SFdMRW0PU7bVI6RiVKC+MMqLVxKrUM1po\ndIYpEK/Fdx2vbhJBlMGq1sC/qjUjodsiSY+/51/2tTqEgiD5L2ky4g/1HhTgtYCDbUZw4Q33/TfW\nw2xIkXBJlNJDGPiVF8X/XPrWJGn02EDhKdATldht6ReHCIjUEY0hLqIzPIJnffKPr1isruVqNltY\n0YccJiQO/g+tMaGaLJGaU6l9UXz0mVBnoBbaX4tJjMYyq4RLooiIiIiigUmUiIQYsVwBfrFo7NOA\nmwbiSo4Ry0kpanVZKcXGcsRy0WtIEnQyrY9YLv3SEsVgNfYVKZOoZKCxTkdERJQIki6J4rQvFBKn\nfYk6rfZOrcaliITeufjGaV8UqSpqki6JIiIiIlICkygiIiIiGZhEEREREcnAJIqIiIhIBiZRpLho\n/dlVqyMOJxQ2cVLR7NAlKkiiXXVLxn1WG5MoIiIiIhkSLomKh79EqkmtIRwUKVWh2CIphlODyKVe\nuyk37Ut8Xc7YF6NHqZZOuiPGb65ExddVh4iIiEgjjGIrOJ1OrFmzBseOHUNqairWrl2L0tJS9/Jt\n27bhD3/4AwRBwLx583D//fcn1ics//lOole1T11CDJvVd2oRrf62rvQ9HbKKi0bjKFWHRo9jMlHr\nEEjpIqFWUbtrCELob5WTomtqfNYXqV+9ybkcyb1Wa6JdPIh+E7VlyxbYbDZs2rQJd9xxB9avX+9e\n1tfXh4cffhh/+tOf8OKLL6K4uBg9PT2qBkwyJFJSS0REpBGiSZTFYsGSJUsAAAsXLsShQ4fcy/bv\n349Zs2Zhw4YNWLVqFSZOnIi8vDz1olWAZqd9CfJY7XrJh7ut1P+8I2vaF13gx4qJQV/RbO/0aItE\nO4VCTRmUUL8kJBK1j4vU8iO7KTW81bV7dXAT/Tmvr68PJpPJ/dxgMMBut8NoNKKnpwe7d+/Gq6++\niszMTHzjG9/AwoULUVZWFrJMi8USeeRBdHR4fxPmqss8+vzwkSNhxbG/cj9SjXr39idPnsRpmfGn\n1dXh3CB1N7f0uh+7vuZsbW2FxTLgtZ5YzF1dXQDGeb3W2NSEKRK3D8Zut3vHsS94OcPDwwHrmRJg\nXa8yJcbm+ha4paUFFku/17KOzk6/Y+5ZbmtDA4C8oPUZOztxnsfzzs7OsOOsra1DNtpD7wSAnpqT\nADJClnn2xCl4ftZpbW0FADgFJ6xWK5A1ku6Fe1xbW8+4H3tu6/A5zoODg7BYLKirt3q97nA4FTuP\nPb/WFytzQl0dgPEB1+070Si5HF++61ssFpyq73M/P3bsmOyyg3Fdq4KdL5HuQ319PSwp/v0XALq7\nuwAAQ4NDXq8fPnwYbW0j55RTwjGOpC0s+yzQh3hTPXjwIBpNom9RkvQPDLjPFZdjx45hsCdN0vau\n/ayrr/d7LdB6Uh08dBC5WYH3sb+/36u8wuZmFHnUY3dIO288lzU2NsIRZF3P/h6qnHHV1Zjps+xk\nTQ3ShltCbg8AHR0dAEoAANXV1dAPNIVcv7p5UDQe38fRJtpDTSbTyMV6lNPphNE4sllubi7mz5+P\nSZMmAQDOP/98HDlyRDSJMpvNIZdHYldNJXByLF7fuuaecw7wyb/F43h+5IK8aOEipKeNNVN5eTkg\nN36PZNS37uOdR4GDI4mUTqcDBAEFBQUwm+e6YxGNGUDHh4eBNu/XphQXS97ez2jdrmPuLqfCDLwQ\n+ARISUkJWE9bgHW9ypQYm+6FRggCUFhYCLP5HK84J02c6FeO5/ODtT1AozV4fS3eF4KJEyd69aeg\n23kco7KyaTCbp4ruR22vEzjcHLxMAM3ONMBy0v28oKAAOFINvU6PrKwsACPf5IR7XKvaDgNVZ/3q\nNvy/NmB42P08PT0dZrMZPY5TwKdjH1AMBr3889ijrQDvbz5Eyzx4EDgYeN1WfRbw2TFp5fjEYjab\n/c6zdlsd8NlpAMDs2bOBDzrCK1vEp7UHgGprwPPFYrGI1+PTju71R18vLS2F2VzquxUAYOsRC1Df\niLT0NKBvLHGeO3cu2gYaR/pYqGPs2W7hcm1bYYZeHyCJGl0+f/585Odlhl9+APv+vguA97ees2fP\nxrzpE0Jv6LGfFosF00pLgd097tcCrSeJax/PnY/JeZl+xxIAMjMzvct76y33Q7PZjGG7A9jUFDQW\nz3Vdr02ZMgULg8To2d8Dcdcx+kHOU/n06TAvKPJ73deZT04Ap0Yez5gxA+a5BSHX12W1Ax8F/iDg\nikfSuRKhUEma6M95FRUV2L59OwCgsrISs2bNci+bN28ejh8/ju7ubtjtdhw4cAAzZsxQIGQiIiIi\nbRP9Jmr58uXYsWMHrrvuOgiCgHXr1mHjxo0oKSnBsmXLcMcdd+DGG28EAFx22WVeSRYlqegNWU5E\nJJmQ5Pd7JdOI9NEimkTp9Xo8+OCDXq+Vl5e7H19++eW4/PLLlY+MiIiISMM42GaCUeuDlpY+v0UU\nS5J/EpVLM80WKpBA99doWHxFG+cUamzNnAfRwi+uRDGJIiIiIpKBSRQRERGRDEyiRPh9mxnTG/Nk\nfpeswHfQfrPfRFyiOpSOS86NmNFoG886IrlZVizWYMu1evzjkmqNGdm8L2pf6kT7XhLcBB1y2h0t\n7L7UaV9kFC1397T254DkS6Jied9EqIOv5ujIGut0mjbaVrpoXMGkHBeV+muwUanZU8Z4zyKQWC0T\n8lIUvTAoAF2w9ELl67jk0iOII+xN46AzJl8SRURERKQAJlFEREREMjCJIiIiIpKBSRQRERGRDMpM\nkU3kQYjSf7e08OeVhKeJvwhRtCT64RY87lROhn//+YrGPu/evRu33nqrex7doaEhXHHFFVi9enVY\n5fzmN7+BIXMiBs8MwNpWhQmzlgdc77PPPsOUKVOg1+vxhz/8AWvWrIl0F8KScElUsH8dJQ+V/s2l\nSCEKDhss82KQ9N1DJs20W4hA4u7cj7Nw45ly/65MsoMmM+f6j//4DzzyyCMAAJvNhssuuwxXXnml\nrLLSc4qQnlMUdPm7776L5cuXo7y8POoJFJCASRQREVGy+99tz+KLJ3YAL/8YT/f0AwBy/6YD7skI\nuP7FQ3Ys6rd5vZZhG8A1e18ZefLyj4FrrwW+9KWw4ujr64Ner8e3v/1tpA0CjWeGUPS56/HU4xvw\nf2fa4XQ6ceutt2Lx4sV47720OxrJAAAgAElEQVT38MQTTyAvLw/Dw8Oo+MIy9HeexJlTn6Kw4hs4\nc2oPWk5ux1UlJbjk97/HggULUF9fj7vuugsPP/ww7rrrLmzevBk7duzAo48+irS0NOTm5mLdunU4\ncuQInnrqKaSkpKCxsRFf+cpX8P3vfz/8hvXBJIqIiIgU8+mnn2L16tXQ6XRISUnBvffei6effhqL\nSqajz1CB03W7MHtmDp764yPo6enBN7/5Tbz66qtYv349XnnlFeTm5uK73/2uV5n2oT50V2/FF+av\nxJ+fuxW/tdnwuc99DqWlpdiwYQNSUlIAjPxkee+99+KFF15Afn4+/vrXv+KJJ57A0qVL0dzcjNdf\nfx02mw1LlixhEkVERET+Nl78bWy8+Nt447dX4sY7XgMAPGBORcWq/wq4/rZddfjDSwe8Xvuvynfw\nzsKR9d/47ejPcW+/LVq35895Lk8//TTycycAZ4Ghs604YGl13ydlt9vR0dGBnJwcjB8/HgCwaNEi\nDHlsP9zfhdTsAhgMRugA/OQnPwlYd09PD0wmE/Lz8wEAn/vc5/C73/0OS5cuxaxZs2A0GmE0GpGe\nni66H1Lw33ki/G7Ei+LNiH5Vy73nQ4VpXzR7B6oG4opGCJ51qFld8GlfkuzeEBWp9UcMKf0wZN2q\nz/sS+3M11jR/c7vU01zibuhH34tSTZPwhQuX4bnnnsNTTz2Fyy67DBMnTkRvby+6u7sBAAcPHvTa\nNiVzAoatHXA6HQCAW265BW1tbdDpdF7tOH78ePT19aG9vR0AsGfPHkybNm1kd1S4bzLpvomK6c2n\nIW+K9XoGRd8a4+2G21hyTfsSjaokTOni2189n0cSY7Bt2VXGeLZFwrWLtBmoKAZ0wS79ak/7IrX4\niKZ9Gdk2p+Q/0Nz0Eb75zW+ir68Pq1atQmpqKu677z7ccMMNyMnJgdHonZ4Y00wYX74U+w+9jJVT\np+JLc+ciPz8fs2bNwp133olf/vKX7jrWrl2Lm2++GTqdDjk5OXjooYdw4sQJ2XGHknRJFBEREalj\n8eLFWLx4sd/rzz33HD56bBNwBtAbjPjej3+Oz88r8Fpn6dKlWLp0qfv5/mPt+NfJXcicWA4AyJl6\nPuZmFeJPz/4Q+N73AAArVqyA2WwGAGzevBkAcMEFF+CCCy4IGdeOHTsi31nw5zwiIiIiWZhEERER\nEcnAJIqIRGn9/leihBeDk5B/IBHHJIqIwqbjpDuKUW40bZ9y+f5H0ZLEfY1JFBEREZEMTKKIiIiI\nZGASRURERIq45ZZb8OSTT7qf9/X14dJLL8XRo0cDrr9p0yYMDw+HVUez0Yh//etfEcWpFCZR4Yrq\niOXedcX2LhTvH721ekeM0odHXnnRaB3B45H8GxLkthdvOFVOTEcsDzVguXKhxKT8eKdWvwgrBqk3\n1nmEumbNGvzzn/9EdXU1AODXv/41Vq5ciTlz5nhvMtr5nnzySTidzjBiAj7NzMS+ffskb6MmDraZ\nDHiHKRFRUrph7fvux48ctCHV47mngSG732vb5lzkVc4XzyvG9SJZQ15eHu6991784he/wG233YbG\nxkY88MADAdd98cUX0dHRgdtuuw1//OMf8dvf/hZ79+6F0+nEt7/9bRRMN+N03U70NloA6JCeOxWF\nRRX4c14eBt98E4sWLUJubq54I6go6b6J0uy0L9JWU7xe8uFqqyh84yjlsOh0ep/n4W0fvOCwFyQh\nXcCHiSDBdifBBLn2aOU6LiGOSy65BGVlZbj77rvx0EMPjb3v+mx67bXXYtKkSXjkkUewbds2NDY2\n4oUXXsDf/vY3/OlPf0K/9SzONOzF5HOvQsmFP0KqaTIEAN/t7sZXv/pVLFu2TPn9CxO/iSIiIkpQ\nf/nFl3HFHa8BAG6bn4qKVV8OuN57n9bj8RcrvV67+Oh2vLfgUnc5AIB3GyTVe9VVV2FwcBD5+fmS\n1j9+/DiqqqqwevVqAIDdbkdHeysKzluBnpptGO7vRvr4UiC7WFJ50cIkioiIiGJCp9PB6XRi+vTp\nWLx4MX75y1/C6XTij3/8I/ILinHm1F8wef7V0BtS0Lj7afRmTIBeEMK6j0pNSfdzHqkvWvfe+954\nT0SRSaZzKnn21IMGj+/555+P7373u7jkkkuQmZmJVatW4eqrrwYAZGRmIm1cARp2PoGGXU/CkGpC\ndnYBZtls+PDDD/HWW2/FOHp+E0VEREQKW7x4MRYvXiy63oYNG9yP7777bq9llcfbkVOyGDklY+UY\nepowd2gI7733HgDAYrEoFLE8CZdEaeTWu5hR7d5DnQKfYBQKLqL7qbVyc2ac0UyzhfpzhmaClCa+\noo1vSnWNOOtimrFp0ya8+eabON3UjoaBkR/A1tX+A2vuvRuLFi2KcXSRSbgkioiIiLRj5cqVWLly\nJT56fBN+W5sOALjnfz+PRecWxjiyyPGeKCIiIiIZmEQRERERycAkSoTfnxli+e+GmP4g7zPti/b+\n5AFA+X8XyZl6IRpt41WHiv0i2L5o9PDHJbX6i5RiQ62jdj8WK1+r1xglhZx2RxP7L+3aIus6GfYW\nru20dWMak6hkwLshiYiIFJd8SZRem9O+qDrDBJMo6UbbSheN71qkHJcQ/VUXQU8Jti27yhidmudk\njIX6J2O8/csx0QSfkUnd4yK5+AjiCPeaFck1LlqSL4kiIiIiUgCTKFKcNn7LJ6JwJfqpK3h+i5Lo\nOxsAr83KYxJFREREJINoEuV0OnHfffdh5cqVWL16Nerr6wOuc+ONN+KFF15QJciwaP8nVFVp+X4G\nQakRyyMbslyRGJKPRtot5H2FGolRIi2fqxQYDxn5Ek2itmzZApvNhk2bNuGOO+7A+vXr/dZ59NFH\n0dvbq0qARERERFokmkRZLBYsWbIEALBw4UIcOnTIa/m7774LnU7nXoeIiIgoGYgmUX19fTCZTO7n\nBoMBdrsdAHD8+HG8+eab+PGPf6xehEREREQaJDoBsclkgtVqdT93Op0wGkc2e/XVV9HW1oZvfetb\naGpqQkpKCoqLi3HRRReFLNNisUQYdnAdHT0B6zKPPj9cdTisOCorK5GeqndvX1NTgx6Z8ac2NmJ+\nkLqbm8Z+DnUKTgBAa2srLJZBr/XEYu7s7ARg8nqtoaEBUyVuH4x9eBhIG3u+b9++4Os67AHrKRap\nQ2psztG/mLS0tMJiGfBa1tXV5XfMPcttbzgFIDdofcaeHpznU164cdbV1cGi7xDZC+BMdQ2A1JBl\n9lU3ej1vbW0FMDIye5+1D8iUFpOvlpYz7see27o+ILkMDQ3BYrGgrs7q9brT6VTuPB7t776xBJJX\nWwtgQcB1rXUtksvx5bu+xWJBff3YPh89elR22cG0t49cq+z2wOdLpPtwqr4ellT//gsAXV3dAADb\n0JDX60eOHEFr68g5JQjixziStti3bx+MhuA3GR06dAjN2aJvUZL09/e7zxWXY8ePwXbG/x7fQFz7\nWVtb5/daoPWkCriPo9e3gYFBr/IKmprc11CLxQK7Y+yvdqHq9VzW1NQEBFnXs7+HKmfciROY6bOs\npqYGGfbWkNsDQHt7OzD6bnTy5EmkDDWHXL+mdTDoMs/9UjOnECPaQysqKrB161Z85StfQWVlJWbN\nmuVeduedd7ofP/bYY5g4caJoAgUAZrNZdB25dtcdAE6MdQbfuubOmwt8tE88judH3rzOW7gQpowU\n98vTy8oAufHn5gaNq7rnGPDvkURKr9MDcCI/vwBm8zx3LAJ0om3X/dFRwKdfTi0pCVqvqNG6jSkp\nXi9XVCwCNjUF3MRoMAasR+wUkxqbflMTnBBQWFgAs3muV5x5eRP8yvF8XtXQC9T1Bq+vwzv5ycub\nANT0i8f5/FiyU1o6DWZzif86PuoHdMDBhuBlAmg1moA9Y2/gBQUFQNVZQKeDKcsECCP/1A73uB7t\nOAocOuZXt/G1dmDI5n6empYGs9mMM0IDsGvsA4per5d/Hj/vnRh63mAtWuaRI8D+wOu2p1UDO6uk\nleMTi9ls9orLbDajy14P7B7Z5zlz5gDvd4RXtojP6v8NnKiFIcD5YrFYxOvxaUf3+qOvl5SUwGwu\nC7jp9uP7gNp+pKamAtaxDyLnnHMOum3NwOGz0OlCXG882y1co9tWVCxCitEQdPm8c+ehaKLJf7kM\nlc9/6vfa7FmzMX/GxNAbumOtwL59+1BWNg34dKRPeO17uO3hu4+ex3L0fEjPSPcu79133Q/NZjOG\n7Q73NThQLJ7rul4rLi4OGqNnf3cRPHJc93adnX7blpVNh3mh2MdkoG/XSaB25PH06eUwzy8Mub7x\neAfwL5/6RtvHFY+kcyVCoZI00SRq+fLl2LFjB6677joIgoB169Zh48aNKCkpwbJlyxQNNBp0uhiO\n6hBqlGCvIcsV/gsI/1IinWvE8qgMWC5+XHzX0fkOoy0zzmBVx6KraLV3esaVTP+kS6Jd1aSgsyWo\nPWK55BUjGLE83G3joC+KJlF6vR4PPvig12vl5eV+6918883KRUVESYPj/xFRvOJgm0REREQyMIki\nFUTnuwVOYUCksAQ/pwSvxwm+swEIvGgqLuGSqDj4CVVVmr6fQbHgIvhNXq/lBtIuzfSrUPcVxtmx\nja9o45tybc2jRt4SLokiIiIiigYmUUREREQyMIkiIiIikoFJFBEREZEMTKLE+P6bIYr/bvCtSdDQ\nTY3a/ZOHFgKLQgweB0BQ8a7vYHuihVZOHOq0ppRSQ62j9jkuWr6GOplqbRHyAKhUZxgkv+fIilXe\nDmrpfRBgEpUcNPPXKiIiosSRdElUTP8GHerv2SrO+sIkKgzutlL/Y6CsaV+CPA677qCvR7+vaLV3\nJvJpk8j7Fu+CTjml9rQvUouPaNoXddePhaRLooiIiIiUkHRJlOCM4Q/NIX5Y91yk+O/vUb6BKVrV\nqTL6rrtM9T8CSYnfdx0hyOOw6w76uu89gBFUIpEGbv0ISLv3/UUu2L4l+ijenvfTaPX4ClJvQ5IT\nv8T3IOUrlrepVo+Rp6RLopJSPPRESlpau1GUiEiqhEuipNxnEmEF6pbvWZXfcwnJkErxaeltLvQu\nho5U9f4hIQZlqhirQ6dikuyqRjPHP+R9hZqJUhoNhBsqhHhrzlCU2hfl71eVuSxKJL3nRFC6vK20\n9aVAwiVRRERERNHAJIqIiIhIBiZRRERERDIwiSIiIiKSgUmUCL9b2JJ02hf/WLRKC5El0LQvQf8K\nr4G7XhOEWpcUKeXGdNqXCJdHk2qxJMi0L/KGxuC0LxQvEulvNkRERBqRdEkUp32hkDjtSwSlyqO1\nvyy7JPJpk8j7Fu847YsiVUVN0iVRpD5tviWSPDyaySThx+X1fFdO9H0NQJVZHpIckygiIiLyFw9f\nBcVYwiVRyX7I1dp/Rc4lhU7IiH7G4kVBHjVvXg/niCbQiOXxFm88U+pnah4x8pVwSRQRERFRNDCJ\nIiIiIpKBSRQRERGRDEyiiIiIiGRgEkVEREQkA5MoEX7DakRz2hefugS5fw1R4F9A/s2gzfFGlA5L\nzn5Go2m8q+B/huKZelOKiJccahW1u7HYuaWpa4xKsWh81hfJMchpHrlNqoV28ZR8SVQs/1YssW7F\nI+RfqcOmi8YFXMro+b7HTqmR7YNtHIOuotURyz0bI9FOoVB/+efQC7EW5HxQ/bhILD+SOMLcNhYz\nKIQr+ZIoIiIiIgUwiSLlRelreC19209E2id4PU6+C0i418ywBsJNUomXRCX5MVfrq3hFSlUqtoi+\nTlYmhGSjmWYLdexjObm4DPEVbZxTqrH5Uyf5SLwkiojiCj/tElG8YhJFREREJAOTKCIiIiIZmEQR\nERERycAkioiIiEgGJlFEREREMjCJEuE39UBMByeS+S8mRaZ90fm+oElKTxUhp7RoNI3nbqpZX7Cy\nNXr441MMphSRVLfa1zqR4rU0DpxaoYS6Xmlh9wWJ7x3RvE5KjSlamERFU4iD7zm+k+J9RGOdLh5E\no8WkjOml8xn7yHsaBPlRamjWF81O++LZRvEw/UQ4QnW9xNrT+BO0/VW+jksuPoI4wt40DjqjUWwF\np9OJNWvW4NixY0hNTcXatWtRWlrqXv7ss8/irbfeAgBcfPHF+NGPfqRetBQXovWWqM233gTDRk4q\nmpr0VwWe32Ik+K4GlpQ7rS7Rb6K2bNkCm82GTZs24Y477sD69evdyxoaGvD666/jn//8JzZv3oxP\nPvkER48eVTVgMUk/eaaGRyxX6mvYyOa/TPL+IZNmmk3it7lxIc7CjWeKDViuUDmUOES/ibJYLFiy\nZAkAYOHChTh06JB7WUFBAZ5++mkYDAYAgN1uR1pamkqhEhEREWmHaBLV19cHk8nkfm4wGGC322E0\nGpGSkoK8vDwIgoBf//rXmDt3LsrKykQrtVgskUUdQnvb6YB1mUefH6qqCiuOAwcOICvd4N6+trYW\n3TLjT21pwfwgdTc19bofOxxOAEBbWxssliGv9cRi7ujoAFDi9Vp9fT1cP8DKbfvh4WEgdez5/sr9\nQdd1OBwB6ykSqUNqbE53+7TCYhn0Wtbd3e13zD3L7ayvB5AdtD7D6dNY6FleV3fYcdbX18Ni7BTZ\nC6D3RB1cp2CwMq11LV7PW1qa3Y/7+vqADGkx+Wr26G+e2w4PD3utN2SzwWKxoKau3+t1p1NQ7Dz2\n/AlJrMzxNTUA5gZcd6CpQ3I5vnzXt1gsqKuzup8fOXpEdtnBuK5V9iDnS6T7cOpUAyyWnoDrdnaO\n9Gubz/E+evQoWlpGzilBQgyRtMX+yv1INQb/MaSqqgrtjSmyy/fUZ7W6zxWXEydOwNnXIGn7ffv2\nwaDXoba21v2aEsesqqoKbQ3e++g6HwYHBrzKy29sxBSPeoYd0s4bz2XNLS1B1/Xs76HKyT5+HLN8\nltXW1iJbaAu5PQC0tbUDKAYA1Jw8ifThlpDr17UNBV3muR9q5hRiRJMok8kEq3WscZ1OJ4zGsc2G\nhoZwzz33ICsrC/fff7+kSs1ms/hKMlkaDgLH+4LWde68ecCWPeJxPN8IADjvvPOQYxr7dq2srAxl\ncuOvrw8aV+2ZE0DlYQCAwaAH7A7k5+fDbD7XHYtozADOfHICOOX9muc9bGG3/WjdKSneJ/qihYuA\nzc2BtoDBYAhYT+jTRXps+pdaAIcD+fkFMJvnecWZl5fnV47n82Nt/UB1d/D6urq8nuZNyAN8EoiA\n23kco9LSUpjNpf7r+Gi0pwD7a4OXCaAj4ySwc+zb38LCIuDQMQAj5yYcIWIK4UT3MeBgr9+2KW90\nAINjF6601FSYzWZY9Y3AzrGEUq/XyT+PPdoK8P4ZTrTM6mqgM/C63ePqgW2V0srxicVsNvudZ6ed\np4BPR5KQc+acA7zbHl7ZIvY1HgSO9cEY4HyxWCzi9fi0o3v90ddLSqbCbJ4ecNMd1fuBmlNITUmB\nuxMBmDNnDs4624Cqs9AhxL56tlu4RrddtHAR0tMCvAWNLp83bx6m5meHX34Ah/65G3B6vzZz5kws\nmj059IajsVRUVOBA5f6RLwl2Brh+hNseHvs4ZXK217F0nQ/pGRne5X34ofuh2WyGbdgBbGoKGovn\nuq7XigoLg8bo2d8DcW93+rTfsrKyMpgrpvi97mvgs1pg9PPb9PJymBeE/miderIT+LAj4DJXPJLO\nlQiFStJE74mqqKjA9u3bAQCVlZWYNWssBxUEAT/4wQ8we/ZsPPjgg+6f9YiIiIgSneg3UcuXL8eO\nHTtw3XXXQRAErFu3Dhs3bkRJSQmcTif27NkDm82Gjz/+GABw++23Y9GiRaoHTkRERBRLokmUXq/H\ngw8+6PVaeXm5+/HBgweVj4qIiIhI4zjYZriiOM6G32DpMfwLt+9ea3a0EaUDk1FetIdiUbNfBB+x\nnH/2Vop6o2FHVrfa3Viz15AA1DqnQ5argQaSfG2R00Cy909b1x4mUUREREQyMImKplADBYZ4pma9\naojWNzFqjq4cjalIJE374rOO11QkkQw6GrQ++WXKpa3PlWO82kKrQSpMABJ8Xz1GLI9hFKHogl3X\nlJgDNcQ1MzrTvoS3bTx0RSZRRERE5Ic/24tLuCQq2Q+5Wt8kKFLsaHA6wSmyonqxxN3UIBqhmQl4\nE2jaF820aRJQqmvEWRejKEi4JIqIiIgoGphEEREREcnAJIqIiIhIBiZRRBRTWv2XFBGRGCZRRERE\nRDIwiSIiIiKSgUmUCL+xyaI57YvPDx2x/NnDvxm0+SOMb5tFXJ6s4rTZNhSc4Ixs2I2I6lZrShEJ\n/TBU3Wqf4mLXEG1dY6Ifiyb2XuqsLzKKlnutFjQ2zASTKFKBJk5/IgpXgp+6QtAnyUFTeWmCYBIV\nTSEHCpS0muL1UhBRuNrImvYlyOPwKw97gWq02zt1AR4lhlB9j4OAapTq13H/8gOOWB7RfFNhTvsS\nB+9diZdEab/NVaVWp1Oy2KBzQ0ndPrIhyyOqO2lppdlCHT+9VoKUhl0xmpRpbCaY5CvxkigiIiKi\nKGASRURERCQDkygiIiIiGZhEEREREcnAJIqIiIhIBiZRRBRTHLqGiOIVkygiIiIiGZhEifAbmj6a\nQ776VhXDgWXi5dsCpQ+PnKkJoj0qsKBivwi2L/HSH6QSnFqaVCmaxWr3SGopMtWm5glRsNJTWMkh\n9doip33ktmnAAUBjiElUNIXskCqOjhzlATijlUSoWY8uGhcwKYND+q7jNbS9/KqDDRoYizxdW5fE\nMd6zCGg1SnlCnbsJtqtePJMCLSQpgQS99ihwYEIlbZKLjyCOROxbCZdEJf2IsirtviLFKnYGRXIS\nJ3n/kEkzzaaZQCieaODSQwkq4ZIoIiIiomhgEkVEREQkA5MoIiIiIhmYRBERERHJwCSKiIiISAYm\nUUREREQyMIkiopjS5mg9RETimESJ8b3CR3E4ar+qYzhIiW/d0R6VWzLlhyyPxiYREXTqncbB9kVr\nowZHKtQghOrXrVK5MaxbCtG6NXSNUSuUkOVqYP+ln+dRvFBqbKw4JlGkuGid+1odcZgoXiXTOaXZ\nD4JqSsqdVheTqGgKkUF7LVI601Zr2hdVStUGXRSuNVJGT/ddx3vWF+WnX4jFMdVqP1LzlNSyJNpV\nTQra/ip3wkClB5w7L5JpX8LsXfFw3iVcEhUPja4mtaa9UXLaF12En4YiOcY6KfPVkR/NTKeUQCc4\npyCKHsVmfeEhIx8Jl0QRERERRQOTKCIiIiIZmEQRERERycAkioiIiEgGJlFEREREMogmUU6nE/fd\ndx9WrlyJ1atXo76+3mv55s2bcfXVV2PFihXYunWraoESERERaYlRbIUtW7bAZrNh06ZNqKysxPr1\n6/HEE08AADo6OvDcc8/h5ZdfxtDQEFatWoUvfvGLSE1NVT1wIiIioljSCSLzHTz00ENYsGABLr/8\ncgDAkiVL8PHHHwMAPvzwQ2zbtg0PPvggAOCHP/whbrrpJixYsCBoeRaLBWazWan4/Tz+/Gd4z9Ls\nfv6d8d0jD155GQAw+JWv4bmBfO9lATzVkwcA+GbOaWTone7tMXceMGeOvOAGBoB33h55fPU1Xot2\n9Gfi8FC612uFxmF8NfusO5Zrd7+I3MuWhaziYONZfJpV6vXad5q2AcePB6xXjKtug9MBh97gfv1b\nuT346+nxQbcL1LbDH3+MlI52PPWl74ytt/Up9/NQxyNQTEXGYVyefdbrtQVnG7C4JGtkRdcx89jn\n1i4r3tBPDV6fzQa8+YY7pmy9A2edBq9VAm3nqh8AFqYP4HMZA6L7cbp3EC86ioLHAqB/wIZ/DBa4\nn89IHUK1LQ0AMP1sC2qyC0NuH8x2axaOjZbjua3nfrh8Z3w3Dg2lYVd/lt/rcgSqQ3KZNTV4avz5\nAdcdGLDh76NtFW5fujG3C0+fnuAVx78H07F7IBMAcGV2L147Oy6sssW822dCw3BqwDJ7TvdgfG7w\n88szds+YPV+flzaICzL7A277au84dDj8P0NfZjqLE7ZUnAzQNwLVLactXNt+K7cHqQFGtXUtv2rc\nGUwyOMIuP5Atp4ZQO3quuCzN6sPMVJukWP83twdnz3SjOaMAO0fPg0DnTbj97r/HncFEgyPgOZHX\nfxrXFDvHXqjcD9TUjDy++ho4ADwToN5A/cL12oX99TinODtgTJ793SV7oBdnM3z6fUszsGvXSF2j\n18nPZfRjYfqg2G7jSNNZfJI58v70hYx+nCuyTeNwCt7p84/3O1ufAq6+BgaDHrnnFuCLV1wqWnck\nQuUtoknUz3/+c3z5y1/GxRdfDABYunQptmzZAqPRiNdeew3Hjx/HT3/6UwDAnXfeiauuugoXXHBB\nyGDU9MErB7FjMPTFh4iIiOLf12zHUfHtS1SvJ1gSJfpznslkgtVqdT93Op0wGo0Bl1mtVmRnB85y\npQSjhPPmnYvch/+JQr0APQRMmJwzsqCnB3A6gQkT0NzQhXHjMmDKyQxaTr9dh55hoDhjNMe0DQMt\nLUBpSWQBNrcAOTlAln/dtVYdpmQAeh1QbwXKsgTodCOxdLf3YEqWHsgZJ1pFfW07CgvHo0tIhckI\nZKcIwO7dwMxZQF54CaarHYrSBdScbMPEKZPQLxhQmCHA4dShtn/kxrrcVMDZ2ITeCQUozTbAoPfP\nzVtb21AwMABH8VTsPWvE7Gwgt70R7XYj0rKzkDPBJCkmh1OH+n5gummsjuEhO5qaujFt+uSxFTu7\nAIMeGO+9zw31nZg0MRvpWWmBK2hugXPcONTChOlZArptoyOt60amnpqQ5r9vw04daq1Aqh6YliV9\nRPamU50Yn2dCpik96DpV9WcwMOzEzLwU5OSZUNOnQ0kmYNSPHJOSqRNhTDUE3T6YWqsOqb1tKC4c\nazOHoMPRXiBbZ8dwZzfKyibBNcj7nm4dys40o3fIibKZhdAb5P0vxe7U4bMeIMsAnDMOMMKJj6t7\ncX7ZOGSmiA8JfaauGYPF9xIAAAetSURBVEMTJmNytv/lq6mhC7k5mcgalyEplt4eKwYGbMgvGo+u\n9jNoGk7BzIIsZBhGjmG9VYfCdCDVIKChX4eJaXAvU0JNnw6lmfA7X1pb21BQkB9y2/6zA+jpscI+\nYRIK0oG00biGnTo0DoxcP4IRhJHjX5oFdA2NzHdpcwJTMwV3XK4+FsgZmw5DTmByevht0WfX4eww\nUJgReNsBhw6dQ2OxKMV1rjgMRrQNASUSyj87rIPVDhRkCO5j4tknXE7bdLCF0R4DDh26hoApozHY\nnDp80qnD0klOCA4n6uo6ML0833+E9Lp6oLgYSBnp+62DOmQagHEpY/W6rtkpPd1ISx+5ZgwN2NDa\negalZZNCxuXat/5hBw439OIL5Tlo3XsIppnTkT3e45vohkZg0iTYUtPRMgCUhnHNq67thDMnF7Py\nRNOPkar6dbC2dKB8Si4aGrpQmu6AITcHMGXBYDBAKFusak4BhP7yR/SbqPfeew9bt27F+vXrUVlZ\niccffxxPP/00gJF7oq6//nq89NJLsNlsuPbaa/Haa68hLS3ImxPU/zkvWnVQ+HhctIfHRJt4XLSH\nx0SbYp1TiKaCy5cvx44dO3DddddBEASsW7cOGzduRElJCZYtW4bVq1dj1apVEAQBt912W8gEioiI\niChRiCZRer3efeO4S3l5ufvxihUrsGLFCuUjIyIiItIwDrZJREREJAOTKCIiIiIZmEQRERERycAk\nioiIiEgGJlFEREREMjCJIiIiIpKBSRQRERGRDEyiiIiIiGQQnfZFaWpPQExERESkpGDTvkQ9iSIi\nIiJKBPw5j4iIiEgGJlFEREREMjCJIiIiIpKBSRQRERGRDEyiiIiIiGQwxjoAJTmdTqxZswbHjh1D\namoq1q5di9LS0liHlbAOHDiA3/zmN3juuedQX1+Pn/3sZ9DpdJg5cybuv/9+6PV6PP744/joo49g\nNBpxzz33YMGCBWGtS9INDw/jnnvuQVNTE2w2G77//e9jxowZPC4x5HA48Itf/AK1tbXQ6XR44IEH\nkJaWxmOiEV1dXbj66qvxzDPPwGg08rjE2H//93/DZDIBAKZMmYKVK1fiV7/6FQwGAy688EL86Ec/\nCvo+X1lZKXldRQkJ5L333hPuuusuQRAEYf/+/cL3vve9GEeUuP785z8LX/3qV4Vrr71WEARBuOmm\nm4RPP/1UEARBuPfee4X3339fOHTokLB69WrB6XQKTU1NwtVXXx32uiTdSy+9JKxdu1YQBEHo6ekR\nLr74Yh6XGPvggw+En/3sZ4IgCMKnn34qfO973+Mx0QibzSb84Ac/EL785S8L1dXVPC4xNjg4KFx5\n5ZVer33ta18T6uvrBafTKdx4441CVVVV0Pf5cNZVUkJ9E2WxWLBkyRIAwMKFC3Ho0KEYR5S4SkpK\n8Nhjj+HOO+8EAFRVVeHzn/88AOCiiy7Cjh07UFZWhgsvvBA6nQ5FRUVwOBzo7u4Oa928vLyY7WO8\nueyyy3DppZcCAARBgMFg4HGJsf/8z//E0qVLAQDNzc0YN24cdu7cyWOiARs2bMB1112HP//5zwB4\nDYu1o0ePYmBgANdffz3sdjtuvvlm2Gw2lJSUAAAuvPBC7Ny5Ex0dHX7v8319fZLXVVpC3RPV19fn\n/ioQAAwGA+x2ewwjSlyXXnopjMaxHFwQBOh0OgBAVlYWzp4963c8XK+Hsy5Jl5WVBZPJhL6+Ptxy\nyy249dZbeVw0wGg04q677sIvf/lLXHHFFTwmGvDKK68gLy/P/QYL8BoWa+np6bjhhhvwl7/8BQ88\n8ADuvvtuZGRkuJcHa2eDwRC07aOREyTUN1EmkwlWq9X93Ol0er3Rk3r0+rF83Gq1Yty4cX7Hw2q1\nIjs7O6x1KTwtLS344Q9/iFWrVuGKK67Aww8/7F7G4xI7GzZswE9+8hOsWLECQ0ND7td5TGLj5Zdf\nhk6nw65du3DkyBHcdddd6O7udi/ncYm+srIylJaWQqfToaysDNnZ2Th9+rR7uaudBwcH/d7nA7V9\nsHWVzgkS6puoiooKbN++HQBQWVmJWbNmxTii5DF37lzs3r0bALB9+3acf/75qKiowCeffAKn04nm\n5mY4nU7k5eWFtS5J19nZieuvvx4//elP8fWvfx0Aj0usvfrqq3jyyScBABkZGdDpdDj33HN5TGLs\nH//4B/7+97/jueeewznnnIMNGzbgoosu4nGJoZdeegnr168HALS1tWFgYACZmZk4deoUBEHAJ598\n4m5n3/d5k8mElJQUSesqLaHmznPdiX/8+HEIgoB169ahvLw81mElrMbGRtx+++3YvHkzamtrce+9\n92J4eBjTp0/H2rVrYTAY8Nhjj2H79u1wOp24++67cf7554e1Lkm3du1avPPOO5g+fbr7tZ///OdY\nu3Ytj0uM9Pf34+6770ZnZyfsdju+853voLy8nOeKhqxevRpr1qyBXq/ncYkhm82Gu+++G83NzdDp\ndPjJT34CvV6PdevWweFw4MILL8Rtt90W9H2+srJS8rpKSqgkioiIiChaEurnPCIiIqJoYRJFRERE\nJAOTKCIiIiIZmEQRERERycAkioiIiEgGJlFEREREMjCJIiIiIpKBSRQRERGRDP8f/+wMPXexaUAA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)\n",
    "print('Random Forest accuracy: {}%'.format(RF_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting accuracy: 99.57000000000001%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAEuCAYAAACu4EdXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXeAHNWV7//tyVGTZzRZM6OcNQ3G\nBmNYsGwvPL8fNrsk/7TrXfy8ttfGb81zXIO1WE8I44AX24AJwsgGSxiTbCNAIBAIhFBLI2k0Go0m\n55xz6H5/dJiq6uqu6urq7uqe7+efmaq699xzY52+VXWOyWaz2UAIIYQQQnwiKtQKEEIIIYSEIzSi\nCCGEEEI0QCOKEEIIIUQDNKIIIYQQQjRAI4oQQgghRAM0ogghhBBCNBAT7AItFkuwiySEEEII0YzZ\nbJY9H3QjCvCsjF5YLJaAl0F8h/1iPNgnxoT9YjzYJ8YkGP3ibfOHj/MIIYQQQjRAI4oQQgghRAM0\nogghhBBCNEAjihBCCCFEAzSiCCGEEEI0QCOKEEIIIUQDNKIIIYQQQjSgyog6ffo0duzY4Xb+zTff\nxI033oibb74ZBw4c0F05QgghhBCjouhs89FHH8VLL72ExMRE0fm5uTnce++9+NOf/oTExETceuut\nuOaaa5CdnR0wZQkhhBBCjEL0zp07d3pL0N/fjx07duD111/HP/7jP7rOX7x4EefOncNNN92E6Oho\nNDQ0wGq1YtWqVV4L7OrqQkFBgS7KyzEzt4C/vHUBU68cxlhiKnILMu0XXn8daGsDVqzAWyfbYXr+\neaR/ZBvwxS8C6en2NA0NwAsvANu2of7QMZw/dBwlbReAb34TkNmJk+XwYbuc0lLgkUeAwkIgNRUY\nHASeeALYtAmoqQGOHAFOnwaiooDMTMw/+Cu88pX/Qs6ZDzG9Zj0O//BBlI93I2rjBjQd/hDVB99H\n6aXrxWX9+78D588Dl1/uOmW12nDw/WZkLkuA5dApWN9+GxnmTfaLQ0N2HZYvB37/e2DrViA62mt1\nGjtGcK5pALkZSXj1WDMmpubQsvsBFD3/NAZu/We8GFuGljMNGK2tR9drR3CqfwH5r7+EePM2e92e\nfBJITgayslx9PzAyhfv2nUBBdjKyn3sa7w+aMBWTgKy0RKCuDvjyl4GyMnvbyTB4sQWH73kY5aeP\nIursWcBsRk/fGI4++AwqKvJgSkwErrsOeOcdICsLKC4GmpqAq68GPvc5vH5uAEkJsUhNigPGx4F/\n+Adg2TJg9Wp7ATYbRvb8DG/s/RvKWmtw2pSBwWf+jPbXj2LgX76C3M9uBzIzRTr1Dk7iuZ8dQPcf\n/oyK6z8Bk8lkvyCov4vz54FXXwU2b8YbB08h/uDfsOzSbcDICPD448DGjUBsLGC1Ao88gl8cbMLf\njjaheEUOlp2x4OCrZ5C3phTRnZ04uOtx5H37G0iYngBycuxj64UXgNpae/862bcPiI8HHD9y5uat\nOPheE7AwgRUli+08PDaDJx87hO5v/QAX+6ZRdMl6xMZEw/bii7j7xUZEvXcU54ZtKCrJRmyMYOzY\nbMCnPgVMTAAWC1BZCTjbQELf0BR2P3kcNU2DWL08Gfjv/8Z3X2xBeUkGsrKXLSYUzEcAwNQU8Nvf\novp8F9rPNSJ/s2OtGR52tdvhM92IiY5CWkq8e8E2G/Doo/Z2SksDAJx74TBaP6xBwZbVOFnbi2ff\nqEPx1ABSX/8brPPzOPiN3ci8dDOSMpbh0M/+gITn/4TUpx4HqqqAv/s72foBABYWxPO/vx/YuxfY\nsgWIsf92nR2bwMGfPY382RHEf/C+fW1woGadbGgfxvmmQZw7WY/0gy8j+dJtgMmE3r3P4J3GMVS8\n+hxMf/g98MlPLs7z06eBI0cwfegwXn32CAq2rIalaQQX24bQ0jWKFQVprrGRl5mEhLgY+zj86lft\na8arrwItLTh7oRudCZnIz072qqOIvj7gySdRl7UCFztHUZSbKr7+2mtAezua4zNx5t2zKD36mn0M\n/+53wLFj9vlRXu69jJ/9DHj6aeAznxGNv7n5BRx8rxl5y+LQ/90f4cSHDSi76pLFfM3NwLPPuo3b\n2uZBNHaOoDAnBV1dXVh+4gQO7juEjN3/haQLNfY1pbYWZ+76GTrH5pG/aSVwzz3AX/4CbN/ucQ40\nvXUCZ/96FKUf2WAvvmsUP9rzF1yVZ8JE4jK8caIN5YVpiDp7xn6v2LBBVs67+9/E/LkaZGxc7Rpz\nDZ2jOP/aMXQkZWF6dh5ZaYno7BvHB+e6UT4/DBw4YB+HDz8MzM8Dt94KbN0K63N/xsHjbcisPYOq\nX/0BTxysw5XvvYijF4dgPX0GGZUb7YXabPb5dvw4uvvH8f6ACRVF6fL90dVlHzfJycBVV8H24YfY\n+3Ybao9UYdOF44v3oEceAR58ELjhBlGb2Ww2HDzWgneqOpCfnYy3T7aj8O2DiIs2AffeC5w8ia6V\nKwNqU9ir4Xk+mmw2m01JQHt7O771rW+JHtmdOHECv//97/HAAw8AAH75y1+ioKBAZGjJEejYeW+d\nHcVbZ0ddxztvKwIAmC+xT5i33vkAP3u+CwDw8s9vsOt04gQAoPKyy2BaWMD5ffvwnVP2Cf7CLz6P\naJsVdb/+NcYuu0yxfGc5jbt2ofyHP8RUWRlqnn0W5f/n/yDjrbfQ+b/+FwoefVSUp/muu3D+xeP4\nzSe/ijWdtZhJTkVzWiG++ep/I+OJe7Fzv13fH342EzGpSQCAhPp6bLjlFpH+AHCudRLPvjuIpPgo\nTM5YAQD/95oYzC1fjrLvfQ+Zhw650rZ961vove02r/XZ+XQ7AOCSlck4UT/hOv/yz2/Av/3Lr9GZ\n4W7ofKThOHZsisLExo2yOv73y90YHJt3yfnst16wl3Vbkav9pHmEPPHwcbQuK8B/vPIArjn/Fhp3\n7cL3B9diEjG469hvkXtNJVbs2iWS45Rbl7cSd37hp4iKAu6+pQjF992H3GefFZWXfPo0fv+3dpwu\n3YJ/fXsvnrjqX0Tlv/zzG9x0+8lzna72/mrJEPI+vsljHzl1eXP/S/jFB/Y8PytvR/aLLyLz9dfR\n9cUvovPrX0f6oUNI2f1zfOlLv3Xlvf2tx/H41bdjRW48rnz1Gezb9nlsbanCj5/b6dZO5w4cwHR5\nOeLa27HpBvFY/+DCOF6xDKMkJw7/uj3Xlefx13rR1j/rOv5Y1gw+/ekKzNzyddz7P7/nOn/JymT8\nj49kuI7TDx9Gxbe/7TpuuP9+DHswMu7/cycmpu31LsUYMloaUFVqN/ic8xUQzMff/Q6TGzYg/5FH\nUPDoo6LxAgBl3/8+Ml9/HdX/+lV8P/3TbnKcLDt6FKu++U3MZmfj7MGD9nSO8b3ztiLX/4C9j99d\ndTnu++x3sHyiHzeZk/DftUmua8K2lCPzlVdQdtddmCovR82BA1j1ta9h2fHjaL/jDvT80z/Z8z95\nGC/HrcJHGo7jrhd34/Rrr2FeYpx7Q6hv9lg/fljRj/GtW3H/C90YSUrHPX/6Eba1nhbNc+fYe/Lj\nO/DcR27EZcMX8UH64g/fb92Qj+qWSbx2agRlefH452tz3PoWgFsfqMHZBp7yOnVzXn/m119Az3/d\nhYrvLY47b20eMzCALZ+293/jrl0Y+sxnXNferRnFoapRrDMN47zNfsO/02xF6poSAMDWq65C9MQE\nLvz2txivrHTlE44PAJi95d+x+39+H3nD3Xjsia+41nmnzvdeacPGz30OAFD72GOYEP6QEeCU+4PP\nLENc5jLXcdFAG0xrKtDWP4vrLknHV2+7GgBw8t13YUtIEMmYX7Bh1/4Ol34ZBw+KdHGVJRjbDzz/\nn6hoOofB7duR+frronTO8Z4/1ImuDLvBIFz/fnxdChbS05Fy8iTWfPnLAIDP3fEs5mNi8aVP5aIo\nO86tnutuuw1JdXWu4+Nll+DHn/uhXZd9/4HYf/lH9N10k6vv63/6U4xcfbUrfWvfDJ54vU8k8+rz\nb+HOVx5wHZ8+eBDzQXgCpnvsvJSUFExMLN5UJyYmkJqa6iWHsjJ68NYFC4BFI0pa1pq1GwB0yeuz\nsAAAWJeXB2BSlGb1smWAD3qXx9kHVGJTk11+aysAoGBqyi3tiuhovL0sBwDQkFuB+ZhYAEB/SjY+\naTYD+/8CANi4bj2S8xw7GpOL+gnr2DHRAGDQdUMHgM3l5cC6dUD74qILAMVWK4qV6uSYfFPWBAAT\noktyBhQANGeXYkVMl33HS6CjM8bR4NMvyuaT9pWncbJzmV2ngRR7W5QnJGDSMZRn+kewIkr8qp9Q\nzmiifafDanWcHxhwT9fejsZcex/0Lls0MLzpNim4oaUlLrNfHx/3WpfCnAIA9nyr09OBDvuCmD8+\njnyzGXj3XTTHiR+jO/XpG7NiwGRfVBtzymR13FBUZB+zgvZw6lHVUQ1gGN1DcyLddh94WSRjwpoA\ns9mMl1KzROdnbIniOh09KrpekZzscb5MCNqqzZaCnvw1bvoBEM9Hs9m+yyXAldbRbqlTs0C6jBwn\nZ84AAOL6+xevO3Qxm82u/50MptgNmu7kbOQuSwAwLV++HI4fK4mNjfZ0jY0AgKL5eRQ58h15wt5m\nzdmlAIAtq1fbd7ChMh6YQN/+1GxUJE0CK1diJMmu50iSfbdNbp53Om6S3dHiNXvN2vWo62sEMIL+\nMatdh2PHPKrg01peX+9T3rnoGFRIdnK85mlqcv1bnpAgGn/vN1YBGEW3Lcl1rihnOVY60zjG1prM\nTPG4FYwPi8WCwWT7mOhJt69tznXeycbiYtf/a7OzPd8zHHLXVqxERnmx67g9qxhxw/YfmImpi4ZB\n5ebN9h1NAdMz84DDiDKbzcCbb8oWJRzbMzP2OZXpGI9CnOPdaUABQGd6vuv/rWvX2ndWm5td55z3\nqsKScpjX5bkXLjCgAGAoZfGH13hCKrYAKBG00cqUFFGbzVV3ARAbUa1ZJaLjqOnp8IydV1FRgZaW\nFgwPD2N2dhYnTpzANue2OyGEEEJIhOPzTtTLL7+MyclJ3Hzzzfje976H22+/HTabDTfeeCPy8mQs\nUUIIIYSQCESVEVVUVOR6H+qzn/2s6/w111yDa665JjCaEUIIIYQYGDrbJIQQQgjRAI0oQgghhBAN\n0IgihBBCCNEAjShCCCGEEA3QiCKEEEII0QCNKLUoO3YnLkxBaS+b0xFfJPSNwetg8lU9g9cnZBi8\nXQynnc7tJZRms8rI1rM8A8qyQT4MjWEw+PyQI+KMKF2GiFxH+tq5HmImKcoRZLOZxOlVROiRbwBn\nPqlOgR6wntoggNgUyjRJbxNy6YWxm7RUQXW7ekjnqb88lOGxzir0UKuptN1satpRBarKV9MeAEyK\ncyuI41FalteydfoxIMmvNBcA7zfV0N/P/OgvifKmQPW9v40kZ8hpFR3o8a23QSu5v6nOJxkXivM+\nwEScEaVECO7rqgr2+Zd+yCoSBhilbRQNOuX00gXCJPo/AIuH1rYLZZsHrGxB+xplTCmhUk+vNx6p\nCL3qrkWOL3mM0Ed66KBWhu7VVbme+FNHHw0eA/SoIkvOiCKEEEII0QMaUYQQQgghGqARRQghhBCi\nARpRhBBCCCEaoBFFCCGRTOg/syMkYqERRQghhBCiARpRhBBCCCEaoBGlFm6Jq0arEzWfy3H9EwF9\nY9Q6OP1c+qqfUesTakLRLj749VHl0DeYBFAf2boa0Mu4rrKM4EvLG0YbfyqIOCNKF8+0ct6gg+Sx\n3CZxpyhKb7UqFyvnnmwpeSxXcM/mZgwoeSzX4O5N9Y3IUzoVHrp98vTtNY22y26itXosV5MvQj2W\nu5YW5yW9PZarGLveSgz17UzV2PCY2bOTWn9wG2N+9pm3tcJnySH0WK7F+LZpDA/mFkWCHsuXCIoD\nXCcjjejSNrqEElDSQ5VnaM+Lts9e7kVFyeumuen8aHO/eytAc0HYvgELG6I3aj2We1lv3OpKj+VB\n1UG1w3Kd66t6PQniXA+HeUcjihASUkK940EIIVqhEUUIIYQQogEaUYQQQgghGqARRQghkUwYfvFE\nSLhAI4oQQgghRAM0ogghhBBCNEAjihBCIpkw+EyckHCFRhQhhBBCiAZoRKmFL2eqRqsnWp/R6k3e\niBi2Dna9GPZFJwzeLoZTT2eFRF7cl2DYF6N1rxuGG4DK0IiSQxj2BRpv1FrDvghDjpjExoga1/ry\nTq+XUNgXX8sMSNgXtQkVBHgL+yLI67HOKhRRSuG87s3DtT1BAPtaZdgX2BTCIhkt7Ivjr+Y1RoqW\n/N6aJMQ3NL9Kdwvv5I8woZgghn3R656jF3qGqIH7/U09Qb6PKbDkjKiQuZH3N+6Xj/KWNHqEXtDj\nN5tSn6tILx0XYr206+hJM80tF4lhXwTtGw7hJwCobwsvQ0fNuNQEw76oE6FzOvXlqlxP/JnrPt/n\nNBcVNJacEUUIIYQQogc0ogghISX83oIIM8LwPRNCwgUaUYQQQgghGqARRQghhBCiARpRhBBCCCEa\noBFFCCGRjBG+WiMkQqERRQghhBCiARpRauEXLurR7ETNN3RzUmgEjFoHp1r0WK4PbBff0N1juVD0\nEvRYbvRdyTCcHxFnRBlmjIRIEa+lBlsnw3TGIm7O3hQ8locUo+gBFT7vgqGrr85LfcyvKz54LCeB\nR7e2NvJNPgCOSXVxOhzhRJwRpQvCiWKSOaeXbKWkkvQ2q0YdPJVp5AXBKGhZfdW2q1/9IhOayJMc\nrwumQhgiT+eDOXb8bU+jo1cMSEl+NbsO3sIahbw1/TF6gzUW/C1H65ruLyr1Vow1uJgwoHqIsgQ7\nfJkCS86ICtmvP39/Rfsob0mjR+gFPealUp9LL6vYrRDqpTqEgkw6T6ppbrpIDPsi+jEVJvNNTk+5\n/vcl7gvDvgRVB7Ui9K6u6vXEr7nu28JqgB5VZMkZUYQQQggheqBoRFmtVtx99924+eabsWPHDrS0\ntIiuP/HEE/j85z+PG2+8Ea+//nrAFCWEEEIIMRIxSgkOHTqE2dlZ7N+/H1VVVdizZw8eeughAMDo\n6CieeuopvPbaa5iamsINN9yA7du3B1xpQkjkEPJ3bwghRCOKO1EWiwVXXnklAGDr1q2orq52XUtM\nTERBQQGmpqYwNTUFkxGeSRNCCFmE6zIhAUNxJ2p8fBwpKSmu4+joaMzPzyMmxp41Pz8f119/PRYW\nFvBv//ZvgdOUEEIIIcRAKBpRKSkpmJiYcB1brVaXAXXkyBH09vbijTfeAADcfvvtqKysxObNm73K\ntFgs/ujslYGBQdmyzI7js9VnPerjTHPx4kUApaI0zS0tGFCht1NGW1sbigXyN0xPIwHA4OAgMiV5\nOjs7Pco7efKk6//q6mrE93YAAJJra7FWoj8AtLaNu8k4V1OD6dlZrJ+aQqLgfG9vL9pU9sXY2Jiq\ndE66urowIqOjt763WCyu9lNKK6S9vR2IW+s67ujsRKEXucLzK0dHkSYpb1l9PYB0r2V6062ntxcW\ni8VjHzl1aWxsBBw90tTcjLypKSQBGBoeRqPFgpzWVo9fpywsWL3qBwAXLlzAeFISEmtrsV6iR3f3\nsGxdpHKnp6Zk6zo+Pi46n93SIpoxra2t6FPZf8I6yrXTxfp6jFosWDEwgCyZtOsc7TY6OiYrx0lm\nUxPKPFxXGmttba2AZOZ6y5Pb3i6a/5vn5hALoK+vD62OfNMzM0D8Yp6z1dWYHZbvFzW0tbVh+Kz7\n+tbT04N2yRrnROo25dy5c+jtsa/3CwsLsFgsbn0rxBcdN8/PI9ZLXrk52t7ejiKV5cV2d8N552lv\nb0ePIG1f3xAA+/3L+fyluaUF45YkUdmNjY0YkinDU7kdHR2itaampsY11xoaGjCs0D41588jaUJ8\nz3LOwZ6eHte5qqoqLCxbJko3v7DYdxaLBbmCe46S7tMzM0jwqpk7Z8+exWx/P9Lq67FScq2+vh7R\n0x1ueeT6VIhzbDrTSdeN+o4pVboF0qZQQtGIqqysxOHDh3HdddehqqoKq1evdl1LS0tDQkIC4uLi\nYDKZkJqaitHRUcVCzWalptXOOxdPAo2THsvatHET8GK3V31WrVoFtMyKzq0oKcEKH/QuLl4czmaz\nGUiwD9nMTKkJBRQUFADVdbJyKisrgT8dBABs3LgRy4qX2y/MLuon1L97qhE4MSySsWH9emDTJiAx\nUXQ+NzcXuUp1erodAJCamgr0DXhP68AGIH/5cuSvXTRszGaz3Zgxm10ypUj7weM4ceS3Oe6+RYWF\nQN/i5cKCAlVyzGYzIFiYXOm6u4Hz/fJle5IpqFNebq79+vS0Vx3Ky8uB810AgLLSUlf/ZKSn29Mf\nO4Y2D+VHRyt/WLtmzRrAbAaiFtM69TjbdQ44X++mW/RzXcD8gus4ITERZrMZr0reXEpJSRHXSbKI\nlRQXo0Sh/+SQa6dVK1fa65GVJZ/W0W7LlqV6lYOaGvfrDl28jUsAKC4uASQ/ULyuY2+9JU4Xazcf\ncrKzkePIdzT+qCjLpo0bgYoKAFicK96Q6FtcXIziTZuAw1Wi83l5ecjzIMsUJTbT169fj7axFuDC\nOKKjo+06nDjhUQWf1vIY8e1GTd6iwkLRsdc8bYuzpaioCEWCtMebTwMXJ2ASzIXSkhKslcgrLyuz\njzUngvEhd6MulOi3fv161/8V5eViWUIcctevW4estWWivoyOjgLmF5CXl+c6t3XLFkBy75ibXwD2\nd7j0w5EjskWJx7a9vxPi42XTemPTpk1AaSnQ7j5PVq5cCfOG5T7LlI5N6bphTewG3la+7wTSpgC8\nG2mKRtT27dtx9OhR3HLLLbDZbNi9ezf27t2LkpISXHvttXjvvfdw0003ISoqCpWVlbjiiit0Vd5X\nTEbxLBGq9xC8lUuP5e5+SpaCx3I95Cj5kKHHcu9lyY6z4KhCoNv7uobuMnosDwmKRlRUVBTuuece\n0bkKx68lALjjjjtwxx136K9ZKBF6Cdc7PpuCHKE3VpskBp1mL9FLyGO53rGhvHl09phHbbP60S/i\nJAoey73I03tIBYQI91ju0lpvj+Uq1i5vJYa6ObXMvcXM4eGx3NuaHtAq6OyxXMu9SXp/U53P5xyB\nhc42CSGEEEI0sPSMqFDtxyrukCza1yab8kvDhnnkZER0aRsdfu/4qoeqkB2CceK7RmqU0JgthLsG\nQZgLYTPd1D6m8dLmbq9EMOxLkHVQK8OAr4wo4eNUDwe3SUvPiCKEEEII0QEaUYQQQgghGqARRQgJ\nLWGwZR/WsH0JCRg0ogghhBBCNEAjipBIhzsRhBASEGhEEd3R6v/Dd3T24RVKAlkHf2RrzRoJfRII\nQtEuPpRpuF7Tvb2Efo8CXJ4BZdmM/nsqDNcNGlFqCcPOJYQQQkjgiDgjyjBPLkKkiNdiGfYFJqkt\nvBTCvuiAIUKqRFjYF8OEqFoC6NXShg6DwrAvISHijChdEIZaMen8yEgx7IuwSySPxRj2RRG/QkXI\nydNw41UdAsFTMjVhX4T/e1JRh/41xAiJ9LAveq0x0vyqhq63RKFtzyUf9iWQ7a932BctKkjvb77k\nE+DNcWwwWHJGVMh+/Sn9ihYMBFWDwkC7FIZDh7bRZWIq9bn0suxuhc3zsR86evIErLnpItBjuWhO\nhst802GHQc241AQ9lgdLhLZy1c5DPxSMxJ2tJWdEEUIIIYToAY0oQgiJZIywQ0NIhEIjihBCCCFE\nAzSiCIl0uBNBCCEBgUYUIZGO0b9ao5EXWIze/4SEMTSiCCGEEEI0QCNKLfw15wPBCfuiuw+vUGLY\nOujsm2ypY/R2MZp6OreXyL+anGwDhmrRU5befvR0x+jzQwYaUYEiVB7LvV6kx3K3uwQ9lqtC0YeM\nATyWK1oAhvNYToKGXo1t5Js8PZaHBBpRhBBCCCEaoBElhzDsi8w5vWQrJpXqYtUYvmUJhX3RHw1h\nX9T+evOjX0TRgDzp6GP4GF+uqw5towcRHvbFNcZ0Dvui5tGNtxJD3ZwewxmpyhwmYV88relAYB+t\n6h32RYuuJo2vfUh30Rj2JbiE7AmJv8FTfZS3pNEj9IIOavjc53KPfKRPHwXH/my1e1JNc70jMeyL\nsH3DZb4FQk+GfQmqDiG7RaldT/zoE1/DaRmhS5VYckYUIYQsKcLhTkRImEIjipBIhzdRQggJCDSi\nCIl0Qv1yCwkt7H9CAgaNKEIIIYQQDdCIIoQQQgjRwJIzokK2s61QsM9qGXiLXuqaIaDlyJWlQ9m6\naO9Ln3tIK/3MW3jsj/dhj94VNEtUWYAc/r6zFaCxpvbzbt+EBnhe+PvJfQBk+iXHlzw+uvQIiMdy\nke8RbbLU94EmvwKOP+5zTm49kf1iz48+sSnNdWl6497mXEScEWXS4yVagQzdP/n2QT+TVJcoFXm9\neeBW4UXZF7GByRRY3BYFRY/lvs9ik14fKXtpP+Elj58mq2h/pRQm19/ArWaKkj2NX2kypdoYzGO5\nm7r+6qchv7cchpi+Wu+i0k/rdaqMGrckPsnztqb7KjoQ7iCEza/3gLDZtMmUjokQD9SIM6IIIYQQ\nQoIBjSg56LFcMWkY7LLqBD2Wa8mnK6rHaXiOSt3WGF8fncB7iwX+yaNSASF04BqkcozusVzUBV7z\naPQ8rofH8hBDI4qQSMdgiw4JMux/Eskw7EtwYdiXJYBRQi8o9bmKd2KWQtgXv9uaYV8WYdiX0GKU\ntUdTuQz7ooUlZ0QRsuQIh09cSOBg/xMSMGhEEUIIIYRogEYUIYQQQogGaEQRQgghhGiARhTRHZvW\nT1e1lANExjsfRq2DVrWMWp9QY/B2UXY7EGQCqI+sewE9yzOgLGkEBMNhtPGnAhpRagnDziWEEEJI\n4Ig4I8own0SGSBGvpfrzGbqBwr6YbFY/8kpPKIV9CSFG0QNQ3pHyQ1fVP0+UXEYEUEefURH2xUC9\nG/Ho1da+fqIfVAId9oXIEnFGFCFEgpGMMUIIiSBilBJYrVbs3LkTFy5cQFxcHHbt2oXS0lLX9bff\nfhu//vWvYbPZsGHDBvzoRz/SLdhjyBD92tD5vRtfQq1I3i2yedqBCUnYF2P2sZpQF77J05JJbTo/\n+kWYxlOdVclRTqJrPk1lRXgSa47rAAAgAElEQVTYF73e7ZOGfXH+42VO6BzIwyeU4s/6NZfDJOyL\nt/wBrYHOYV+0NIM0rJlP+QyE4k7UoUOHMDs7i/379+POO+/Enj17XNfGx8dx//334+GHH8azzz6L\nwsJCDA0NBVRhQoiPGPkRBAk87H8SyRg97IvFYsGVV14JANi6dSuqq6td106dOoXVq1fjvvvuw223\n3Ybs7GxkZmYGTlsdMGzYF8FAUPXcPdx3+wKJLm2jw8RUDPWj5r0Zm8djxXeAtKC17SIw7ItwcTZF\nhcl8U9kW3tYYtycJDPsSXB1CVQ/Vu1PB6xO3NdKAKD7OGx8fR0pKius4Ojoa8/PziImJwdDQED74\n4AO88MILSEpKwhe+8AVs3boVZWVlXmVaLBb/NfdAX594J8xZltlxfObMWY/6ONPU19cDKBelaW1t\nRZ8KvZ0yWltbUSKQv356GokAhoaGkCHJ09nZ6VHeqVOnXP/XnKtBwlAvACDp/HmsE+ofHQ0AaGkd\nd5NRU1ODKasV6yYnkSQ439fXh1aVfTE+PqYqnZPu7m4MSXWE9763WCyu9lNKK6SjowNI2iQ6LvQi\nV3i+YmQE6ZLyUi9eBJDotUxvuvX29sFisSCpttat/sDiGGlsagRgn1vNLS3ImZxEMoDhkRE0WCzI\nbmlBrIcyrCperr9QV4fxZcuQWFuL9RI9urtHZOuyMD8vkjE9PS1b1/GJCdH5zKYmCGd9W1sbejXM\nc7l2qq+vx4jFgtL+fmTLpF3raLfRsTFZOU4yGhtds1p6XWmstbS0AK6Ropwnp61NNP83zc4iDkB/\nfz9aHPmmp6cg7ODq6mrMjC/OX1/Xyfb2dgyeOeN2vqe3F+2SNc6J1KVBTU0NenomAQDWBSssFguy\nmpuxwkOZvui4aW4OccK8Jy2IEtxk5eZoe3s7ilSWF9Pfjy2O/zs6OtAtSNvTOwwAsNpsLqu9ta0V\n05J2aWpqwqBMGZ7K7ejsFK01586dwwbH/42NjRhSaJ/aC7Vomx0VnXPOwd6eXte506dPY16yQTG/\nsNh3FosFOYJ7jpLu0zMzSPCqmTtnq6sxOzyMZfX1WCW51tDYiPi5Lrc8lfD+I8k5Np3tL1036jun\nVekWSJtCCUUjKiUlBRMTE65jq9WKmBh7tvT0dGzatAk5OTkAgEsuuQTnz59XNKLMZrnpog/vN1YB\nDYv6SsvavHkT8IK4s6VpVq5cCbSLb1IlJSUo8UHvkpLF4Ww2m4EE+5DNyJCaUEBBQQFwZsTtPABs\n27YNeP4QAGD9hvXIKC+2X7Au6mc2m11GVN9sM3B8WCRj/fr1wLZtQFKS6HxOTg5ylOr0dDsAICUl\nFegd8J5WwPLly7F83TrXsdlsthszZrNLphRpP3gcJ39oFR0WFhYCQ5JjFXLMZjOQluaebmAAOCOv\no0eZgjrl5ubYrwsMEjkdysvKgQv2hXJFaamrf9LT0uzpLRZ4Mq+jTMrfhKxZvRowm4GoxbROPc71\n1ADnxtx0i36+B5ibcx0nJCTAbDbjkER2SnKyuE41NaLrxcXFKPbUf9L+F9zI5dpp5cqV9npkZ4vO\nu9I62m1ZaqpXObh40f26Qxdv4xKA/T3QZvEc9bqOHT0qThdnNx+ys7OR7ch3LOE9UZaNGzcCa9YA\nwOJc8YZE36KiIhRt3gwcOiE6n5ebizwPsqS7TuvXr0fPVDtwvh5R0VF2HaqqPKrg01oeK/5JYK40\nI0phh6+oqEh07LW8rsV1vbCwEIWCtCfbzgIXxkVGW0lxCTZI5JWVlaFMeE4wPuRu1IUFBaLjDRs2\nuP4vLy+3j1s5HHLXrlmL3E2rRH0ZHRMDzM0hNy/XdW7Lli1Abq5IxNz8ArC/w6Uf3n9ftii5sZ0Q\nHy+vlxc2bdwIVFQA3d1u1yrKy2HeXCCTyzvSsSldN0zJvcBb/YpyAmlTAN6NNMWVuLKyEkeOHAEA\nVFVVYfXq1a5rGzZsQF1dHQYHBzE/P4/Tp0/bFzxCCCGEkAhHcSdq+/btOHr0KG655RbYbDbs3r0b\ne/fuRUlJCa699lrceeed+NKXvgQA+MxnPiMysshSJTgey3X/cjKUREIdhERaffSC7eIbOreX6Kti\nOdkG9DKupyyjflXtIgznh6IRFRUVhXvuuUd0rqKiwvX/9ddfj+uvv15/zYxGGHYuIQCM8cItIYRE\nIHS2GShC5bHcW7ER47HcH4NWkpcey1Uh/UowNEootIfSuKDH8qWLXh8YGmEeeIIey0MCjShCIh2j\n76IayFgkhBBfoBFFCCGEEKIBGlFyCEOt6BWSQUa2MpIXtK0aw4QEIuyLQXcP9H5xUos8qe8dLwl9\nOy9M4uF/f+T4ct0WzH1+o++k+U1gwr6o2+HzEhImwM2udeypEx4eYV+8rRWq1xFtBatLpzbsixYV\nNH6A5HbvMbrHcqITiguawDuymiFpUCPGEOjQNrpEa1fSQ3pZNr3UelVlPiniKb6l5pYLZZyzQL17\nJyoiTOabWo/lXsaOmwR6LA+qDiELqqE6ofY+8fmdMgN0qRI0ogghhBBCNEAjihBCCCFEAzSiCCGE\nEEI0QCOKkEjHCO+KEEJIBEIjSi1G+FIiTLCZEJQ6uUqIhPYzah206mXU+oQag7eL4dTTPeyLUPQS\nDPsShB9UH3zwAT72sY9hx44d2FFQgJuKi7EvPV1dZkE9f/rTn+LtN/6G6ZFODNS97jHLsZoa9PT0\noK+vDzt37vRTe9+JOCPKMF/RhEwPL+VGiMdyf3D76m4peCzXYQHW5WtFv5Xw3h6KOhrMY3k4fHkU\nKZj0amwDTAOPGMhj+Uc/+lHs27cP+zo78fv2duzNyMBolDZzIyGtAFmrt3u8/tdjxzA+Po6cnJyQ\nGFGKsfNIEDDKTZsQQkhE8C9vP4krLh4FnvsmHhuaBACkTwzbLzY1uaW/qvYItrWcEp1LnJ3CjSf+\nbD/YHwfceivwd3/nkx7jJhOibDZ8sagI8dYhtB/7LQou/Vc8Gj2FXx4/Duutt+J/JybisqkpvNrY\niIduuAGZmZmYm5tD5ceuxWR/A0ZajyG/8gsYaT2OroYjuKGkBNdMTGDz9DSaurvx3e9+F/fffz++\n+93v4sCBAzh69CgeeOABxMfHIz09Hbt378b58+fx6KOPIjY2Fu3t7bjuuuvw1a9+1feGlRBxO1GE\nEEIICR3Hjh3Djh078E8FBfh2fj7u6utDstWKS02JKProlzHa9iFSbSb84SMfwW9+8xvck5uLOQB7\n3n8fe/fuxeOPP46EhASRzPmZcQzWH8a2jf+A51tbMWsy4dLJSZQtX4777rsPsbGxAOyPae+66y78\n6le/wu9//3tceumleOihhwAAnZ2dePDBB7F//3489thjutSVO1GEkNDCnVhCdGfvVV/E3qu+iJd/\n9v/hS3e+CAD4r+d2orKlCigrAy5eFKV/e+0n8OvtXxOd+/uqV/DK1r8HALz81Q3AypXA3/6mWPZH\nP/pR/OIXvwBiYoCFBQDAYxkZyEM0AGBmrBuno+ax4/hx4I47MG8yoS8mBmnx8cjIyAAAbNu2DTMC\nmXOTA4hLXY7o6BiYAPyf/n7ZsoeGhpCSkoK8vDwAwKWXXoqf//znuPrqq7F69WrExMQgJibGzUjT\nCnei5BCGfZE5p5dsxaSSF7Q9hgEIQdgXw6L3/VjDDV51W/kT9kWQxOPLojp0msewL8EcD+obNKBq\nBAqbs/t0DvuiJmSRzVuSgMd9CeA7bBEQ9iWgw1nnsC9qdY1yCIxLycHHrLHYd+mlePTRR/GZsTFk\nz89jdHYWg4ODAICzZ8+K8sYmZWFuog9Wq90ouyM/Hz0xMTCZTKJ2zMjIwPj4OHp7ewEAx48fx4oV\nK+zVCcAPtiW3ExWyF88VX4oVHKgZkPz17hmjhF5Q6nMV6U2SsWDyNTyQ2rI9q6BSYCSGfRG0dbjM\nNx1eGHYTwbAvQdUhZLcoteuJP2FfHHM9reSj6Dz7Cv7/Dz/E+C234Lb5ecQBuPuKK3D77bcjLS0N\nMTFi8yQmPgUZFVfjVPVzuLm4GH83MYG8+XmsLS7Gd77zHfz4xz92FGnCrl278I1vfAMmkwlpaWm4\n9957cVGy86YXS86IImTJYYSbCyFkSXDZZZfhsssuczu/r70db60tBwBERcfgK9YkfOTSS4EHH3St\nUVeXluLqX/7SlefUhV682fA+krIrAABpxZdgfXI+Hn7y311pbvvkJ7Hh5psBAAcOHAAAXH755bj8\n8su96nX06FE9qsvHeYQQQgghWqARRQghhBCiARpRhBiFQL0Mq8cL5kqPBMPmywNCiFrUfJyw1KER\nRUio8GSYhME7TGGgYtgg/XhAN7lh+rWiCxrm4cMSXg9oRBFCCCGEaIBGFCGEEEKIBmhEEUIIIUQX\n7rjjDjzyyCOu43GTCZ9esQK1cXGy6ffv3485H8vojInBm8nJfmipHzSi5BA+i3e+/BEKj+UwGcZj\nuVtWg74Uo/eLkJpaSG27+tUvQq/62j2Wax0ChvRYHqbv0ITUY7mXNIFuTUX5S8BjOaye8we0Bir1\nFq3zKj2W79y5E3/84x9RX18PAPhJTg5uHhnB2tlZ9yw2Gx555BFYfehrmwk4lpSEk4mJqvMEEjrb\nJCTSMajBSwgJPLfves31/y8+878RNz8LxMYAV8+L0k3FuceSe3vtJxbl/KEeV1wyh39VsBoyMzNx\n11134Yc//CH+IyEB7bGx+C9HCBYpz3Z0oK+vD/+Rn4/fdHbiZx98gBO33gqr1YovfvGLWF5uxnDz\nexhttwAwISG9GPkFlfhtZiamTSZsm5pCfoh/PC25naiQ3U4UQ4D4GM6DN0bP6NE2ekxMPcK+SMaC\n8Esuv77q8qiaxrYL5XgMVNmiHenAFKE7KtvCpy/3GPYlyDoYoB7eUFHHa665BmVlZfh+Tg7u7enx\nWKN/LCxETk4OftHVhbeTktA+NoZnnnkGTz31FB5++GFMToxhpO0EcjfegJKPfx1xKbmwAfjy4CD+\nx9gYrp2Y0LVqWuBOFCGEEBKhPP7DT+Gzd74IAPiPgw+gsqUKqKgAGhpE6V7d+En86lNfF527qvYI\nXt38abucL6wEVq8GDrapKveGG27A9FNPIW9+XjkxgLr4eJzr68OOHTsAAPPz8+jr7cbyLTdhqPFt\nzE0OIiGjFEgtVCUvWNCIMgJh+i4HCRM4vpY27H9iYEwmE6wAymdncVlBAX68bx+sVit+85vfIG95\nIUZaH0fups8jKjoW7R88htHELETZbLCGWnEHNKKI7khfiA9YOXq/9B9KIqEOQiKtPnph8Hbx+PFK\nqAigPrKi9SzPqLIMxiWXXIIv19XhqfZ2HI+NxW233YbJyUl88pOfRGJSEuKXLUfbew8hKiYeMQlp\nSE1djtWzs3goKwsbZmZQFmL9aUSpJZCD2AjP8gkhkQnXFxICLrvsMlzW16eY7r777gN+8hMAwPc/\n9jHgzjtd16rqepFWchnSSi5znYse6sD6mRm82twMAKjRV22fiTgjyjDLRYgWLq/FBlunAJVn8sOg\nVfMyt2FuOkbRA/61uX5KKLyor6RiMNtTWpbsRwMkWOj2bryRQ+kY+CX8/Wlp+EtqKoatA2h772EA\nwO7ocewcHsa2oGqiP0vu6zxCCCGEBI+bR0awr70dd0Zlofjyr6D48q/gBwsp2JaeHmrV/IZGFCGR\njoF2tAghJJKgEUUIIYQQogEaUXIIQ61A5y/AfJFjknzl5ilEQCjCvhj0jQ69w9Foqaf6L5y094so\niScVdRizHoeO35J1UMI9YUDVCBS6rTFuYV/UlK1anO4oLlv+rDFhEvbFZvP8oX5Av5RULVtd2Beb\nhrlnk97f1OYz2L2HRhQhhBBCwhOGfQkyoTJilXZIBAOBYV/8RIe20eUrHF/1kEvvtkD4OE5kZQAm\nDxNBc9NFYNgXoVRTuMw3VWPI+9eWbnVl2Jeg6hCqaqgu1o8+8fUrX0/rlJFYekYUIYQQQogO0Igi\nuqP1Wbfv0GO5YYm0+uhFKNrFh50Dw/Wazu0lkiYn26hexnWSpfc7o7oThusGjSi1hGHnEkIIISRw\nKBpRVqsVd999N26++Wbs2LEDLS0tsmm+9KUv4ZlnngmIkj5hFEM7VB7LvV6MEI/l/mSWGsP0WB4+\nKL5X6Gd+PVHjsZzdG3YYwnO/J8Lt/bEIQdGIOnToEGZnZ7F//37ceeed2LNnj1uaBx54AKOjowFR\nkBDiJ1wwCSEkICgaURaLBVdeeSUAYOvWraiurhZdP3jwIEwmkysNIYQQQshSQNGIGh8fR0pKius4\nOjoa8/PzAIC6ujr85S9/wTe/+c3AaUgIIYQQYkBilBKkpKRgYmLCdWy1WhETY8/2wgsvoKenB//8\nz/+Mjo4OxMbGorCwEJ/4xCe8yrRYLH6q7Zm+viHZssyO4zOnz3jUx5mmoaEBwCpRmvb2dvSo0Nsp\no6WlBaUC+eumppAEYGh4GBmSPJ2dnR7lnaqqcv1fc/48kiYGAQCJtbVY7zh/8uRJ2OLiAADNLRNS\nETh//jwmo6KwdmICyYLzff39aFXZF+PjY6rSOenu6cHg+fMuHZ1t7K3vLRaLq/28pxW/l9DZ2Qmk\nLsYCb29vR5EXucLzFcPDSBccA0BqXR2AaI96etfN3q4WiwVJ589jnUx6py6NTU0AlgGwj5dsR/+M\njI6i3mJBVnMzEjyUocabcV1dHcYyM0VjxalHV9eIbF2cP5CczMzMyNZ1cmJCdD6jqQnlgutq5wsg\n9ncl104NDQ0YtlhQ0teHHEE+Z1rnuB4TjFE5ndMbGlDh4brSmtTc0gIgVXWenNZWlAjSbZydRTyA\ngYEBNDvyTU1OiUSeO3cO09PTqnWS0tHRgf7Tp93O9/T0oF2yxrmQjKPz58+ju3vKcckKi8WCzOZm\nlHko0xcdnW3g5OTJk4iJXnzULDdH5eayJ2KGhrDF8X9nZye6BGl7eoYBOOaNo8jWtjbMStqlubkZ\nAzJleCq3s6sLBYLj6upqbHT839TUhEGF9rlQdxGdtinROecc7Ovrc507c+YM5rq6xOkWxPMmW3DP\nUdJ9embG49riierqasyMj2PZxYuSuyPQ2NiIxPlutzzbbDavOzW9fX1oE6zP0nWjsXtaPqOEQNoU\nSigaUZWVlTh8+DCuu+46VFVVYfXq1a5r3/nOd1z/P/jgg8jOzlY0oADAbJabLvrwQfNp4OKiISEt\na/OWzcDz4sEoTVNRXg44kjg/CS0qLESRD3qXli4OZ7PZDCQmAgAy0tLc0hYUFABVg27nbTBh25Yt\nwEtvAwDWrV2L7HWOW1XU4tCs3LYNSLBPicH5FuADsSG5bu1awGwGkpNF53OyspCjVKen2wEAKSmp\nQO/A4nmF12yW5+Vh+bp1rmOz2Ww3Zsxml0wp0n7wOE72NYsOCwoKAMf90wagqKhIdN2THLPZDAii\niLvSjYwAlga7PA/vE7nJFNQpOztbVV3KV5QBdfY2LS0tdfVPWmqqPf3p03BflhwI9PIUBmH1qlX2\nfheMFacetX21QPUFN91iXuwFZmZdx3Hx8TCbzXhTIjspKUlcpwsXRNeLioo8zxcP/S/VxUlFebm9\nHjk58mkd7ZYq2DGX7fOmJvfrDl28jUsAWFFaCtSL56jXdeyDD8TpHD9ysrKykOXI92HSewAWx9iG\n9euBjfZbsGuueEOib2FBAQq3bAFePWY/4ZCbl5uLPE+yJON73bp1GJztBGrGYDKZ7DpIXuEQ4tNa\n7mgDJ5WV2xAb4/nHis2kfi4DAARGR0F+PgoEaas6qoHacZEj0ZLiYmySyFuxYgVWCM852riyshIn\nT550K7Jg+XLR8caNG13/l5WVoUxhDqxetRIF5g2ivoyJiQFmZpEtGO+bN28GCgpEIubmF4D9HQAc\n7fLhh7JFyY3tBElfeMImGB4bN2wA1q4F+vvd0pWVlcO8tdBdgJf3MW0wITc7G7mCNpKuGzF1fcCb\nkvKkMm22gNoUgILxrpR5+/btOHr0KG655RbYbDbs3r0be/fuRUlJCa699lpdFQ0GIfOA6u3lXpvk\nNqjmAxC+LOwZPbwG6/ERjoIebpflvuCSHgt3Dvz4UsiTaqHwWO53WwfqK1BhFIFwmW+B8JhNj+VB\n1SFkHsvVrif+eCz31ROZAbpUCUUjKioqCvfcc4/oXEVFhVu6b3zjG/ppRQhZMtjCYKEkhBA56GzT\nCBjhFxQhJDLh+kJIwKARpRYjO1kzIkFoL9cORiT0TSDrEIqbaCT0SSAwersYTT3dw74I3iVcimFf\njP58zOjzQ4aIM6IMM0RC5bHcW7ER4rHcn5Xe7Zk8PZarwud3GQKihMI7ZkoLsNE8lhtntYp4wuad\nNn8It/fHIoSIM6IIIYQQQoIBjShCCCGEEA3QiCIk0gnD9wyIjrD/CQkYNKIIIYQQQjRAI0qOQH61\n4YMcm0mc3mOoDyWZWvN5E2nUl2KN8MKk2nb1p1+E48JTX+gwZj1JCOrexlLZSfG3npL8auaotzSB\nbnZl+X7MZb2/6vM4Efwsx+olfyDbX6XeovHhLY8WXSX3N7UY7d5DI4oQQggh4UmIf2QtOSMqZBsV\nPnyerepzciPsuBgVXdpGh4mp1Ocq0ks/2zeJ/vfH1YOn8xrbLhLDvgjbN1zmm0qXHapDfHiSqQWG\nfVEnIkQ7LarXE3/Cvvho8BihS5VYckYUIUuOcFiJCCEkDFlyRlTIdv4UCrYJbnSqnvka+D0Rm8kU\nHI/lznaSlqVL2ToYHkp9riKtTWIA2UT/a9fR87tO0rZUK1B7H/gdOy9AY031+yA+CQ35i0b2ZB4M\na7f+90GmcqEa5PiSR0Va0fwJxLuvOgQIV9sHmsR7+UGlej3xo088jTuP6Y17m3Ox5IwoQoixMNqL\nosRAhMNdNBzh7rRuRJwRpYt7/0DK8EG29F0RdXXz8k6EilAUHqVqaZJAvavi9SsR74uuW95AhH0J\nxjskgmse32XQ4/0M59AJ5c3M0/iVJlMrJxhomWshGHfe3oMJ6/us9F0cvcSqCRvlC1He5riPsgLw\n/lhA3wu0aZOpex/4ScQZUYQQCfw1v7Rh/xMSMGhEEUIIIYRogEYUIYQQQogGaEQZAW63E0IIIWEH\njSg5hCE1nC+tLfGwL25fsBv1rVMj6GW0sC9+tEkAIgbpp4TWdAZDtzXG18/JEeKwL4rXDRT2JVDl\nREjYF1m3DEow7AvRDSPc+AkhkQnXFxLJMOxLcGHYlyUAw74oYqiwL/62NcO+LKL203WGfQkMDPvi\nNS3DvhCiAhuC47Ecej9qDSWGrYNGvQxbnxBj8HYxnHqBVCgQHssNLsto3euG4QagMjSi1BKGnUsI\nIYRoJhy2gkJMxBlRhunyEA0+r8X688jFSB7L9cwbCI/leqGXHgb/AaD6RdEI81geNHUM3v/BQLcn\nkkZuy3B79OnEyG2qgogzogghhBBCggGNKEIIIYQQDdCIIoQQQgjRAI0oQiIdI73/QAghEQSNKEII\niWRoRBMSMGhEySEXUiMUYV8k/pZsnkIEhCLsi+qcwUXvkABawtuoblY/+kWUwpOKAfWxEzjR7mVF\neNgXvdYYLSE0vAzvQLemxzBWzuv+TGW9x0KA4h95a4OAtr/qsC/q8mhpBq3+BI02y2lEBQvFm7HA\nY7magcVfl57Rw2uwHouwkh7Sy7LppXoIx4kWpbyVJaOTv/LUZDWox3Kxw/IwmW9qPZZ7aXO3uoaL\nx3IjoIe+oaqy2mkYRFcKofLe7gs0ogghhBBCNEAjSi1h+qggNAQn7Ivuj1pDSbDDWwSaSOiTQMC+\n8A2ddRdKkxVtwFAtesry9XUHvV+PUC4w/MZq5BlRRtn9o8fyAD5m8WeiSfIuBY/lOmAIT80+PBLX\nll9H1HgsD5IqZIkQrh7Lw5zIM6IIIWFF0H/tEkKMgxF+oPkBjShCIh3+6iSEkIBAI8oIhLklTggh\nhCxFaEQRQgghhGiARpQR4OMWQkig4PpCSMCgEUUIIYQQogEaUXKIwr64n9NLtjISf0taQw8EIOyL\nUT/Q1vvtMi1fjimFsxAI9yRARRlCMR50dCbyY+x6VDGYwRf8bU/DE5iwL66QRV7leov7EuAGVVq2\n/FljAuhfSs9yvIZ98UW2r3qoDfsi3MXUOUSNzaQx7ItkZzXU7ldoRAULhS11YRgGVUsHt+g9o0fY\nFx3UUO5z5fTS0C7iBUP74qF31JdIDPsimpNRkTXfvLqTczvBsC+qMcrao6lclfPQD39UPhs8YdD9\nMUoJrFYrdu7ciQsXLiAuLg67du1CaWmp6/qTTz6Jv/71rwCAq666Cl//+tcDp20o4Rd0qrGZEByP\n5c4JFgl9Y9Q6aFXLqPUJNQZvF592P4JBQD2Wy8g2oJdxXWUZ3Sg12vhTgeJO1KFDhzA7O4v9+/fj\nzjvvxJ49e1zX2tra8NJLL+GPf/wjDhw4gHfffRe1tbUBVVgJwwQKDZkeXsqNEI/l/uxcuP0Sosdy\nVfi9W6SLEup3c7Xk1xU1QXyN070Rj173hVA/OvIKPZaHBMWdKIvFgiuvvBIAsHXrVlRXV7uuLV++\nHI899hiio6MBAPPz84iPjw+QqoQQTXDBJISQgKBoRI2PjyMlJcV1HB0djfn5ecTExCA2NhaZmZmw\n2Wz4yU9+gvXr16OsrEyxUIvF4p/WXujtGZYty+w4Pn36tEd9nGkaGhsBrBWl6ezsRJcKvZ0ympub\nsUIgf+3kJJIBDI+MIF2Sp7Oz06O8KoG+tRdq0TY7CgBIqK/HBsf5U6dOwZqU5Ch3wk1GbW0tJuLi\nsGZ8HCmC8/0DA2hR2RdjY2Oq0jnp6e1Ff02NS0dnG3vre4vF4mo/pbRCurq6gIzF47a2NhR7kSs8\nXz405MrqLC/lwgXFMr3pNjAwAIvFgsS6OqyXSe/UpbGpCU7FW1tbkenon9GxMVy0WJDZ1CTqL1+5\nePEiRnNzkXjhgpsend6BQXYAABhHSURBVB2jsnWZm5sTyZiZnZWt6+TklOh8RmMjygXXOzo60K1h\nnsu2U2MjhiwWFPf2IlcmrXNcC8eonM5p9fVY6eG60lhramoCkKw6T3ZLC0oF6TZMTyMBwODgIJoc\n+SYnJiHs4JqaGkzNz6vWSUpHZyf6ZNa3nt5etEvWOCfSR1q1tbXo6pq2X3PokNnUBE+rui86bpyZ\ngfAn9qmqU4iLWXwYIjdH2zs6UKSyvOjhYWx1/N/V1YVOQdru7hH7Pza4dgDb29thlbRLS0sL+mXK\nOHnyJKJl3ovr6upCvuD4bHU1Njn+b25uxoBC+1y8eBE9MeI555yD/f39i3LPnsWs4BgA5hYW+85i\nsSBLcM8RItdm0zMzSPCqmTvnzp3D9PQ0UuvqsFpyrampCam2Hrc8W61WRHuR2dfXh1bB+ixdN5p7\nZlTpFkibQglFIyolJQUTE4s3ZqvVipiYxWwzMzP4wQ9+gOTkZPzoRz9SVajZLDdd9MHSdhaoG/dY\n1pYtW4A/d3nVp6K8HOgVyy0oKECBD3qvWLFCLN9h5KSnpbmlLSgoAE71ycrZumUL8LejAIC1a9Yi\nd9Mq+4W4OFeabdu2AQ5Dd8TWBrw/JJKxdu1awGx2pXGSnZWFbKU6Pd0OAEhNTQV6B7ynFZCXm4u8\n9etdx2az2W7MmM0umVKk/eBxnPyuXnSYn58PTC8eFxcXi657kmM2m4GMDPd0k5PA+zXyZXuSKahT\nVlaW/bpgnsjpUF5WBjTYjf6SkhJX/yxLTbWnr6mB/KhQx6pVq+z9Hr24jDn1uDh4ATg76qZb7Mt9\nwPTiwhUfFwez2YwjEtlJSYniOjU0iK4XFhai0FP/eeh/qS5OysvL7fXIzZVP62i31NRUr3LQ3u5+\n3aGLt3EJwP7j8IJ4UfC6jp08KU6XYL9lZWZmItOR72Ty+6Is69evB7ZsAYDFueINib6FBQUoFKwX\nTvJyc5HnQZb0MdfatWsxZu0Bzo3B5NTdyysaPq3lkqcU27ZuQ0K891tQUWGh+vIGFten/Px85AvS\nVnfXADVjokeoRUVF2CaRV1pailLhOUcbV1ZW4nTVKbci8/PzRcebNm50/b9ixQqsUJgDq1atQpF5\ns6gvY2NjgekZZGdnL8rdtAkQvIsMALNzC8D+DgCOdpExoF3XJGMlQcMTow0bNgAbNgDDw27XysrK\nYK4scs8U5f2NoZycHORUVrqOpetGXEM/8IbyKhhImwLwbqQpvhNVWVmJI0fsS2hVVRVWr160QW02\nG772ta9hzZo1uOeee1yP9QghhBgEPs4lJGAo7kRt374dR48exS233AKbzYbdu3dj7969KCkpgdVq\nxfHjxzE7O4t33nkHAPCtb33LvjNCCCGEEBLBKBpRUVFRuOeee0TnKioqXP+fPXtWf60IIfph5C+K\nSOBh/xMSMOhsUw7hoqPK669G2UpJJf6WPPpwCYHHcqnXWMOgt14axKluVh36BfDSFzqMWT+cquuH\nao/l4Wks6ObvTOqxXMXg9VZioFtTUb4/c1lv/1IBmgi6+eUKscdyze2gKZ+x7j00ogghhBASnjDs\nyxJByVGg2JWu3/JCiU0a8y+Q5QDuZekSekEH/ZX6XHjZZpMP+yLRQ5hCteM/Wbmqk2orw4f+l4a2\n8btsnRC1r15lGGW3zIMeNsB9cIQy7IvuLOoQEI/landuVIrwV5abGG/l+rGeqE2ruK7aPK93RoVG\nlFqMsvgRQgghQcCvINBLhIgzogzT5SH61eW1WIOEffG3afzJriq4qiF+McM4esAg4S58DejsY35d\nURH2xWSc1Sri0W0zzQjhjzzBsC8hIeKMqLDECDcoQgghhPgEjShCCCGEEA3QiDIC3FolhAQKri+E\nBAwaUYREOgZ/XGzjPT6wGLz/yRInzMcnjShCCCGEEA3QiCKEEEII0QCNKDmEoVY8OXTUQbZiUonT\nSkOFfVGdM7jo7ddEmzw/w5SEKhSEUQlC2Beb1ao5r7/otsZIw76oGLrexnegh49SyBO/itdd+QDN\nVavn/D6JDlDYF9Hw8JJHSytodcpstMf/NKKI/piC47Ecehu4oSQS6iAk0uqjF0ZvF4Or5y+i6gXC\nY7nBZRk25qkTLfVk2JcwQc9wAHKXBdNblUM3o0+GUKJH2+gxMX11Dqkm7ItNOE78wGNmjVL9ceRq\n0LAv4iIia755W2PcnIBGVNiXAKNLHY3TTrI7lUF06hkO8y7yjCijtHmoPJZ7vWgQj+WatXDk98fA\nURNnzygT1yh6GAXF9lAYF0bzWM7uDTsM4bnfE/RYHhIiz4gihBBCCAkCNKIIISSS4a4DIQGDRhQh\nkY6RH0GQwMP+JyRg0IgihBBCCNEAjShCCCGEEA3QiCKEhBSjOc8jhASRMH/cTCOKEEIIIUQDNKLk\nEIZaMbmf00u2IlLP355CBIQg7ItRv/jR2yOvFnmqm1WnsC8eddRhzHpWMYi/HlWHffGjCC/hNzxn\n0qkNnP2nc9gX4zjN04Zfc1nn8elRnJ/leJtHPs2xAIV9EfWBV119K94lW0vYF+m4psfyMCHAHsuF\n8lV5cDaAEWOyyccbswFBGdgeDVwd2kaV13hFIT7oYbPJp3drR5uH/33DzSu187zWppNm9KH/jeqx\nXBRFQK8yDPLowlObyw7DCPJYLpo9cm2g5zqvUZZsM+kV9sWLYa16zfPDqaeis1KDzA9fiDgjytPN\nIej4egMNRrkG8VgeysXUbaGgx3JVGMJTs7/tYTCP5SR4hEP4EL+hx/KQEHFGFCFEghEMIEIIiUBo\nRBkB/ioghBBCwg4aUYQQQgghGqARRQghhBCiARpRhBBCCCEaoBFFCCGEEKIBGlGEkJDCsC+ELGHC\n/OthGlFyiDpVJ2/CsrIVkkrS2zw4twyFx3JvTtuWOuo9DevlsdzDNNbDY7mP5wOCao/l2rXS5IFd\nNweIOsmT5FczR/X28O8LytU1kMfyAJWjm+f/QHksh1rnoRrroaX+BvuanUYU0R2t7vx9LkdvAzeU\nREIdhERaffTC4O1iC655HFJkDRg9+8eIsoxlf7hj8PkhB40otQQ47Iv4qoqyDGCNG8KLtRx6hH3R\no2o+9bl8ercIHDbh/9p3aDw6k1cnUSajH97wjRr2RRiKyQDzTV88N7qacamJiGtDGXQJORUa5NYT\n2Z1Kf8K++Gikh8OQiTgjyjCNHiJFvJZqkLAv/raMP3Ht3G7YDPuiCl1iCfqtRGSFfTFO70Y+erW1\nYX84Agz7EiIizogihBBCCAkGNKIIIYQQQjRAI4oQQgghRAM0ogghhBBCNEAjihBCCCFEA4pGlNVq\nxd13342bb74ZO3bsQEtLi+j6gQMH8PnPfx433XQTDh8+HDBFCSGEEEKMRIxSgkOHDmF2dhb79+9H\nVVUV9uzZg4ceeggA0NfXh3379uG5557DzMwMbrvtNlxxxRWIi4sLuOKEEEIIIaFE0YiyWCy48sor\nAQBbt25FdXW169qZM2ewbds2xMXFIS4uDiUlJaitrcXmzZsDp7ECU+NTouOXdu21/7PtegDA9C+e\nAZBhv+Y4B0kaHDwLZF8GAHht03Ykzk4CJ7oW03nDKeP1msX/d+0F5nLsx/0xi+edWLpxrOIy1+FE\nQgoA4ESZGct/9TyAArvIP76N9L99YE80Mrwo5+f7AYfhenY6AUCSuA32vwO8XQ+MpYrLbplXUadM\nAMD5hn4Iva28suXTsEZFy+aYiU3ASzXjwBMHRW0wNDyEjlfOYMGauaibQJ+Xdu0V6+dBt8k4e/4T\n5WbkjfYAZweAdfZr7636GBbeaXSX4zjuTlsuLq/NKu4nAOjpQUemXeCp0q1u5b+07XoZ3Rbr9F7D\nCOZ27QWGh9xlA65zw387BSSssss82gyMptivjTrSX7yIyY3bRaV8UHGp6//j5Yv/vyQdUwDw4gng\ndD8w5K7HBxPJAOIX28FB/0imSETzdDRe2rUX1Ws/ITrf2j0myoeGBnGbW7q9jC1xGRMJyYv1kGkn\nvFoNNO0FakZEZbz04yfs/m4c43pqLElejpOWFpn+sOvylz1PAchazL/tepwpXlzH3nj5OGAqcl0D\nAPzfJz3726k9Ly4rvgzYVgZ0LJZ9YjwWSAGm4xLtMve9CbxyBgBcc8U74nZ86WQP8Ms/AcgHALy/\n6qMwwQpcnHBb406u2GYvJzFNJOPwk6/g4mwcRGOjvt59zXKWqWZNdJK8Cti2ynX4yk//gDih4zZJ\nGYc2XIOcE12q1gQAwOzsYtoz/aK0x8dTAMTBKljD3j7WgM4Wydr/9kVgQFiGvY3/umcfxkYG0bnm\n464rL2273r72CMfkwy8vHr95AejxPgfeeP59ZL91GsK+7B+ZBgBYLI2LY+23fwVSU0USFgRyXrr3\nKaCuTraf7H1kT3dkzZXoS8121Hm1KJ1wvDs5XbJ47qXfHQIyTwJdnW7lvPvCUYwffMO9mhVXig7P\nF6xz/f/Omo9jqKEN2LNvUd6JTlG/tc/FAhDXuz8129Uu0VYrCnqH3csNIiabQvCe//zP/8SnPvUp\nXHXVVQCAq6++GocOHUJMTAxefPFF1NXV4dvf/jYA4Dvf+Q5uuOEGXH755R7lWSwWHdV35/U/n8XR\n6YyAlkEIIYSQ0HPD6Bls/cp1AS/HbDbLnlfciUpJScHExITr2Gq1IiYmRvbaxMQEUiXWsi/K6MGW\nDRuRfv8fkT85hqjlechKcuyWDA0BViuQlYXOKROWjQ4g5b0jwN//PZBg/9WFmVmguxsoLcHk2BSG\nhidRuCwWOHcOuPxj6hQYHgHmZoGcHKClFcjPB+JiAasNaG4GVqwApqft+phMQFoakJwENDWjqW0I\nRdmJiFq9Gi0X2lGWEQtTQT4mx6cxODCOotJscVkX6oDYWKC8THS6ZcKE/ARgYHwWKSP9SC2x/zJ1\n6VBSArS2AmVliq58J+dNGJoDChJsaJwwITsemGxuQ/7COBbq6tG09QpExUQjPRawjo5iNLsApQOt\niC5bYZfd3gFkZQGJCeju7sHy5XlYsJpwYghYkwqk97ajd1kO4hPikRZns/dBdTVQUQGkp8nqtDC3\ngJb6LpQnLthDmpStwJzVhI7mHqwozgRiY4ATFiAlBcjLBTIygLl54OhR4Ior0DYXi5x4ICHaZo9+\ncfKkvU1yFtvX2tqGplErylOAwdxioKsLptgY2OrqkPXxjyyOGQdzVhOaOoYRN9iPFVsqFi8I6u9i\negbo7QVKitExtoCM/k4klRWLx0iUo2NaWnEuKgNTiMaq5clImxpF46gVJQXpiFmYQ2NzH0paLyBm\nVYV9zM3NASMj9nYpKV4ss7vH3g7xi4/amyZMiBvtQWF+7mLb2kyoHZhFas0ZzBWVoKwsx65KXz+O\nz6aibHYIoxk5KEuLdqno4sMTQHExMDUFlK3wOKbmrSZ8OAQkRwPrlgExzU14Zy4Nl5SnIylWIHR2\nDujqAkpLFs81NmFkWRZmZheQW+D4sSRot46ZKKTHAskxHn4btrYBeXmudhgdmsDU1CzyCjIwMGNC\nxxSwKn4WiX3dQFYWWs41I3/rGsTFRaOtpR/Z0yNItM3bZa1d47GOAIDmFqCgwDH/rfZjyZxrrO9B\naV4KokeGgaJC13nnXPGGc27Ozy1g+UA74svs7TTX2Y32uAyUTfcDExPAGsGOw+QUMDQEm9WKpnEb\nSlcVYmA+GjYAs1agOMnebo3jJpQkATFRjnasvWAfWwCQkYERUxxmsnKRm+CDB29HG4wXl2Fs3oT8\nREnewSHAZsNUehb6x+dQPNJlH8Mdnfa1MjZWNEdlae8AhoeBjRvcLjnrtHDxInpiUlBSsbgrjdk5\noLMTWFEqyjM2Z8LEPLA80Wbvk+hotIwuIH+oE3Gpyfa2nZ7BcEMrZjNzkJufbu/nqSlg3VqPak6N\nT2NAsKbPWk14t2kcV+dHwZaYhOYJoDzZBtOUvb9QWCArp3tgEkmzk1iW72iXllZMZmRjaHQasdlZ\niI8G0mJtmFkwoXsaKI2bAzo67PVsbrGvGVVVwLZtQHc3WhKykB8zh8meAdRgGT6WOIHu1BykzEwi\ntUTQXm3tQHQ0ZuMS0JWYidJkD+NgfsF+ryksBD78EEhPR31MOqwxsVgdNbE4Hzo6gYEBYPMmNxFt\nk/Y+qEgB2iaB0rFuRKemAF1diI6Jhu22qwJqUwDeN38Ud6JeffVVHD58GHv27EFVVRV+9atf4bH/\n1979hTTVx2EAf87OyszpxS4jJnMVFBEh0tWyoD92kUWSS4TdrCL7S4Y1Z0laazTqzm6St7qwbmJF\nXUV1E0PNLqQFSn8IxMhFZBa0pc3tfN+Llxa9vr5th+XZu/f53HnOF/3tPOjv4Wx4/vgDwF+fifJ4\nPAiFQkgkEqivr8fdu3dRVFQ06/cbHByckxf8u38GZY+55B9mkp+YS/5hJvnJ6E7xyztRmzZtQl9f\nHxoaGiAiCAQCuHbtGmw2GzZs2AC3243GxkaICJqbm/+1QBEREREVil+WKJPJhDNnzvx0zOH48XaF\ny+WCy+XK/cqIiIiI8hj/2SYRERGRDixRRERERDqwRBERERHpwBJFREREpANLFBEREZEOLFFERERE\nOrBEEREREenAEkVERESkwy8f+5Jrv/sBxERERES5NNtjX+a8RBEREREVAr6dR0RERKQDSxQRERGR\nDixRRERERDqwRBERERHpwBJFREREpIPZ6AXkkqZp6OjowMuXLzF//nz4/X6Ul5cbvayC9ezZM1y8\neBE9PT0YHR1Fa2srFEXB0qVLcfr0aZhMJly6dAmPHj2C2WxGW1sbVq1aldUsZW56ehptbW0YGxtD\nIpHA/v37sWTJEuZioFQqhVOnTmFkZASKoqCzsxNFRUXMJE98/PgRdXV1uHr1KsxmM3Mx2I4dO2Cx\nWAAAixcvxq5du3Du3Dmoqgqn04lDhw7Nus9HIpGMZ3NKCsj9+/fF6/WKiMjTp0+lqanJ4BUVru7u\nbtm6davU19eLiMi+fftkYGBARETa29vlwYMHMjQ0JG63WzRNk7GxMamrq8t6ljIXCoXE7/eLiMin\nT59k3bp1zMVgDx8+lNbWVhERGRgYkKamJmaSJxKJhBw4cEA2b94sr1+/Zi4Gm5qaku3bt/90bNu2\nbTI6OiqapsmePXtkeHh41n0+m9lcKqg7UYODg1i7di0AYPXq1RgaGjJ4RYXLZrOhq6sLJ06cAAAM\nDw9jzZo1AIDq6mr09fXBbrfD6XRCURQsWrQIqVQKExMTWc1arVbDXuN/zZYtW1BTUwMAEBGoqspc\nDLZx40asX78eABCNRlFWVob+/n5mkgeCwSAaGhrQ3d0NgH/DjPbixQtMTk7C4/EgmUzi8OHDSCQS\nsNlsAACn04n+/n58+PBhxj4fi8Uyns21gvpMVCwWS98KBABVVZFMJg1cUeGqqamB2fyjg4sIFEUB\nAJSUlODLly8z8vh+PJtZylxJSQksFgtisRiOHDmCo0ePMpc8YDab4fV6cfbsWdTW1jKTPHD79m1Y\nrdb0Bgvwb5jRFixYgN27d+PKlSvo7OyEz+dDcXFx+vxs11lV1Vmv/Vx0goK6E2WxWBCPx9Nfa5r2\n00ZPv4/J9KOPx+NxlJWVzcgjHo+jtLQ0q1nKzrt373Dw4EE0NjaitrYWFy5cSJ9jLsYJBoNoaWmB\ny+XCt2/f0seZiTFu3boFRVHw+PFjPH/+HF6vFxMTE+nzzGXu2e12lJeXQ1EU2O12lJaW4vPnz+nz\n36/z1NTUjH3+n679bLO57gQFdSeqsrIS4XAYABCJRLBs2TKDV/T/sWLFCjx58gQAEA6HUVVVhcrK\nSvT29kLTNESjUWiaBqvVmtUsZW58fBwejwfHjx/Hzp07ATAXo925cweXL18GABQXF0NRFKxcuZKZ\nGOzGjRu4fv06enp6sHz5cgSDQVRXVzMXA4VCIZw/fx4A8P79e0xOTmLhwoV48+YNRAS9vb3p6/z3\nfd5isWDevHkZzeZaQT077/sn8V+9egURQSAQgMPhMHpZBevt27c4duwYbt68iZGREbS3t2N6ehoV\nFRXw+/1QVRVdXV0Ih8PQNA0+nw9VVVVZzVLm/H4/7t27h4qKivSxkydPwu/3MxeDfP36FT6fD+Pj\n40gmk9i7dy8cDgd/V/KI2+1GR0cHTCYTczFQIpGAz+dDNBqFoihoaWmByWRCIBBAKpWC0+lEc3Pz\nrPt8JBLJeDaXCqpEEREREc2Vgno7j4iIiGiusEQRERER6cASRURERKQDSxQRERGRDixRRERERDqw\nRBERERHpwBJFREREpANLFBEREZEOfwLRBqkcH/awEgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)\n",
    "print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)\n",
    "print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. SVM - Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)\n",
    "print('SVM accuracy: {}%'.format(svm_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 5. ANN - Artificial Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_8 (Dense)              (None, 55)                330       \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 2)                 112       \n",
      "=================================================================\n",
      "Total params: 442\n",
      "Trainable params: 442\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 200000 samples, validate on 50000 samples\n",
      "Epoch 1/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0035 - acc: 0.9983 - val_loss: 0.0020 - val_acc: 0.9982\n",
      "Epoch 2/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0019 - acc: 0.9983 - val_loss: 0.0019 - val_acc: 0.9982\n",
      "Epoch 3/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 4/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 5/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 6/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 7/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 8/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 9/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 10/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 11/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 12/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 13/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 14/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 15/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 16/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 17/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 18/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 19/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 20/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 21/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 22/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 23/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 24/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 25/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 26/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 27/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 28/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 29/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 30/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 31/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 32/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 33/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 34/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 35/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 36/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 37/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 38/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 39/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 40/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 41/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 42/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 43/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 44/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 45/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 46/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 47/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 48/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 49/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 50/99\n",
      "200000/200000 [==============================] - 2s 8us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 51/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 52/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 53/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 54/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 55/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 56/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 57/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 58/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 59/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 60/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 61/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 62/99\n",
      "200000/200000 [==============================] - 2s 9us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 63/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 64/99\n",
      "200000/200000 [==============================] - 2s 10us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 65/99\n",
      "200000/200000 [==============================] - 2s 11us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 66/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 67/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 68/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 69/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 70/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 71/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 72/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 73/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 74/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 75/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 76/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 77/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 78/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 79/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 80/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 81/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 82/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 83/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 84/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 85/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 86/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 87/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 88/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 89/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 90/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 91/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 92/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 93/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 94/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 95/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 96/99\n",
      "200000/200000 [==============================] - 3s 13us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 97/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 98/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 99/99\n",
      "200000/200000 [==============================] - 2s 12us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "50000/50000 [==============================] - 1s 18us/step\n"
     ]
    }
   ],
   "source": [
    "ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, df_train, X_test, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ANN accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6. MLP - Multi-layered Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_10 (Dense)             (None, 55)                330       \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 55)                3080      \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 2)                 112       \n",
      "=================================================================\n",
      "Total params: 9,682\n",
      "Trainable params: 9,682\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 200000 samples, validate on 50000 samples\n",
      "Epoch 1/99\n",
      "200000/200000 [==============================] - 4s 21us/step - loss: 0.0034 - acc: 0.9983 - val_loss: 0.0019 - val_acc: 0.9982\n",
      "Epoch 2/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 3/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 4/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 5/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 6/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0018 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 7/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 8/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 9/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 10/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 11/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 12/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 13/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 14/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 15/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 16/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 17/99\n",
      "200000/200000 [==============================] - 4s 20us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 18/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 19/99\n",
      "200000/200000 [==============================] - 4s 19us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 20/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 21/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 22/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 23/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 24/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 25/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 26/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 27/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 28/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 29/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 30/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 31/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 32/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 33/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 34/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 35/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 36/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 37/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 38/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 39/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 40/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 41/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 42/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 43/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 44/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 45/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 46/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 47/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 48/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 49/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 50/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 51/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 52/99\n",
      "200000/200000 [==============================] - 4s 18us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 53/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 54/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 55/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 56/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 57/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 58/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 59/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 60/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 61/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 62/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 63/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 64/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 65/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 66/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 67/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 68/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 69/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 70/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 71/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 72/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 73/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 74/99\n",
      "200000/200000 [==============================] - 3s 17us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 75/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 76/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 77/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 78/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 79/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 80/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 81/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 82/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 83/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 84/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 85/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 86/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 87/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 88/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 89/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 90/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 91/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 92/99\n",
      "200000/200000 [==============================] - 3s 15us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 93/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 94/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 95/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 96/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 97/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 98/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "Epoch 99/99\n",
      "200000/200000 [==============================] - 3s 14us/step - loss: 0.0017 - acc: 0.9983 - val_loss: 0.0018 - val_acc: 0.9982\n",
      "50000/50000 [==============================] - 1s 16us/step\n"
     ]
    }
   ],
   "source": [
    "mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, df_train, X_test, df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP accuracy: 99.824%\n"
     ]
    }
   ],
   "source": [
    "print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 7. RNN - Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " - 4s - loss: 0.0018\n",
      "Epoch 2/10\n",
      " - 5s - loss: 0.0017\n",
      "Epoch 3/10\n",
      " - 4s - loss: 0.0017\n",
      "Epoch 4/10\n",
      " - 5s - loss: 0.0017\n",
      "Epoch 5/10\n",
      " - 4s - loss: 0.0017\n",
      "Epoch 6/10\n",
      " - 6s - loss: 0.0017\n",
      "Epoch 7/10\n",
      " - 5s - loss: 0.0017\n",
      "Epoch 8/10\n",
      " - 5s - loss: 0.0017\n",
      "Epoch 9/10\n",
      " - 5s - loss: 0.0017\n",
      "Epoch 10/10\n",
      " - 4s - loss: 0.0017\n",
      "Train Score: 0.04 RMSE\n",
      "Test Score: 0.04 RMSE\n",
      "RNN accuracy: 14.674133333333334%\n"
     ]
    }
   ],
   "source": [
    "rnn_acc = rnn_pre(df_train)\n",
    "print('RNN accuracy: {}%'.format(rnn_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conclusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest accuracy: 99.842%\n",
      "Gradient Boosting accuracy: 99.57000000000001%\n",
      "Logistic Regression accuracy: 99.824%\n",
      "SVM accuracy: 99.824%\n",
      "ANN accuracy: 99.824%\n",
      "MLP accuracy: 99.824%\n",
      "RNN accuracy: 14.674133333333334%\n"
     ]
    }
   ],
   "source": [
    "print('Random Forest accuracy: {}%'.format(RF_acc * 100))\n",
    "print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))\n",
    "print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))\n",
    "print('SVM accuracy: {}%'.format(svm_acc * 100))\n",
    "print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))\n",
    "print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))\n",
    "print('RNN accuracy: {}%'.format(rnn_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest is the best one for our project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction for test.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ip', 'app', 'device', 'os', 'channel']"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Columns for our current analsis\n",
    "train_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 18790469 entries, 0 to 18790468\n",
      "Data columns (total 5 columns):\n",
      "ip         int64\n",
      "app        int64\n",
      "device     int64\n",
      "os         int64\n",
      "channel    int64\n",
      "dtypes: int64(5)\n",
      "memory usage: 716.8 MB\n"
     ]
    }
   ],
   "source": [
    "# Read the test data\n",
    "df = pd.read_csv('data/test.csv')[train_cols]\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 18790469 entries, 0 to 18790468\n",
      "Data columns (total 1 columns):\n",
      "is_attributed    int64\n",
      "dtypes: int64(1)\n",
      "memory usage: 143.4 MB\n"
     ]
    }
   ],
   "source": [
    "# Read the output of the test data\n",
    "sample_out = pd.read_csv('data/sample_submission.csv')[['is_attributed']]\n",
    "sample_out.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_predict = rf_model.predict(df)\n",
    "# Compare the prediction with the known values\n",
    "df_acc = sklearn.metrics.accuracy_score(np.array(df_predict)[:], \n",
    "                                         np.array(sample_out)[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "By using the best algorithm, the accuracy of the prediction: 99.85809295127225%\n"
     ]
    }
   ],
   "source": [
    "print('By using the best algorithm, the accuracy of the prediction: {}%'.format(df_acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code in the document is licensed under the MIT License: https://opensource.org/licenses/MIT\n",
    "\n",
    "All writing in the document is licensed bt The Creative Commons Attribution 3.0 https://creativecommons.org/licenses/by/3.0/us/."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:anaconda]",
   "language": "python",
   "name": "conda-env-anaconda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}