Runway_Classification_XGBClassifier_GridSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:85% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:85% !important; }</style>\"))\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\datasciencelab\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from cmplotter import plot_confusion_matrix\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "#Set some numpy print options for displaying numpy arrays to fit maximum width of cell\n",
    "np.set_printoptions(precision=8, edgeitems=30, linewidth=1000,formatter=dict(float=lambda x: \"%.8g\" % x))\n",
    "\n",
    "import pandas as pd\n",
    "from xgboost import XGBClassifier,plot_tree\n",
    "from sklearn import datasets\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV\n",
    "from sklearn.cross_validation import StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score,classification_report,confusion_matrix\n",
    "from sklearn.preprocessing import LabelEncoder,label_binarize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Landing_Time</th>\n",
       "      <th>hourofday</th>\n",
       "      <th>aircraft_registration</th>\n",
       "      <th>wind_direction_degrees</th>\n",
       "      <th>Wind_Speed_kt</th>\n",
       "      <th>Wind_And_Gust_kt</th>\n",
       "      <th>Landing_RVR_or_Vis_m</th>\n",
       "      <th>temperature_c</th>\n",
       "      <th>dew_point_c</th>\n",
       "      <th>relative_humidity</th>\n",
       "      <th>pressure_altimiter_in</th>\n",
       "      <th>RunwayTrueTrack</th>\n",
       "      <th>Landing_Visibilty_Class_RVR</th>\n",
       "      <th>present_weather_codes</th>\n",
       "      <th>Landing_RWY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-01 04:34:38</td>\n",
       "      <td>4</td>\n",
       "      <td>GXLEJ</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>VMC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RW09R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-01 04:38:22</td>\n",
       "      <td>4</td>\n",
       "      <td>GSTBH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>VMC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RW09R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-01 04:40:38</td>\n",
       "      <td>4</td>\n",
       "      <td>GCIVH</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>VMC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RW09R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-01-01 04:42:36</td>\n",
       "      <td>4</td>\n",
       "      <td>GVYUM</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>VMC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RW09R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-01-01 04:44:43</td>\n",
       "      <td>4</td>\n",
       "      <td>GBYGB</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>VMC</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RW09R</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Landing_Time  hourofday aircraft_registration  \\\n",
       "0 2016-01-01 04:34:38          4                 GXLEJ   \n",
       "1 2016-01-01 04:38:22          4                 GSTBH   \n",
       "2 2016-01-01 04:40:38          4                 GCIVH   \n",
       "3 2016-01-01 04:42:36          4                 GVYUM   \n",
       "4 2016-01-01 04:44:43          4                 GBYGB   \n",
       "\n",
       "   wind_direction_degrees  Wind_Speed_kt  Wind_And_Gust_kt  \\\n",
       "0                     NaN            1.0               1.0   \n",
       "1                     NaN            1.0               1.0   \n",
       "2                   110.0            1.0               1.0   \n",
       "3                   110.0            1.0               1.0   \n",
       "4                   110.0            1.0               1.0   \n",
       "\n",
       "   Landing_RVR_or_Vis_m  temperature_c  dew_point_c  relative_humidity  \\\n",
       "0                6000.0            2.0          1.0              93.08   \n",
       "1                6000.0            2.0          1.0              93.08   \n",
       "2                6000.0            2.0          1.0              93.08   \n",
       "3                6000.0            2.0          1.0              93.08   \n",
       "4                6000.0            2.0          1.0              93.08   \n",
       "\n",
       "   pressure_altimiter_in  RunwayTrueTrack Landing_Visibilty_Class_RVR  \\\n",
       "0                  30.21        89.713523                         VMC   \n",
       "1                  30.21        89.713523                         VMC   \n",
       "2                  30.21        89.713523                         VMC   \n",
       "3                  30.21        89.713523                         VMC   \n",
       "4                  30.21        89.713523                         VMC   \n",
       "\n",
       "  present_weather_codes Landing_RWY  \n",
       "0                   NaN       RW09R  \n",
       "1                   NaN       RW09R  \n",
       "2                   NaN       RW09R  \n",
       "3                   NaN       RW09R  \n",
       "4                   NaN       RW09R  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load Dataset\n",
    "data = pd.read_csv('data/2016_EGLL_Testdata_cleaned.csv', parse_dates = ['Landing_Time'], infer_datetime_format = True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 220940 entries, 0 to 220939\n",
      "Data columns (total 15 columns):\n",
      "Landing_Time                   220940 non-null datetime64[ns]\n",
      "hourofday                      220940 non-null int64\n",
      "aircraft_registration          220799 non-null object\n",
      "wind_direction_degrees         203226 non-null float64\n",
      "Wind_Speed_kt                  220940 non-null float64\n",
      "Wind_And_Gust_kt               220940 non-null float64\n",
      "Landing_RVR_or_Vis_m           220940 non-null float64\n",
      "temperature_c                  220940 non-null float64\n",
      "dew_point_c                    220940 non-null float64\n",
      "relative_humidity              220940 non-null float64\n",
      "pressure_altimiter_in          220940 non-null float64\n",
      "RunwayTrueTrack                220940 non-null float64\n",
      "Landing_Visibilty_Class_RVR    220940 non-null object\n",
      "present_weather_codes          33015 non-null object\n",
      "Landing_RWY                    220940 non-null object\n",
      "dtypes: datetime64[ns](1), float64(9), int64(1), object(4)\n",
      "memory usage: 25.3+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-Hot Encoding Categorical Variables pertaining to Visibility and Weather conditions which are interesting features\n",
    "data = pd.get_dummies(data, columns=['Landing_Visibilty_Class_RVR', 'present_weather_codes'])\n",
    "# Dropping Categorical variables which are not going to be used by the GBM model\n",
    "data.drop(columns=['Landing_Time', 'aircraft_registration'],inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 220940 entries, 0 to 220939\n",
      "Data columns (total 65 columns):\n",
      "hourofday                                   220940 non-null int64\n",
      "wind_direction_degrees                      203226 non-null float64\n",
      "Wind_Speed_kt                               220940 non-null float64\n",
      "Wind_And_Gust_kt                            220940 non-null float64\n",
      "Landing_RVR_or_Vis_m                        220940 non-null float64\n",
      "temperature_c                               220940 non-null float64\n",
      "dew_point_c                                 220940 non-null float64\n",
      "relative_humidity                           220940 non-null float64\n",
      "pressure_altimiter_in                       220940 non-null float64\n",
      "RunwayTrueTrack                             220940 non-null float64\n",
      "Landing_RWY                                 220940 non-null object\n",
      "Landing_Visibilty_Class_RVR_CAT1            220940 non-null uint8\n",
      "Landing_Visibilty_Class_RVR_CAT2            220940 non-null uint8\n",
      "Landing_Visibilty_Class_RVR_CAT3            220940 non-null uint8\n",
      "Landing_Visibilty_Class_RVR_Marginal VMC    220940 non-null uint8\n",
      "Landing_Visibilty_Class_RVR_VMC             220940 non-null uint8\n",
      "present_weather_codes_+RA                   220940 non-null uint8\n",
      "present_weather_codes_+RA VCTS              220940 non-null uint8\n",
      "present_weather_codes_+SHRA                 220940 non-null uint8\n",
      "present_weather_codes_-DZ                   220940 non-null uint8\n",
      "present_weather_codes_-DZ BR                220940 non-null uint8\n",
      "present_weather_codes_-DZ FG                220940 non-null uint8\n",
      "present_weather_codes_-RA                   220940 non-null uint8\n",
      "present_weather_codes_-RA VCTS              220940 non-null uint8\n",
      "present_weather_codes_-RADZ                 220940 non-null uint8\n",
      "present_weather_codes_-RADZ BR              220940 non-null uint8\n",
      "present_weather_codes_-RADZ FG              220940 non-null uint8\n",
      "present_weather_codes_-RADZ HZ              220940 non-null uint8\n",
      "present_weather_codes_-SHRA                 220940 non-null uint8\n",
      "present_weather_codes_-SHRA TS              220940 non-null uint8\n",
      "present_weather_codes_-SHRA VCTS            220940 non-null uint8\n",
      "present_weather_codes_-SHRASN               220940 non-null uint8\n",
      "present_weather_codes_-SN                   220940 non-null uint8\n",
      "present_weather_codes_-TSRA                 220940 non-null uint8\n",
      "present_weather_codes_//////                220940 non-null uint8\n",
      "present_weather_codes_BCFG                  220940 non-null uint8\n",
      "present_weather_codes_BR                    220940 non-null uint8\n",
      "present_weather_codes_BR BCFG               220940 non-null uint8\n",
      "present_weather_codes_BR MIFG               220940 non-null uint8\n",
      "present_weather_codes_BR PRFG               220940 non-null uint8\n",
      "present_weather_codes_BR VCFG               220940 non-null uint8\n",
      "present_weather_codes_BR VCTS               220940 non-null uint8\n",
      "present_weather_codes_DZ                    220940 non-null uint8\n",
      "present_weather_codes_FG                    220940 non-null uint8\n",
      "present_weather_codes_FZFG                  220940 non-null uint8\n",
      "present_weather_codes_HZ                    220940 non-null uint8\n",
      "present_weather_codes_MIFG                  220940 non-null uint8\n",
      "present_weather_codes_MIFG BR               220940 non-null uint8\n",
      "present_weather_codes_PRFG                  220940 non-null uint8\n",
      "present_weather_codes_PRFG VCFG             220940 non-null uint8\n",
      "present_weather_codes_RA                    220940 non-null uint8\n",
      "present_weather_codes_RA HZ                 220940 non-null uint8\n",
      "present_weather_codes_RADZ                  220940 non-null uint8\n",
      "present_weather_codes_RADZ FG               220940 non-null uint8\n",
      "present_weather_codes_SG                    220940 non-null uint8\n",
      "present_weather_codes_SHRA                  220940 non-null uint8\n",
      "present_weather_codes_SHRA HZ               220940 non-null uint8\n",
      "present_weather_codes_SHRA VCTS             220940 non-null uint8\n",
      "present_weather_codes_TS                    220940 non-null uint8\n",
      "present_weather_codes_TSRA                  220940 non-null uint8\n",
      "present_weather_codes_VCFG                  220940 non-null uint8\n",
      "present_weather_codes_VCSH                  220940 non-null uint8\n",
      "present_weather_codes_VCTS                  220940 non-null uint8\n",
      "present_weather_codes_VCTS -SHRA            220940 non-null uint8\n",
      "present_weather_codes_VCTS VCSH             220940 non-null uint8\n",
      "dtypes: float64(9), int64(1), object(1), uint8(54)\n",
      "memory usage: 29.9+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_features = data.copy()\n",
    "X_data = input_features.drop('Landing_RWY', axis=1)\n",
    "y = data.Landing_RWY.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hourofday</th>\n",
       "      <th>wind_direction_degrees</th>\n",
       "      <th>Wind_Speed_kt</th>\n",
       "      <th>Wind_And_Gust_kt</th>\n",
       "      <th>Landing_RVR_or_Vis_m</th>\n",
       "      <th>temperature_c</th>\n",
       "      <th>dew_point_c</th>\n",
       "      <th>relative_humidity</th>\n",
       "      <th>pressure_altimiter_in</th>\n",
       "      <th>RunwayTrueTrack</th>\n",
       "      <th>...</th>\n",
       "      <th>present_weather_codes_SHRA</th>\n",
       "      <th>present_weather_codes_SHRA HZ</th>\n",
       "      <th>present_weather_codes_SHRA VCTS</th>\n",
       "      <th>present_weather_codes_TS</th>\n",
       "      <th>present_weather_codes_TSRA</th>\n",
       "      <th>present_weather_codes_VCFG</th>\n",
       "      <th>present_weather_codes_VCSH</th>\n",
       "      <th>present_weather_codes_VCTS</th>\n",
       "      <th>present_weather_codes_VCTS -SHRA</th>\n",
       "      <th>present_weather_codes_VCTS VCSH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>93.08</td>\n",
       "      <td>30.21</td>\n",
       "      <td>89.713523</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   hourofday  wind_direction_degrees  Wind_Speed_kt  Wind_And_Gust_kt  \\\n",
       "0          4                     NaN            1.0               1.0   \n",
       "1          4                     NaN            1.0               1.0   \n",
       "2          4                   110.0            1.0               1.0   \n",
       "3          4                   110.0            1.0               1.0   \n",
       "4          4                   110.0            1.0               1.0   \n",
       "\n",
       "   Landing_RVR_or_Vis_m  temperature_c  dew_point_c  relative_humidity  \\\n",
       "0                6000.0            2.0          1.0              93.08   \n",
       "1                6000.0            2.0          1.0              93.08   \n",
       "2                6000.0            2.0          1.0              93.08   \n",
       "3                6000.0            2.0          1.0              93.08   \n",
       "4                6000.0            2.0          1.0              93.08   \n",
       "\n",
       "   pressure_altimiter_in  RunwayTrueTrack               ...                 \\\n",
       "0                  30.21        89.713523               ...                  \n",
       "1                  30.21        89.713523               ...                  \n",
       "2                  30.21        89.713523               ...                  \n",
       "3                  30.21        89.713523               ...                  \n",
       "4                  30.21        89.713523               ...                  \n",
       "\n",
       "   present_weather_codes_SHRA  present_weather_codes_SHRA HZ  \\\n",
       "0                           0                              0   \n",
       "1                           0                              0   \n",
       "2                           0                              0   \n",
       "3                           0                              0   \n",
       "4                           0                              0   \n",
       "\n",
       "   present_weather_codes_SHRA VCTS  present_weather_codes_TS  \\\n",
       "0                                0                         0   \n",
       "1                                0                         0   \n",
       "2                                0                         0   \n",
       "3                                0                         0   \n",
       "4                                0                         0   \n",
       "\n",
       "   present_weather_codes_TSRA  present_weather_codes_VCFG  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   present_weather_codes_VCSH  present_weather_codes_VCTS  \\\n",
       "0                           0                           0   \n",
       "1                           0                           0   \n",
       "2                           0                           0   \n",
       "3                           0                           0   \n",
       "4                           0                           0   \n",
       "\n",
       "   present_weather_codes_VCTS -SHRA  present_weather_codes_VCTS VCSH  \n",
       "0                                 0                                0  \n",
       "1                                 0                                0  \n",
       "2                                 0                                0  \n",
       "3                                 0                                0  \n",
       "4                                 0                                0  \n",
       "\n",
       "[5 rows x 64 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of Classes in the output: 4 and the classes are: ['RW09L' 'RW09R' 'RW27L' 'RW27R']\n"
     ]
    }
   ],
   "source": [
    "# Number of unique values Ytrn\n",
    "n_classes = len(np.unique(y))\n",
    "print('No. of Classes in the output:',n_classes,'and the classes are:',np.unique(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoded-Y: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 1 0 1 0 1 0 1 0 1 0 1 0 ... 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "## One Hot Encode categorical variable output converting the class labels to integers\n",
    "encoder = LabelEncoder()\n",
    "encoded_y = encoder.fit_transform(y)\n",
    "print('Encoded-Y:',encoded_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Convert class labels to binary representation for each of the four classes such as [0 1 0 0], [1 0 0 0], [0 0 1 0] and [0 0 0 1]\n",
    "# y_bin_enc = label_binarize(y, classes=['RW09L','RW09R','RW27L','RW27R'])\n",
    "# print('Output-Y:',y_bin_enc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split the dataset into train and Test\n",
    "seed = 42\n",
    "test_size = 0.3\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=test_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset contains 154658 rows and 64 columns\n",
      "Test dataset contains 66282 rows and 64 columns\n"
     ]
    }
   ],
   "source": [
    "# Sanity Check\n",
    "print(\"Train dataset contains {0} rows and {1} columns\".format(X_train.shape[0], X_train.shape[1]))\n",
    "print(\"Test dataset contains {0} rows and {1} columns\".format(X_test.shape[0], X_test.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train possible labels: \n",
      "['RW09L' 'RW09R' 'RW27L' 'RW27R']\n",
      "\n",
      "Test possible labels: \n",
      "['RW09L' 'RW09R' 'RW27L' 'RW27R']\n"
     ]
    }
   ],
   "source": [
    "print(\"Train possible labels: \")\n",
    "print(np.unique(y_train))\n",
    "\n",
    "print(\"\\nTest possible labels: \")\n",
    "print(np.unique(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_predictions_counter(y_test,y_pred):\n",
    "    '''Takes in the predictions of a model output and returns the no. of correct predictions.\n",
    "    Arguments: y_test, y_pred'''\n",
    "    correct = 0\n",
    "    for i in range(len(y_pred)):\n",
    "        if (y_test[i] == y_pred[i]):\n",
    "            correct += 1\n",
    "    return correct"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Grid Search for HyperParameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for the XGB Classifier used as estimator in the GridSearch\n",
    "params_xgbclassifier = {\n",
    "    'objective': 'multi:softmax',    \n",
    "    'silent': 1,\n",
    "    'num_class': n_classes\n",
    "}\n",
    "\n",
    "# Parameters for the Grid Search\n",
    "params_gridsearch = { \n",
    "    'max_depth':[8,16],\n",
    "    'n_estimators':[500,1000],\n",
    "    'learning_rate': [0.01,0.0075],\n",
    "    'subsample': [0.5,0.75]\n",
    "}\n",
    "\n",
    "fit_params = {'eval_metric': 'mlogloss',\n",
    "              'early_stopping_rounds': 10,\n",
    "              'verbose': False,\n",
    "              'eval_set': [(X_test, y_test)]              \n",
    "             }\n",
    "\n",
    "# Specify Cross Validation Strategy\n",
    "cv = StratifiedShuffleSplit(y_train, n_iter=5, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use one hot encoded output for the train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=test_size, random_state=42)\n",
    "\n",
    "# gsearch = GridSearchCV(estimator = XGBClassifier(**params), param_grid = param_test, scoring='f1_weighted',n_jobs=4,iid=False, cv=cv)\n",
    "gsearch = GridSearchCV(estimator = XGBClassifier(**params_xgbclassifier), param_grid = params_gridsearch, scoring='accuracy', n_jobs=4, iid=False, cv=cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid search..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\datasciencelab\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grid Search time: 112631.91185426712\n",
      "Wall time: 1d 7h 17min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import time\n",
    "print(\"Grid search..\")\n",
    "search_time_start = time.time()\n",
    "train_model_gs = gsearch.fit(X_train, y_train,**fit_params)\n",
    "pred_gs = train_model_gs.predict(X_test)\n",
    "print(\"Grid Search time:\", time.time() - search_time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAHCCAYAAADl4BDwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzs3Xd8FVX6x/HPAwFsIF2BgIIiUsTQkQVkV0FUVOxYsaL+cF1X110L9rLWtax9lVXXAioqiAoiig2QLkVBQFBDkSaCgpTw/P6YE7jENCA3N8l837zmlTtn2pnhJs+cMmfM3REREZGyr1yqMyAiIiLFQ0FfREQkJhT0RUREYkJBX0REJCYU9EVERGJCQV9ERCQmFPTLGDPb3czeNrOfzey1XdjPWWb2flHmLRXM7D0z67uT295hZivMbGlR52sH85HnOZjZ/mbmZpZW3PnaESGPBxbRvsaY2UVFsa/iYma3mNmLSdz/LDPrFj6bmf3XzH4yswlm1sXM5iTr2FK6KOiniJmdaWaTzOwXM1sS/rB3LoJdnwLsA9Rw91N3difu/pK79yiC/GzHzLqFAPBGjvRDQ/qYQu6nUH9E3f1od39+J/JZH7gaaObu++7o9nns08zscjObbmbrzGxpCGB98ttuZ88hHLOzmY0NN4GrzOxzM2u3c2dQ8pnZA2Y2MkfaQ2Y2PGG+spn9y8wWmtmvZva9mb1uZu0T1vGw7Jdw4/eKmVUt4NjJ+p0ukLs3d/cxYbYz0B1Id/f27v6puzcpjnxIyaegnwJmdhXwEHAXUYBuADwOnFAEu98P+MbdNxfBvpJlOdDJzGokpPUFvimqA4QAuyvf7/2Ale6+bCeOnVep+xHgSqKbiRpAPWAA0DOP/ezSOZhZFWA48G+gejjercCGnd1nPscqKTUNNwIHmNn5AGZ2GNF369IwXwn4EDgE6AVUAZoCg4BjcuzrUHffC2gEVANuyeugSf6d3lH7AQvd/ddd3VEJ+n+VouLumopxAvYGfgFOzWedSkR/QBaH6SGgUljWDcgkChzLgCXA+WHZrcBGYFM4xoVEf6heTNj3/oADaWH+POBbYC2wADgrIf2zhO06AROBn8PPTgnLxgC3A5+H/bwP1Mzj3LLz/yTQP6SVD2k3AWMS1n0Y+AFYA0wGuoT0njnO88uEfNwZ8rEeODCkXRSWPwG8nrD/e4DRgOXI45Fh+y1h/8+F9OOBWcDqsN+mCdssBP4BTCcKqmk59nkQkAW0LeD7UdA5lAfuB1aE/7f+if+fOfbVFlhdwPEuAL4GfgJGAvsVdP3DsluA14EXw/KLQt6uB+aH78FkoH5Y34kC79xwrMcSr3sB+egOzCb67j0KfJx9PfL5jq0k+q5/DVySsOwiot+ZPQu4Lg4cmDD/f8D7u/A7fQvb/x6+BiwN5/QJ0Dxh2THAV+EaLgL+FtJrEt3ErQZWAZ8C5RK+f0cS/c7/Fr5rvxD9TegGZCbsvy4whOjmewFwRX7/r8n6W6gpNVPKMxC3iShgbc7tj3TCOrcB44HaQC1gLHB7WNYtbH8bUCH8gVgHVAvLc/5xyTm/f/iDlgbsGX6xm4RldbL/+JAQ9IlKiT8B54TtzgjzNcLyMUR/6A8Cdg/zd+dxbt2IAnwn4IuQdgzRH/qL2D7on01UIk4juslZCuyW23kl5ON7oHnYpgLbB8w9iGoTzgO6EAXO9PzymTB/EPArUQCqAPwdmAdUDMsXAtOA+sDuuezvUqLSV0Hfj4LO4VKiAFg//L98RN5BvwpR8HseODr7O5KwvHc4h6bhWAOAsTtw/TeFfZQL/+/XADOAJoABhyZ8R5woYFUlKgUvB3oWlA+iQLeGqNmqAvBXou9/vsEIeCr8/45h+5uLQYSbuAK23xr0iUr57wO37cLv9C1s/3t4AVCZbTf40xKWLWHbDW41oHX4/E+im+UKYeqSfW7h+3dkzt/dnN/l8H81megGuyJRLca3wFF5/b8m+2+ipuKdVL1f/GoAKzz/6veziP7ALHP35UR36+ckLN8Ulm9y93eJ7uh3ts1uC9DCzHZ39yXuPiuXdY4F5rr7/9x9s7u/QhR4jktY57/u/o27rwdeBTLyO6i7jwWqm1kT4FzghVzWedHdV4ZjPkD0B7Kg83zO3WeFbTbl2N86okD2L6KSzJ/dPbOA/WU7HXjH3UeF/d5PFOg6JazziLv/EK5BTjWJguZWZpZpZqvN7Dcz268w5wCcBjwUjrOKKBDkyt3XELXvOvAfYLmZDTOzfcIqlwD/dPevw/fxLiAjOy+FuP7j3P0td98SzvkiYIC7z/HIl+6+MmH9u919tbt/T3Szkv0dyS8fxwBfufvr4Vo8lPM65uFTot+1l9w98QUj2/0/mFlG+D9Yk0tntylmtpro5qEB0Y1EbgrzO70ddx/o7mvdfQNRoD3UzPYOizcBzcysirv/5O5TEtLrENWCbPKorX5HX57SDqjl7re5+0Z3/5bou5HYryTn/6uUIQr6xW8lULOAtrK6wHcJ89+FtK37yPEHZh2w145mxKM2v9OJSo9LzOwdMzu4EPnJzlO9hPnEP8SFzc//gMuBPwJv5lxoZleb2dehE9pqomrUmgXs84f8Frr7BKKSjRHdnBTWdtfA3beEYyVeg/yOvZLoD3ZiXtKJzqdSyE9h9lM3x/Kc/y/bCYH0vHCsFmH7h8Li/YCHQ9DLrjI2wjkV4vrnzGd9ohqfvOT1HckvH9udbwhy+f4fh74i94fzvC1HB7zt/h/cfZq7VwVOIvp/SNQ6LNuNqGnoUzPbLZdDFuZ3OjF/5c3sbjObb2ZriErpsO3ankx0s/OdmX0c+iUA3EdUI/K+mX1rZtcW5ng57AfUzb7W4XpfT9QPIVu+11dKNwX94jeOqM2tdz7rLCb65czWIKTtjF+JqrWzbdcT3d1Hunt3oj+Es4nu+gvKT3aeFu1knrL9j6it9N1QCt/KzLoQtZGfRlQtXZWo/TM7OOZVwsm35GNm/Yn+uC8mqqIvrO2ugZkZUZBLvAb5HftDIN3M2hbiWPntZ0k4brYGhdhftFP32cBzRMEfoj/ul7h71YRpd3cfW4jrn1s+fwAOKGx+cmyXaz7Icb4J1z0/DwEj3P2vRO3l9ycsGw30MLM9C5u5UMPwDNCQbdcuUWF+pxOdSdTB70iiG6n9Q7qF40109xOImvfeItychpqBq929EVEt21VmdkRhzyP4AViQ41pXdvfETox69WoZpqBfzNz9Z6L2tMfMrLeZ7WFmFczsaDO7N6z2CjDAzGqZWc2w/s4+4zsN6GpmDUL14XXZC8xsHzM7PvwB3EDUTJCVyz7eBQ4KjySlmdnpQDOiNtqd5u4LgMOBG3JZXJmonXQ5kGZmNxG1UWf7Edh/R3q3m9lBwB1EVfznAH83s3ybIRK8ChxrZkeYWQWiNu4NRP0tCuTuc4iqhweZWXeLxlMoz/bNA4XNxxVmlm5m1YA8S3tmdnAoraeH+fpE/THGh1WeBK4zs+Zh+d5mlv2YZ0HXPzfPALebWePw5EHLHE9o5CW/fLwDNDezk0JJ+gpy3LjmOOdjiPpdXBWS/gz0NrM/hvkXiG4k3jSzFqHUvRtRp8e89lkeOJ+oY+W3OZcX8nc6UWWi785KohvyuxKOVdGiMTL2Djcbawi/k2bWy8wODDc+2em5/b7mZwKwxsz+kf0dDNehzD7GKdtT0E8Bd/8X0R+lAUR/VH8gquZ+K6xyBzCJqCf4DGBKSNuZY40CBod9TWb7QF2OKHgtJqpSPZyo5J1zHyuJHm+6mugP1d+BXu6+YmfylGPfn7l7brUYI4H3iDrefUdUkkqsdsweeGilmU2hACFgvAjcE9qa5xJVa/7Pose4CsrnHKKbhX8TtfEeBxzn7hsL2jZBf6LH9v5FdL0ziZ56OJ2o815h/Ifo2nxJ9L14I5911wIdgC/M7FeiYD+T6P8Rd3+T6AmGQaGaeSZRhz8o+Prn5l9ENyXvEwWlZ4n6PeQrv3yE79ipwN1E373GRE82/I6ZVSa6gbgi9HfAo0curwb+Y1G/ld+ImpO+IrqhWAPMIWrrPi3HLr80s1+IOq32BU7M3m8u51DQ73SiF4iu6aKQj/E5lp8DLAzX4lKi7x3h3D8gujkfBzzu257NLxR3zyL67mYQ9dxfQXSztnd+20nZkd3zU0RERMo4lfRFRERiQkFfREQkJhT0RUREYkJBX0REJCYU9EVERGJCb1ACyu1WxctXrpXqbJRphzSoluoslGlW8CoiJdp33y1kxYoVxfJVLl9lP/fNuz7CsK9fPtLdc31LZkmloA+Ur1yLGifek+pslGmfPXpyqrNQppUrp7AvpdsfOhRmsMqi4ZvXU6lJzmEZdtxv0x4raFjwEkdBX0REYsag8IN5likK+iIiEi8GWDxrxxT0RUQkfmJa0o/nWYuIiMSQSvoiIhI/qt4XERGJg/h25IvnWYuIiMSQSvoiIhI/qt4XERGJASO21fsK+iIiEjMW25J+PG91REREYkglfRERiR9V74uIiMSEqvdFRESkLFNJX0REYia+g/Mo6IuISLzoLXsiIiIxEtOSfjzPWkREJIZU0hcRkZhRm76IiEh8lFObvoiISNkX47H343nWIiIiMaSSvoiIxI8e2RMREYmD+Hbki+dZi4iIxJBK+iIiEj+q3hcREYmJmFbvK+iLiEi8mMW2pB/PWx0REZEYUklfRETiJ6bV+/E8axERibfsKv5dmQo8hNU3s4/M7Gszm2Vmfwnp1c1slJnNDT+rhXQzs0fMbJ6ZTTez1gn76hvWn2tmfRPS25jZjLDNI2b5Z0xBX0REJDk2A1e7e1OgI9DfzJoB1wKj3b0xMDrMAxwNNA5TP+AJiG4SgJuBDkB74ObsG4WwTr+E7XrmlyEFfRERiZkwOM+uTgVw9yXuPiV8Xgt8DdQDTgCeD6s9D/QOn08AXvDIeKCqmdUBjgJGufsqd/8JGAX0DMuquPs4d3fghYR95Upt+iIiEj9F03u/pplNSph/2t2fzv1wtj/QCvgC2Mfdl0B0Y2BmtcNq9YAfEjbLDGn5pWfmkp4nBX0REYmXonvL3gp3b1vg4cz2AoYAV7r7mnya3XNb4DuRnidV76fAg33bMPP+Xoy5ufvWtKcu7sAHNx7JBzceycS7juaDG4/Mddt61Xdn0JVd+OTWHnxySw/q19gDgLeu6bZ1+2n3Hst//++w7bbL2K8ai548mV6t870JjIXMH37g6B5/onXLZrTNaMFj/34YgDeGvEbbjBbstVt5pkzedvO+adMmLr7wPNq1bknrls24795/pijnZcP7I0fQsnkTmh98IPfde3eqs1Pm6PqWLGZWgSjgv+Tub4TkH0PVPOHnspCeCdRP2DwdWFxAenou6XlSST8FBo/9joEfzeff57fbmnbJf77Y+vmWU1qyZv2mXLf99/nteejdr/nk62XsUak8Hu7pet83Zus6z1zakZHTtv2/lzMYcPIhjJm1tGhPpJQqn5bGXffcT6tWrVm7di2dO7blT0d2p1mzFrw8eAhXXH7pduu/MeQ1Nm7YwMQp01m3bh1tMppz2mlnsN/++6fmBEqxrKwsrryiP++8N4p66el07tiOXr2Op2mzZqnOWpmg61tYxfPCndCT/lnga3f/V8KiYUBf4O7wc2hC+uVmNoio097Pofp/JHBXQue9HsB17r7KzNaaWUeiZoNzgX/nlyeV9FNg/NwVrP51Y57Lj2ubzpsTf/hd+kF1KlO+vPHJ19FN4boNWazfmLXdOntWSqNzk9q8lxD0L/zTgbwzZREr1m4oojMo3erUqUOrVtGTMJUrV6bJwU1ZvGgRBzdtykFNmvxufTPj119/ZfPmzaxfv56KFSpSuUqV4s52mTBxwgQOOOBAGjZqRMWKFTn19D4Mf3towRtKoej67oBieGQP+ANwDvAnM5sWpmOIgn13M5sLdA/zAO8C3wLzgP8A/wfg7quA24GJYbotpAFcBjwTtpkPvJdfhlTSL2E6Nq7JijW/sWDZL79b1mifyqxZt4lnLz2MBjX34NOvl3HHGzPYktCCc0yrunw2exm//LYZgH2r7sYxrepx8gMfk7F/gU1PsfPdwoV8+eVU2rXvkOc6J550Cu+8PYwD9qvLunXruOe+f1G9evVizGXZsXjxItLTt9VS1quXzoQJX+SzhewIXd8dUAwlfXf/jNzb3QGOyGV9B/rnsa+BwMBc0icBLQqbp6SftZllhbubmWb2tplVDelvmlnvhPXmmNmAhPkhZnZS+HxdGHhgjpkdlbDOX8J+Z5nZlQnpz5nZKck+t2Q4sV39XEv5AGnljA6Na3Lr69PpedeHNKi1J6d32n/77ds32G7720/P4PYh298YSOSXX37hzD6ncO/9D1Iln5L7pIkTKFe+PPMWLmLWnG955KF/seDbb4sxp2WH+++/iAWMJSI7QNdXClIc1fvr3T3D3VsAq9h2FzMW6ARgZjWAX4DE3meHAWPDQAZ9gOZEgw48bmblzawFcDHRQAWHAr3MrHExnE/SlC9nHNO6HkMnZua6fPFP65n5/Wq+X/ErWVucEdMW07JB1a3Lq+1ZkYz9q/HB9CVb0w7drxpPXdyBiXcdTa/W6dx9Zit6ZtRN+rmUdJs2beLM00/h9D5nckLvk/Jd99VBL9O9x1FUqFCB2rVr07FTJ6ZMmZTvNpK7evXSyczcdlO6aFEmdevq+1hUdH13QPFU75c4xd2mP45tzxB+Tgj64edwoFYYhrAh0c3CUqLBCga5+wZ3X0DUbtEeaAqMd/d17r4Z+Bg4sRjPpch1bVqbeUvXsmT1+q1p+1bdjdf+2hWAaQtXsfceFaixV0UAOjepzTdL1m5d97g26XwwfQkbNm/Zmtb++vdoF6bhUzK59uWpjJiWb+fOMs/dueySi2hy8MFcceVVBa6f3qABH4/5CHfn119/ZeIXX3BQk4OLIadlT9t27Zg3by4LFyxg48aNvDZ4EMf2Oj7V2SozdH0LyYpncJ6SqNja9M2sPFEbxrMhaTLQwswqEgX9j4FGRMG8FdFNAUQ3CeMTdpU9+MBM4M5QS7AeOAYodPHLzPoRDV1Iub1q7txJ7aQnLmpPpya1qL5XJabccwz3DfuKVz5fSO929XlzwvZV+/vsvTubt0RBfIvDra9P57WrumJmTP/uJ178dFs1c+929fn3iNnFei6l0bixn/PKS/+jeYtD6NiuFQC33HYnGzdu4Oq/XsGK5cs5qXcvWrbMYNg7I7jk0v5cevEFtGt1CO7O2eeexyGHtEzxWZROaWlpPPjwoxx37FFkZWXR97wLaNa8eaqzVWbo+kpBLLc2oCI9gFkWMAPYnyjQ93D3rLDsc+Bq4J/AScAZwEaioD/D3Z80s8eAce7+YtjmWeBddx9iZhcSNRf8AnxFVDvwVzN7Dhju7q8XJo8Vah3gNU68p6hOuUhd8McDyFy1jve/XFLwyiXYt4+enOoslGnlypXOqkaRbH/o0JbJkycVyxe5XLX9vdIfb9zl/fz25kWTCzM4T0lSbG36wH5ARbbvmTgW6ApUDuMJjycq9XdiW0k/r0EJcPdn3b21u3cl6i8wN5knkgoDP5pf6gO+iEhJY2a7PJVGxdYo4e4/A1cAfwsjFEEU2C8Bvgzz04neRNQAmBXShgF9zKxSaOtvDEwAyB6v2MwaENUUvFIMpyIiIqWYoaBfLNx9KlGA7xOSxhK1448LyzcTDUc4yd23hLRZwKtE1fcjgP7ZzQPAEDP7Cng7pP+UcLinzCwzTOOSfGoiIiIlXtI78rn7Xjnmj0v4vIwcAxe4e7dc9nEncGcu6V3yOOZ5O5dbEREp84y8h8wp4zQin4iIxEzprZ7fVaXzQUMRERHZYSrpi4hI7MS1pK+gLyIisaOgLyIiEhNxDfpq0xcREYkJlfRFRCRe9MieiIhIPFiMH9lT0BcRkdiJa9BXm76IiEhMqKQvIiKxE9eSvoK+iIjETlyDvqr3RUREYkIlfRERiRc9siciIhIfca3eV9AXEZFYifNz+mrTFxERiQmV9EVEJHbiWtJX0BcRkfiJZ8xX9b6IiEhcqKQvIiLxYqreFxERiY24Bn1V74uISOyY2S5PhTjGQDNbZmYzE9IGm9m0MC00s2khfX8zW5+w7MmEbdqY2Qwzm2dmj1g4uJlVN7NRZjY3/KxWUJ4U9EVERJLjOaBnYoK7n+7uGe6eAQwB3khYPD97mbtfmpD+BNAPaBym7H1eC4x298bA6DCfLwV9ERGJlezBeZJd0nf3T4BVueYh2sFpwCv55tWsDlDF3ce5uwMvAL3D4hOA58Pn5xPS86SgLyIi8WNFMEFNM5uUMPXbgRx0AX5097kJaQ3NbKqZfWxmXUJaPSAzYZ3MkAawj7svAQg/axd0UHXkExGReCm63vsr3L3tTm57BtuX8pcADdx9pZm1Ad4ys+bkPqKA7+QxFfRFRESKk5mlAScBbbLT3H0DsCF8nmxm84GDiEr26QmbpwOLw+cfzayOuy8JzQDLCjq2qvdFRCR2iqNNPx9HArPdfWu1vZnVMrPy4XMjog5734Zq+7Vm1jH0AzgXGBo2Gwb0DZ/7JqTnSUFfRERip5ge2XsFGAc0MbNMM7swLOrD7zvwdQWmm9mXwOvApe6e3QnwMuAZYB4wH3gvpN8NdDezuUD3MJ8vVe+LiIgkgbufkUf6ebmkDSF6hC+39ScBLXJJXwkcsSN5UtAXEZH4ieeAfAr6IiISP3EdhldBX0REYqUIOuKVWurIJyIiEhMq6YuISOzEtaSvoC8iIrET16Cv6n0REZGYUEkfaF6/Kh8+WODLiWQXbPGdHipaCqFcXJ8/EtlZMf2VUdAXEZHYiWv1voK+iIjES9G9Za/UUZu+iIhITKikLyIisWJATAv6CvoiIhI38R2RT0FfRERiJ6YxX236IiIicaGSvoiIxI6q90VEROLAVL0vIiIiZZxK+iIiEisGlCsXz6K+gr6IiMROXKv3FfRFRCR24tqRT236IiIiMaGSvoiIxEuMe+8r6IuISKxEY+/HM+qrel9ERCQmVNIXEZGY0Qt3REREYiOmMV9BX0RE4ieuJX216YuIiMSESvoiIhIvemRPREQkHvTInoiISIyY7fpU8DFsoJktM7OZCWm3mNkiM5sWpmMSll1nZvPMbI6ZHZWQ3jOkzTOzaxPSG5rZF2Y218wGm1nFgvKkoC8iIpIczwE9c0l/0N0zwvQugJk1A/oAzcM2j5tZeTMrDzwGHA00A84I6wLcE/bVGPgJuLCgDCnoi4hI7JjZLk8FcfdPgFWFzNIJwCB33+DuC4B5QPswzXP3b919IzAIOMGiDPwJeD1s/zzQu6CDKOiLiEjsFFH1fk0zm5Qw9Svk4S83s+mh+r9aSKsH/JCwTmZIyyu9BrDa3TfnSM+Xgr6IiMjOWeHubROmpwuxzRPAAUAGsAR4IKTnVnXgO5GeL/XeFxGReLHU9d539x+3ZsPsP8DwMJsJ1E9YNR1YHD7nlr4CqGpmaaG0n7h+nlTSFxGRWIke2Ut+7/1cj21WJ2H2RCC7Z/8woI+ZVTKzhkBjYAIwEWgceupXJOrsN8zdHfgIOCVs3xcYWtDxVdIXEZGYKZ4X7pjZK0A3orb/TOBmoJuZZRBVxS8ELgFw91lm9irwFbAZ6O/uWWE/lwMjgfLAQHefFQ7xD2CQmd0BTAWeLShPCvoiIiJJ4O5n5JKcZ2B29zuBO3NJfxd4N5f0b4l69xeagr6IiMROTAfkU9AXEZH40TC8IiIiUqappC8iIvES47fsqaSfYn++7CKa7F+XP7TLyHX5hg0buPDcM2nb8mC6d+vE998tBGDjxo1cfumFdG6fQdeOrfnsk4+3bjPk1UF0bp9Blw6tOLX3saxcsaI4TqXU+O233+jWuSOHtWtFu1aHcOdttwBwyUXn06LJAXRq35pO7Vsz/ctpAMyZM5s/Hf4HalTZnYcffCCfPUthvD9yBC2bN6H5wQdy3713pzo7ZY6ub8Gy37KX7GF4SyIF/RQ746y+vPrW8DyXv/j8QKpWrcqk6bO5rP9fuPXG6wF44b/PAPDZhGkMGTaCm66/hi1btrB582au//tVDH33Az79YirNWxzCM089XiznUlpUqlSJ4SM+YNzEqYydMIUPRo1kwhfjAbjjn/cwdsIUxk6YQstDoxux6tWqc98DD3HFlVenMttlQlZWFlde0Z+hb7/H1Olf8dqgV/j6q69Sna0yQ9e38BT0JSU6de5CtWrV81z+3jtv0+escwA4/sST+WTMh7g7c2Z/TddufwKgVu3aVNm7KlOnTMLdcXfWrfsVd2ftmjXsW6dOnvuPIzNjr732AmDTpk1s2rQp31/gWrVr06ZtOypUqFBcWSyzJk6YwAEHHEjDRo2oWLEip57eh+FvFzieiBSSrq8UREG/hFuyeDF106MRGNPS0qiy996sWrmSFoe05L3hb7N582a+W7iAL6dNYVFmJhUqVOD+hx6lc4dWND+wAXNmf83ZfS9I8VmUPFlZWXRq35pG9fflj0ccSbv2HQC49eYb6dg2g2uvuYoNGzakOJdlz+LFi0hP3zaiaL166SxatCiFOSpbdH0LL1Uj8qVa0oK+mWWZ2TQzm2lmb5tZ1ZD+ppn1TlhvjpkNSJgfYmYnhc/Xmdm8sM5RCev8Jex3lpldmZD+nJktCMf90syOSNb5FZdopMXtmRlnnXs+devV44guHbj+H1fTvsNhpKWlsWnTJgY+8xRjPp/IrHnf06zFITx4/z0pyHnJVr58ecZOmMLs+d8zeeJEvpo1k1tvv4sp07/i48+/YNWqVTx4/72pzmaZk9f3WYqGrm/hqXq/6K139wx3b0H0PuH+IX0s0AnAzGoAvwCHJWx3GDDWzJoRjTHcHOgJPG5m5c2sBXAx0ShEhwK9zKxxwvbXuHsGcCXwZNLOrpjUrVePxZnRWxU3b97Mmp9/plr16qSlpXHnPQ/w8bjJvDT4DX7+eTWNDjiQGdOjzmcNGx2AmdH7pFOZ+MW4VJ5CiVa1alW6dD2cUe+PZN86dTAzKlWqxDnnnsekSRNSnb0yp169dDIzt70ldNGiTOrWrZvCHJUtur5SkOIprWcQAAAgAElEQVSq3h/Htvf8fk4I+uHncKCWRRoS3SwsBU4ABrn7BndfAMwjCvRNgfHuvi68WehjopcW5HfMUuU/Tz7Gf558DICex/Ri0Ev/A2DYm0PocvgfMTPWrVvHr7/+CsBHH35AWvk0Dm7ajDp16/HN7K9ZsXw5AGM+/ICDmhycmhMpoZYvX87q1asBWL9+PR99OJqDmjRh6ZIlQFRaGv72UJo1b5HKbJZJbdu1Y968uSxcsICNGzfy2uBBHNvr+FRnq8zQ9S2kIqjaL6UF/eQ/p29m5YEj2Dbe8GSgRXhbUCeioN2IKJi3IropgChgj0/YVWZImwncGWoJ1gPHAJNyOXRP4K0iPZkkuPi8s/n8049ZuXIFLQ7an2tvuIm538yhQ8fovujsvhdw2UXn0bblwVStVo1nnnsJgBXLl3FK72MpZ+WoU7cuTzzzHAB16tTlmusG0OuoP1GhQhr1G+zHo08W+A6GWPlx6RIuueh8srKy2LJlCyedfCpHH9OLY486khUrluPutGx5KA89+kRYfyld/9CetWvWUK5cOR5/9GEmTp1JlSpVUnwmpU9aWhoPPvwoxx17FFlZWfQ97wKaNW+e6myVGbq+hWPF9MKdkshyawMqkh2bZQEzgP2JAn2PhDcGfQ5cDfwTOAk4A9hIFPRnuPuTZvYYMM7dXwzbPAu86+5DzOxCouaCX4jeSLTe3f9qZs8BhxO9vag20NHds19bmDN//YB+AOn1G7T58uv5RX8RdtIZp5zA8y+/RsWKFVOdlSJTMU19RpMprbyur5Ruf+jQlsmTJxVLJK7SoKm3u2bgLu/nwys6TXb3tkWQpWKT9DZ9YD+gItva9CFq1+8KVHb3n4hK9J3ClF3SzwTqJ2yTDiwGcPdn3b21u3cl6i8wN2G9a4ADgQHA83llzt2fdve27t62Rs2aO3+WSfDK60PLVMAXEZGSIenFA3f/GbgC+JuZZT/o/DnRO4S/DPPTgY5AAyD7PcHDgD5mVim09TcGJgCYWe3wswFRTcErOY65BXgYKJfY619ERASgnNkuT6VRsdQJuvtUogDfJySNJWrHHxeWbwaWAZNCwMbdZwGvElXfjwD6ZzcPAEPM7Cvg7ZD+Uy7HdOAO4O/JOi8RESmd1JGviLn7Xjnmj0v4vIxo+OPE5d1y2cedwJ25pHfJ45jn5ZgfAgzZgWyLiEgZFwXtUhq1d5F6/4iIiMSEXq0rIiKxUy6eBX0FfRERiR9V74uIiEiZppK+iIjETkwL+gr6IiISL0Y0FG8cKeiLiEjsxLUjn9r0RUREYkIlfRERiReL71v2FPRFRCR2YhrzVb0vIiISFyrpi4hIrBiU2rfk7SoFfRERiZ2YxnxV74uISPxY6My3K1MhjjHQzJaZ2cyEtPvMbLaZTTezN82sakjf38zWm9m0MD2ZsE0bM5thZvPM7BELBzez6mY2yszmhp/VCsqTgr6IiEhyPAf0zJE2Cmjh7i2Bb4DrEpbNd/eMMF2akP4E0A9oHKbsfV4LjHb3xsDoMJ+vPIO+mVXJbypoxyIiIiWRWdFMBXH3T4BVOdLed/fNYXY8kJ5/Xq0OUMXdx7m7Ay8AvcPiE4Dnw+fnE9LzlF+b/izAYbuxCrPnHWhQ0M5FRERKohLSke8CYHDCfEMzmwqsAQa4+6dAPSAzYZ3MkAawj7svAXD3JWZWu6AD5hn03b3+DmZeRESkVCiikF/TzCYlzD/t7k8X6vhmNwCbgZdC0hKggbuvNLM2wFtm1jyPrPrOZrhQvffNrA/QyN3vMrN0oruLyTt7UBERkTJghbu33dGNzKwv0As4IlTZ4+4bgA3h82Qzmw8cRFSyT2wCSAcWh88/mlmdUMqvAywr6NgFduQzs0eBPwLnhKR1wJN5byEiIlKyFUfv/TyO2xP4B3C8u69LSK9lZuXD50ZEHfa+DdX3a82sY+i1fy4wNGw2DOgbPvdNSM9TYUr6ndy9dWhnwN1XmVnFwp2eiIhIyRINzlMMxzF7BehG1AyQCdxM1Fu/EjAq3DiMDz31uwK3mdlmIAu41N2zOwFeRvQkwO7Ae2ECuBt41cwuBL4HTi0oT4UJ+pvMrByhDcHMagBbCrGdiIhIbLn7GbkkP5vHukOAIXksmwS0yCV9JXDEjuSpMEH/sZCRWmZ2K3AacOuOHERERKTE0Fv28ubuL5jZZODIkHSqu8/MbxsREZGSLKYxv9Bj75cHNhFV8WsUPxERKdXiWtIvTO/9G4BXgLpEjwq8bGbX5b+ViIiIlDSFKemfDbTJfrTAzO4EJgP/TGbGREREkqG4eu+XRIUJ+t/lWC8N+DY52REREUm+uFbv5xn0zexBojb8dcAsMxsZ5nsAnxVP9kRERKSo5FfSz+6hPwt4JyF9fPKyIyIiknzxLOfn/8KdXAcQEBERKc3MSsxb9opdgW36ZnYAcCfQDNgtO93dD0pivkRERJImpjG/UM/cPwf8l6g25GjgVWBQEvMkIiIiSVCYoL+Hu48EcPf57j6A6K17IiIipVKq3rKXaoV5ZG9DeJ3ffDO7FFgE1E5utkRERJKnlMbsXVaYoP9XYC/gCqK2/b2BC5KZKRERkWQxTB358uLuX4SPa4FzkpsdERERSZb8Bud5k2gwnly5+0lJyZGIiEgymar3c/NoseUixcqbsUelwr5wUKTkqdbu8lRnocz74dOHUp2FMi3L8yxjJkVp7Yi3q/IbnGd0cWZEREREkkvFWxERiZ3CPK9eFinoi4hIrBiq3i+QmVVy9w3JzIyIiEhxKBfPmF9wDYeZtTezGcDcMH+omf076TkTERGRIlWYZo1HgF7ASgB3/xINwysiIqVYOdv1qTQqTPV+OXf/Lkf7R1aS8iMiIpJUZmrTz88PZtYecDMrD/wZ+Ca52RIREZGiVpigfxlRFX8D4Efgg5AmIiJSKpXW6vldVZix95cBfYohLyIiIsUiprX7BQd9M/sPuYzB7+79kpIjERGRJDLQW/by8UHC592AE4EfkpMdERERSZbCVO8PTpw3s/8Bo5KWIxERkSTTMLyF1xDYr6gzIiIiUlxiWrtfqBH5fjKzVWFaTVTKvz75WRMRESl6Zka5IpgKcZyBZrbMzGYmpFU3s1FmNjf8rBbSzcweMbN5ZjbdzFonbNM3rD/XzPompLcxsxlhm0esEIMP5Bv0ww4OBWqFqZq7N3L3Vws8WxERkXh7DuiZI+1aYLS7NwZGh3mAo4HGYeoHPAHRTQJwM9ABaA/cnH2jENbpl7BdzmP9Tr5B390deNPds8L0u178IiIipU00Kt+uTQVx90+AVTmSTwCeD5+fB3onpL/gkfFAVTOrAxwFjHL3Ve7+E1Fte8+wrIq7jwux+YWEfeWpMH0ZJiRWM4iIiJR2KRx7fx93XwIQftYO6fXY/sm4zJCWX3pmLun5yrMjn5mluftmoDNwsZnNB34lesTR3V03AiIiEmc1zWxSwvzT7v70Tu4rt9sI34n0fOXXe38C0JpCVBeIiIiUFkU4OM8Kd2+7g9v8aGZ13H1JqKJfFtIzgfoJ66UDi0N6txzpY0J6ei7r5yu/6n0DcPf5uU0F7VhERKSkKo42/TwMA7J74PcFhiaknxt68XcEfg7V/yOBHmZWLXTg6wGMDMvWmlnH0On+3IR95Sm/kn4tM7sqr4Xu/q+Cdi4iIlLi7FqbfOEPY/YKUSm9ppllEvXCvxt41cwuBL4HTg2rvwscA8wD1gHnA7j7KjO7HZgY1rvN3bM7B15G9ITA7sB7YcpXfkG/PLAXubcbiIiISD7c/Yw8Fh2Ry7oO9M9jPwOBgbmkTwJa7Eie8gv6S9z9th3ZmYiISGlgMS3P5hf043lFRESkTIs68qU6F6mRX0e+31U/iIiISOmVZ0k/oaOAiIhImRLXkv7OvGVPRESkVCvEu2nKJAV9ERGJFbXpi4iISJmnkr6IiMTLro2oV6op6IuISOwU0dj7pY6CvoiIxIra9EVERKTMU0lfRERiJ6a1+wr6IiISN0a5mI40r+p9ERGRmFDQL0XeHzmCls2b0PzgA7nv3rtTnZ0ySde4cNL3qcqIp69g6pABTH79Bvqf0Q2Au67szbQ3BjBh8HUMfuBi9t5r999t2/Kgeox5/momv34DEwZfxyk9Wm9d9sTNZ/LF4GuZMPg6Xr7vQvbcvSIA9159EuMHXcv4Qdcy/a2bWPLJvcVyniXFFZddRNOGdenSPiPX5c89+xRdO2TQrVMbju1+OHNmfwXAmA8/4Igu7enaIYMjurTn048/2rrNxo0buerPl9IhoxmHtW7B20PfKJZzKQmMqHp/V6fSyKJX+MZbmzZt/fMvJqU6G/nKysrikGYH8c57o6iXnk7nju14/sVXaNqsWaqzVmaU5mtcrd3lxXq8fWtWYd+aVZg2O5O99qjE2Jf/wWlXPU292lUZM/EbsrK2cMcVJwAw4JGh2217YIPaOM7875dTp9befP7S32l10h38/Mt6Ku+5G2t//Q2Ae64+ieWr1nL/f0dtt/1lfQ7n0CbpXHrrS8VzssEPnz5UrMdLNPazT9lzrz25vN8FfDph2u+Wr12zhspVqgAw4p23GfjMk7z65jtM/3IqtWvvw7516vL1VzM5rfexzPjmOwDuufNWsrKyuP6m29iyZQs/rVpFjZo1i/W8Eh3ZtQPTpkwullC6X9OWft3AYbu8n8s6NZzs7m2LIEvFRiX9UmLihAkccMCBNGzUiIoVK3Lq6X0Y/vbQgjeUQtM1LrylK9YwbXYmAL+s28DsBUupW6sqo8fPJitrCwATZiyg3j5Vf7ftvO+XMf/75QAsWf4zy39aS83qewFsDfgAu1WqQG6FktN6tuHVEZOL/JxKsk6du1CtWvU8l2cHfIB1637dOq58y0NbsW+dugAc3LQ5G377jQ0bNgDw8v+e4y9X/wOAcuXKpTTgp0I5s12eSiMF/VJi8eJFpKfX3zpfr146ixYtSmGOyh5d453ToE51MpqkM3Hmwu3Szz3hMEZ+/lW+27Ztvh8V09L49ocVW9OeuuVsFn5wF03234fHB32c41jV2K9uDcZMnFNk+S8rnn36cdq1bMKtN17HXfc++Lvlbw99g0MOzaBSpUr8vHo1AHfffjN/6tyOC87pw7JlPxZ3liUFkh70zSzLzKaZ2Uwze9vMqob0N82sd8J6c8xsQML8EDM7ycy6m9lkM5sRfv4pLK8c9ps9rTCzh8KyW8zsb8k+t+KUW4knrm+JShZd4x235+4VeeX+i7jm/iHbldL/fuFRZGVtYdC7E/Pcdt+aVXj2jnO55JYXt7v2l9zyIo163MDsBUs5pUeb7bY59ag2vDV6Glu2qFkypwv7/R8Tp8/hptvu4l/33rXdstlfz+L2m67n/ocfB2Dz5s0sXpRJ+8M68eFnE2nXvgO33PD3VGQ7JeLcpl8cJf317p7h7i2AVUD/kD4W6ARgZjWAX4DDErY7LKyzAjjO3Q8B+gL/A3D3tWG/Ge6eAXwHlNmeKPXqpZOZ+cPW+UWLMqlbt24Kc1T26BrvmLS0crxy/8UMfm8SQz/8cmv6Wcd14JiuLTjvhufy3LbynrvxxiOXcetjw5kwY+Hvlm/Z4rz+/hR6H7F9x7VTjmrDqyNKdv+bVDvxlNN5751t7dWLF2XS94xTefSpgTRsdAAA1WvUYI899uDY46Jy1/EnnsL0ab/vK1CWqXq/eIwD6oXPnxOCfvg5HKhlkYZENwtL3X2quy8O680CdjOzSok7NbPGQG3g06SfQYq0bdeOefPmsnDBAjZu3MhrgwdxbK/jU52tMkXXeMc8efNZzFmwlEde/HBrWvdOTbn6vCM55cqnWP/bpq3pdWvtzbtP/hmACmnlGfzAxbw8/Ave+GDqdvtsVH9bu/KxXQ/hm4Xbqpwb71ebalX2YPyXC5J1SqXKM089xjNPPQbA/Hlzt6aPGvEujQ44EICfV6/mzFOOZ8Ctd9DhsD9sXcfM6HF0Lz7/NGo++WTMhxx0cNNizL2kSrENzmNm5YEjgGdD0mSghZlVJAr6HwONgKZAK6KbgpxOBqa6+4Yc6WcAg30HHkUws35AP4D6DRrswJmkRlpaGg8+/CjHHXsUWVlZ9D3vApo1b57qbJUpusaF1ymjEWf16sCMbxYxftC1ANz86DAeuOZUKlVMY/gT0dMEE2Ys5Io7B7Fvrb3ZHDr4ndyjNZ1bH0j1qnty9vEdAeh30/+YMXcxz9x2DpX33B0zmPHNIq64a/DWY57Wsy2vjYxXB75s/c4/m88//ZhVK1fQssn+/P36m5j7zRw6dIzKTc8+/TiffPQhaRXSqFq1Go8+NRCAZ55+nAXfzueBe+7kgXvuBOC1oe9Rq1ZtbrrtLv7v4vMY8I+rqFGzFo888UzKzi8VSmlBfZcl/ZE9M8sCZgD7EwX6Hu6eFZZ9DlwN/BM4iSh4byQK+jPc/cmE/TQHhoXt5+c4xlfAOe4+OczfAvzi7vcXJo+l4ZE9kfwU9yN7O+rS07vyw9KfeOfjGanOyk5L5SN7uTnzlBN47uXXqFixYqqzUiSK85G9hk1b+s0vDN/l/Zzffj89speL9aHNfT+gItva9CFqs+8KVHb3n4DxRKX+TiSU9M0sHXgTODeXgH8okJYd8EWk5Hly8CelOuCXRC+/PrTMBPxiZ1ETx65OpVGxtem7+8/AFcDfzKxCSP4cuATI7gU0HegINCBqvyf09n8HuM7dc6vyPwN4JYlZFxERKROKtSOfu08lCvB9QtJYonb8cWH5ZmAZMMndt4R1LgcOBG5MeDyvdsJuTyP3oD/AzDKzpyScjoiIlFJWBFNplPSOfO6+V4754xI+LyPHtXP3bjnm7wDuyGf/jXJJuwW4ZWfyKyIiZZtBqX3kblfp1boiIhI78Qz5GoZXREQkNlTSFxGR2Ilp7b6CvoiIxE3pfeRuV6l6X0REJAnMrEmOF8OtMbMrw0vhFiWkH5OwzXVmNi+8hO6ohPSeIW2emV27s3lSSV9ERGLFKJ4Sr7vPATJg61D0i4gGmjsfeDDnqLFm1ozokfbmQF3gAzM7KCx+DOgOZAITzWyYu+f/7upcKOiLiEjspKB6/whgvrt/l8+xTwAGhffLLDCzeUD7sGyeu38LYGaDwro7HPRVvS8iIrGTgsF5+rD9QHKXm9l0MxtoZtVCWj3gh4R1MkNaXuk7TEFfRERk59Q0s0kJU7/cVgpvkz0eeC0kPQEcQFT1vwR4IHvVXDb3fNJ3mKr3RUQkXqzIqvdXFPIte0cDU9z9R4DsnwBm9h8g+5V/mUD9hO3SgcXhc17pO0QlfRERiZXsjny7Ou2A7V4MZ2Z1EpadCMwMn4cBfcyskpk1BBoDE4CJQGMzaxhqDfqEdXeYSvoiIiJJYmZ7EPW6vyQh+V4zyyCqol+YvczdZ5nZq0Qd9DYD/d09K+zncmAkUB4Y6O6zdiY/CvoiIhI7xdV7393XATVypJ2Tz/p3Anfmkv4u8O6u5kdBX0REYiee4/Ep6IuISAzFdBRedeQTERGJC5X0RUQkVqLe+/Es6ivoi4hI7MS1el9BX0REYsawmJb01aYvIiISEyrpi4hI7Kh6X0REJAbi3JFP1fsiIiIxoZK+iIjEi6l6X0REJDYU9EVERGJCj+yJiIhImaaSvoiIxIoB5eJZ0FfQFxGR+FH1voiIiJRpKumLiEjsqPe+iIhITMS1el9BX0REYiXOHfnUpi8iIhITKumLiEjMmKr3RUREYkFj74uIiMRHTGO+2vRFRETiQiV9kTIg89OHUp2FMi+9619TnYUybcPsH4rtWFHv/XiW9RX0RUQkduIZ8lW9LyIiEhsq6YuISPzEtKivoC8iIrGj5/RFRERiIqb9+NSmLyIikixmttDMZpjZNDObFNKqm9koM5sbflYL6WZmj5jZPDObbmatE/bTN6w/18z67mx+FPRFRCR2rAimHfBHd89w97Zh/lpgtLs3BkaHeYCjgcZh6gc8AdFNAnAz0AFoD9ycfaOwoxT0RUQkfoo56udwAvB8+Pw80Dsh/QWPjAeqmlkd4ChglLuvcvefgFFAz505sIK+iIhI8jjwvplNNrN+IW0fd18CEH7WDun1gMRRijJDWl7pO0wd+UREJFaignqR9OSrmd1OHzzt7k/nWOcP7r7YzGoDo8xsdgFZy8nzSd9hCvoiIhIvRfeWvRUJ7fS5cvfF4ecyM3uTqE3+RzOr4+5LQvX9srB6JlA/YfN0YHFI75YjfczOZFjV+yIiEjvF0aRvZnuaWeXsz0APYCYwDMjugd8XGBo+DwPODb34OwI/h+r/kUAPM6sWOvD1CGk7TCV9ERGR5NgHeNOiaoU04GV3H2FmE4FXzexC4Hvg1LD+u8AxwDxgHXA+gLuvMrPbgYlhvdvcfdXOZEhBX0RE4qcYBudx92+BQ3NJXwkckUu6A/3z2NdAYOCu5klBX0REYsY0DK+IiEhcaBheERERKdNU0hcRkVjZ9QH1Si8FfRERiZ+YRn1V74uIiMSESvoiIhI76r0vIiISE3Htva+gLyIisRPTmK82fRERkbhQSV9EROIlxs/sKeiLiEjsxLUjn6r3RUREYkIlfRERiRVDvfdFRERiI6YxX0FfRERiKKZRX236IiIiMaGSvoiIxE5ce+8r6IuISOyoI5+IiEhMxDTmq01fREQkLlTSFxGR+IlpUV8l/VLk/ZEjaNm8Cc0PPpD77r071dkpk3SNd9wVl13EwQ3r0rl9Rq7LH//3g3Rq25KuHVtxYq8e/PD9d1uX3XrjdXRun0Hn9hm8OeTVremfjPmQP3ZuR7dObTi2++F8O39e0s+jJEnfpyojnvozU4fcwOTXrqf/GYcDcNeVJzBtyAAmDL6WwfdfxN577Z7r9mf1as+Mt25kxls3clav9lvTTzuqDRMHX8eEwdcy9NHLqFF1TwCqVdmD4Y/3Z8ZbNzL88f5UrZz7fsuKaOj9Xf9XGinolxJZWVlceUV/hr79HlOnf8Vrg17h66++SnW2yhRd453T56y+DH5zeJ7LDzm0FR98Mp5Pxk/luN4nccuN1wHw/oh3mf7lVMaMncTIjz7n0YcfYO2aNQD87crLeeqZFxgzdjInn9aHf917V7GcS0mxOWsL1z74Jq1OvpPD+z7AJad15eCG+zJ6/BzanHYX7U+/m7nfL+OaC7r/bttqVfbghn5H0/XcB+hyzv3c0O9oqlbenfLly3HfNSfT85JHaH/63cycu5hLT+8KwN/O786YCd9wSO/bGTPhG/52/u/3K2WDgn4pMXHCBA444EAaNmpExYoVOfX0Pgx/e2iqs1Wm6BrvnE6du1CtWvU8l3fp2o099tgDgLbtOrBkUSYAc2Z/TafOXUlLS2PPPfekRYuWjP5gJABmxtq10Q3Amp/XsG+dukk+i5Jl6Yo1TJsdXadf1m1g9oKl1K29N6PHzyYrawsAE2YspF7tqr/btvthTRn9xWx+WrOO1WvXM/qL2fTo1AyzqMf6nrtXBKDynruxZPnPAPQ6/BBeHP4FAC8O/4LjurUsjtNMnXAtdnUqjRT0S4nFixeRnl5/63y9euksWrQohTkqe3SNk++lF/7LET16AtDikJaMfn8E69atY+WKFXz26ccsyowC3UOPPkWfk4/nkCb78+qgl/jLVX9PZbZTqkGd6mQ0SWfizO+2Sz/3hI6MHPv7mqi6tfcmc+nqrfOLflxN3dp7s3nzFv5y16tMHHwd3468g6aN9uW5t8YBULtGZZauiG6ylq5YQ63qlZN4RiWDFcFUGiUt6JtZlplNM7OZZva2mVUN6W+aWe+E9eaY2YCE+SFmdpKZdTezyWY2I/z8U1heOew3e1phZg+FZbeY2aKQ/pWZnZGs8ytu7v67NCutt5ollK5xcr066CWmTZnM5X+5GoA/HtGdI486mmOO7EK/C86mbfsOpKWVB+DJxx5m0JBhzJizkDPO7suA6/6WyqynzJ67V+SV+y/kmgfeYO2vv21N//uFPcjavIVB70763Ta5fWfdIS2tHBef2pmOZ95Lo6MGMHPuYq45v0dS81+ixTTqJ7Okv97dM9y9BbAK6B/SxwKdAMysBvALcFjCdoeFdVYAx7n7IUBf4H8A7r427DfD3TOA74A3ErZ/MKSfADxlZhWSdobFqF69dDIzf9g6v2hRJnXrxqvKM9l0jZPn449G8+B9d/Piq29SqVKlrelXXXMdY8ZOZsiwEbhDowMas2L5cmbNnE6bdh0AOPHkU5n4xfhUZT1l0tLK8cr9FzH43UkM/fDLreln9WrPMV1acN6A53PdbtGPq0nfd1u1f719qrJk+c8celA6AAsyVwDw+qgpdDy0IQDLVq5l35pVANi3ZhWWr1qblHOS1Cuu6v1xQL3w+XNC0A8/hwO1LNKQ6GZhqbtPdffFYb1ZwG5mVilxp2bWGKgNfJrzgO4+F1gHVCvys0mBtu3aMW/eXBYuWMDGjRt5bfAgju11fKqzVaboGhedZ556jGeeegyA6V9O5eq//B8vDn6DWrVqb10nKyuLVStXAjBr5nS+mjmDPx7RnarVqrHm55+ZN/cbAMZ8+AEHNTm4+E8ixZ686SzmLFjKIy99tDWte6emXH3ekZxy5dOs/23T1vS6tfbm3ScvB2DUuK85smNTqlbenaqVd+fIjk0ZNe5rFi9bzcEN96Vm1b0AOKLDwcxZ8CMA73wyg7N7RTdZZ/fqwPCPZxTXaaZIUfTdL51F/aQ/p29m5YEjgGdD0mSghZlVJAr6HwONgKZAK6KbgpxOBqa6+4Yc6WcAgz2Xelkzaw3MdfdlRXIiKZaWlvb/7d15vF3T3cfxz1ciCGqKMaYoYhakhio1DzXWGDWWUh4aVWNLUS2KFjVXqJgaM09qHmomhkwiCAnxGGJIpYgh5Ob3/LHWkZ3r3uTeuOeenLu/79frvnLP3uvsvc6+O+e3pr0W5/3tInbYbmsaGj89H98AABmLSURBVBrY/4ADWWXVVWudrQ7F13jmHPzzfXjy8Uf56D/jWb3nshz/u5N57dVRrLt+KtufetIJfDZxIgft1weA7ksuzfU33c7XX3/N9ltvCsC8887LpVf0p3Pn9JV03oWX8fN99mC22WZjvvkX4IJL+tXmw9XID3stlx67e+0dBg04HoBTLvoXfz1uN+aYvTN3XpoaTp8dMZa+Z9zIYt2+x+Q8wG/CJ59z5hX38sR1xwJwRr97mPDJ5+n3y+/hgSuP5OvJDfzfuI845JTrAPjLVQ9w3VkHsv/O6/PWexPY+7h/tPdHbndl7blTU/2YbXJgqQEYASxLCvRbRURD3vckcDRwJrALKXh/RQr6IyLissJxVgUG5vePaXSOl4B9I2Jwfn0qcDCpy2A5YJuIeKiZ/B0CHAKw1NJLr/PqmDebSmZWFz77cnKtszCNvXbbiav/eTNdunSpdVbazJIbH1XrLDTr0D035q1xH3HXYy/WOiszbdIrNzLl8w/aJRSv3mudGPhgU/XL1llu4bkGR0TvNshSu6l6nz6wDNCFqX36kPrsNwbmjYgJwCBSrf+HFGr6kpYEbgf2ayLgrwl0rgT8gvMioiewJ3CNpDmbylxEXB4RvSOi98LdFv4un9PMGhlwy/92qIA/q7vsxsfqOuC3t7YYw9eS0omkpSQ9LOllSSMlHZm3FwedD5P0k8J7fitpdB7kvnVh+zZ522hJJ8zsZ696n35EfAz0BY4pDKp7EvglUBmd8gKwPrA0qf+ePNr/LuC3EdFUkWwvYMB0znsb8DxpEKCZmdlU7TN6fzJwdESsTIpxh0taJe87rzAo/W6AvK8PsCqwDXCJpE65m/xiYFtgFWCvwnFapV0G8kXEUFKA75M3PUVqfn86758MfAA8HxFTcpojgOWB3xdKQ4sUDrsH0wn62WnAbyR5PgIzM/tGewzki4hxETEk//4p8DJTB7U3ZSfghoiYFBFvAKOBdfPP6Ih4PSK+Am7IaVutagP5ImKeRq93KPz+AY3KSRGxSaPXfwL+NJ3jL9fEtlMbvR4M9GxFts3MzNqcpGVJ49aeATYEjpC0H6lF+ujc1d2d1N1d8TZTCwlvNdq+3szkwzVgMzMrnTaahrebpOcLP4c0fS7NA9wK/DoiPgEuBb4P9ALGAX+tJG3i7TGd7a3mpXXNzKx02ugxgfEzGr2fx7LdClyfx5oREe8X9vcjzVcDqQa/VOHtSwKV+Wqa294qrumbmZlVgdKcyFcCL0fEuYXtixeS/RSoPHoxEOgjaY48Wd0KwLPAc8AKknrkOW765LSt5pq+mZmVS/utkrchsC8wQtKwvO13pNH3vUhN9GNJT7MRESMl3QS8RBr5f3hhfpsjgPuATsA/ImLkzGTIQd/MzEqo+lE/Ip5o5kR3T+c9pwOnN7H97um9r6Uc9M3MrFREeafhdZ++mZlZSbimb2ZmpVPSir6DvpmZlU9Zm/cd9M3MrHRaMo1uR+Q+fTMzs5JwTd/MzMqnnBV9B30zMyufksZ8N++bmZmVhWv6ZmZWKmq/aXhnOQ76ZmZWOmUdve+gb2Zm5VPOmO8+fTMzs7JwTd/MzEqnpBV9B30zMyufsg7kc/O+mZlZSbimb2ZmJSOP3jczMysD4eZ9MzMz6+Ac9M3MzErCzftmZlY6ZW3ed9A3M7PS8UA+MzOzMijxgjvu0zczMysJ1/TNzKxUhKfhNTMzK4+SRn0375uZmZWEa/pmZlY6Hr1vZmZWEmUdve+gb2ZmpVPSmO8+fTMzs7Jw0Dczs/JRG/y05DTSNpJGSRot6YS2/RCt5+Z9MzMrnfYYyCepE3AxsCXwNvCcpIER8VLVT94M1/TNzMyqY11gdES8HhFfATcAO9UyQ67pA0OGDB4/1+x6s9b5aIVuwPhaZ6KD8zWuLl/f6qu3a7xMe51o6JDB93Xtom5tcKg5JT1feH15RFxeeN0deKvw+m1gvTY470xz0AciYuFa56E1JD0fEb1rnY+OzNe4unx9q8/XuHkRsU07naqpPoRop3M3yc37ZmZm1fE2sFTh9ZLAuzXKC+Cgb2ZmVi3PAStI6iGpC9AHGFjLDLl5vz5dPuMk9h35GleXr2/1+RrXWERMlnQEcB/QCfhHRIysZZ4UUdPuBTMzM2snbt43MzMrCQd9MzOzknDQNzMzKwkHfbPvQCrrAp1Wz4r3re/hcvFAvg5E0uwR8XWt89GRSdoUWAUgIi7O2xT+j9QmJHWNiM9rnY+OTNKWwIak7/9T8rbZImJKbXNm7cE1/Q5C0urAzyQtUuu8dFSStgEuAr4HHCPpKgAH/LYhqSdwtKSVap2Xjirfw+eSJo3ZS9K1AA745eGg33GsAuwIbCVNnVNakv/GbUBSb+AS4LiIOBNYE+gmqXttc9Yx5Pu0K7A4sLOkFQv7OuV/VfzXWidXDC4Hzo6IK0jfGd0lrdEonb8zOjD/ceucpK4AEXEjcBWwLbBdJfBXSvCS1qxZJjuGRYGRQGdJ3SLiE1KQ6iPpOElzS5qntlmsT5J+DJwcEUOBAaSFYnavBP6IaMhJ18uv3bLSSrng9DVpkpgFJK0YEZOBScDeks6VtLykhVzr79gc9OuYpJ8A50v6q6QNgXuBq4Gtge0lLZzT/QK4pfLaWq5QqLoL6AfsAWwq6WJSQaALsDtwM/D3Sq3UWiY3N99MKqh2jYjHgTuBBYA9KoE/38MPSlqidrmtT4VC1SvAtcDSwA6SrgQWAV4EVgPOBm6V1NmtKR2XB/LVKUnrkL4s9wd6AysB7wFnAD/O2/8JrAwcCuwaEcNrk9v6lAtVOwOfArcCzwKbA0eQmqG3jIgJOe3KwEcR8X6Nslt3JO0AnAj0BQ4nVeIPyPs2Iq07/gYpMO0J7Ol7uHVyoeoa4E3gxxHxeR6MujuwEbBLRLyW084DdI2ID2qWYas6B/06JWlvYL2I6Jtfn0Fq/nw0Ik6TtAsp2PcEdvSXZetMp1D1Z2BtUpC6BRgcEW/WKp/1Kreg3AKcFxEPSFoFOAk4vTI3eQ78BwKbAjv5Hm6dGRSq1iUVpN4CHomIYbXKp7UvL7hTv0YBB0naMSIGktZoHgwsLKl7RNwm6b/AGAelmbIScGdubn48F6p+RBrI9wdJi5IKBF0kveV+0NbJNc7dI+Kz3JT8AWmMxGaksRNExOOSPgFOjIiaLkdab3Kh6jDg9xHxrKSJwEmSVo2IkXlbF2AfoJOkkX7ctxwc9OtILp0vAHxICvDXAqdJ2p/0GNlWwN9JweiMiPh3rfLaATRXqOpWKFQ1AM854LecpLWB+YGPI2JwZaR4RIyXdD7QT9ITeVAfrt3PnBYWqp7I9/AYB/zycPN+nZC0NXA9cCWwG6mZ+VHgE2AhYHRETJJ0NPBZRFxWs8zWqUaFqqHAAcCRwBimLVSNjYgzapTNuiVpW+B80kC9w4HdIuLORoPG/kAKQlfXIo/1bjqFqimSNiEPRq0Uqqx8PHq/DuQR4dsBh0XE8cDewAbArkDn3Fw3SdKvSH2gj9Uut/UpF6ruJtWEbgZ+ATwJbAOcDGyfHxUbBXxUq3zWqzzQ8VzgkIg4GugDnCxpfkidzfn6/hc4UJJbIVspF6oGkL4rnpS0fW6FilywejTvX2M6h7EOzkF/Fidp0fyc8jhgW0lzR8QgUo1pRdKEPJWRt+uSRji/VLMM1yEXqqpL0mLA3MAJEfFoDuiPkgJ8Q0RUghIRcS7pHp5cuxzXHxeqrKUc9GdhkpYBLpTUA7gJ+AzYSFKXiHiB1FR3vKQ1ImIisH9EvFjDLNcdF6qqK9/DfwU+Bu6XpIiYnB91FKnbBNKz4wBExHvtn9P65UKVtYaD/iysMOr+/IgYA7xGqpFunoPTU8BdQGUmOA/QaAUXqqov38NdgHMj4oscgLrkwFSZPGpf4GZJ89Yyr/XIhSprLQf9WZCknpJ6AUTEHsBcko4ELgTGkmbcu1LSUaRBfe/mtA76reBCVfU0uod3B2aXdGJ+/VWuaQ4DjgEOBg6KiE9rluE65UKVtZaD/ixEyWLA/cBl+T8rpL665fPP+cBlwCDSqP1NImJsDbJbt1yoqp7p3MPnk+aQWKuQfD7SBDGHRMSIds5qXXOhymaWg/4sJI+1eQ84FZgA9JV0KKmZbn5gq4hoiIhXIuJ84BT3L7ecC1XVN4N7eCFgffhmpbwBwI8izQlvLeBClX1Xfk5/FqG0sMjcETE0N80dQOqnWwb4grR63o+APhFxb80y2gFI+jlp4ZxupHkP3gR+BgyKiIsL6TrF1BXebAZacQ/vFRH31CyjHcB07uGnIuLSXKjaHXg+Il6vXU5tVuPHNmosT54xO2ne8QZJr0fEHyV9CPQC/kha3AXgJ8Bhkh6JiC9rk+P6VAxIpJkMOzE1IC1HmpTndEljKoUqB/yWmYl7+FBJDwOT3GXSci28h8+UNDYXqm6qWWZtluWafo1JmjMivpQ0N+mRsEOBxYB9gb8BIyPipJx2H+DJiHijZhmuM4WA1A9oACoBaSemDUg7k/r0/0V6pMmFqhbyPVxdM3sP40KVNcFBv4YkbUVaFGME8EykNduR9HdgDmA4qQnv1Ii4r2YZrWMOSNXle7j6fA9bW/JAvhpRWuf6j8CDpEfBfiqpJ0BE/BK4lzQQZz1gV8+g1Xo5IA2QdBppQN7DEbEn8ApwGmlmvc2VpuAlIq7zl2XL+R6uPt/D1tb8n7AGJC1Imud9p4j4l6QlgdNJI/QBiIgblJa+HAG87Bm0WicHpD8A1wCLkALS6IgYFRG/lNSHtHxuJSA95Gvccr6Hq8/3sFWDm/drRNJ2wNnABhHxiaS7SUtfDgXeAfoDn0bEpNrlsj7lgDSebwekSyLimUK6LsAOpIDkRx9byfdw9fgetmpx0K8hpVWxLiA1g65MWrZ1QeAg4AXg6Ij4uHY5rF8OSO3D93D1+B62anDQrzFJW5Am2lg8It7P22YDFoyI8TXNXJ1zQGofvoerx/ewtTUH/VlA/o/9F2CzypemtQ0HpPbhe7h6fA9bW/Lo/VlAnkjjd8A9+T+ztZGIeJC0iM6/JS2at03xl2Xb8j1cPb6HrS25pj8LkTRPpCVcrY3liUxOAXpHxJRa56ej8j1cPb6HrS046FtpOCBZvfM9bN+Vg76ZmVlJuO/NzMysJBz0zczMSsJB38zMrCQc9M3MzErCQd9sJkhqkDRM0ouSbpbU9TscaxNJd+bfd5R0wnTSzi/pf2biHKdKOqal2xul6S9pt1aca1lJL7Y2j2ZWfQ76ZjPni4joFRGrAV+R1jj/hpJW//+KiIER8efpJJkfaHXQNzMDB32ztvA4sHyu4b4s6RJgCLCUpK0kPS1pSG4RmAfSsqmSXpH0BLBL5UCSDpB0Uf59UUm3Sxqef34I/Bn4fm5lOCenO1bSc5JekPSHwrFOlDRK0oNAzxl9CEkH5+MMl3Rro9aLLSQ9LulVSdvn9J0knVM49y+/64U0s+py0Df7DiR1BrYlrRkPKbheExFrAZ8BJwFbRMTawPPAbyTNCfQjLYm6EbBYM4e/AHg0ItYE1gZGAicAY3Irw7GStgJWANYFegHrSNpY0jpAH2AtUqHiBy34OLdFxA/y+V4mLepSsSzwY9J0sJflz3AQ8HFE/CAf/2BJPVpwHjOrkc61zoBZnZpL0rD8++PAlcASwJsRMShvXx9YBXhSEkAX4GlgJeCNiHgNQNJ1wCFNnGMzYD+AiGgAPpa0QKM0W+Wfofn1PKRCwLzA7RHxeT7HwBZ8ptUk/YnUhTAPcF9h30156tfXJL2eP8NWwBqF/v758rlfbcG5zKwGHPTNZs4XEdGruCEH9s+Km4AHImKvRul6AW01FaaAMyPi743O8euZOEd/YOeIGC7pAGCTwr7Gx4p87l9FRLFwgKRlW3leM2snbt43q55BwIaSlgeQ1FXSisArQA9J38/p9mrm/Q8Bh+X3dpL0PeBTUi2+4j7gwMJYge6SFgEeA34qaS5J85K6EmZkXmCcpNmBvRvt213SbDnPywGj8rkPy+mRtKKkuVtwHjOrEdf0zaokIj7MNeYBkubIm0+KiFclHQLcJWk88ASwWhOHOBK4XNJBQANwWEQ8LenJ/EjcPblff2Xg6dzSMBHYJyKGSLoRGAa8SeqCmJHfA8/k9COYtnAxCngUWBQ4NCK+lHQFqa9/iNLJPwR2btnVMbNa8II7ZmZmJeHmfTMzs5Jw0DczMysJB32zmSBpDkk3Shot6ZnmRqxLOjJP1Tsyj6hvvP8YSSGpW34tSRfk474gae1C2qUl3Z8nAHqprUbJS7pC0iqtfE+7T7Ur6bf5uoyStHUzaXrkv8dr+e/TJW8/QNKHeVKjYZJ+UXjPWflv9KKkPZs45oWSJlbvk5m1Hwd96zDyRDnt5SBgQkQsD5wHnNVEflYDDiZNnLMmsL2kFQr7lwK2BP6v8LZtSc+6r0B6dv/Swr5rgHMiYuV8zA/a4oNExC8i4qW2OFa15EJJH2BVYBvgEkmdmkh6FnBeRKwATGDaCYZuzJMa9YqIK/JxtyNNfNQLWA84Nj8lUTlvb9K8BWYdgoO+VZ2kOyQNzrXdQwrbt1Ganna4pIfytnkkXSVpRK7p7pq3Tyy8bzdJ/fPv/SWdK+lh4CxJ60p6StLQ/G/PnK6TpL8UjvsrSZtLur1w3C0l3dbCj7UTcHX+/RZg8zyCvWhlYFBEfB4Rk0mj339a2H8ecBzTPgO/E2lGv8iT/MwvafEc9DpHxAMAETGxMPHOaZJ2bJxBpcV0rs6tA2Ml7SLp7HwN7i08aveIpN75GvXPNd4Rko7K+5eX9GD+Ow3R1EcNK+dZVmmK3iH554d5++KSHtPUhYk2au4cLbzeN0TEpIh4AxhNKvgU8yHShEa35E1XM+OnCVYhzXo4OSI+A4aTChXkQsU5pL+RWYfgR/asPRwYER9Jmgt4TtKtpAJnP2DjiHhD0oI57e9JU7uuDqBvz0DXlBVJU9025FraxhExWdIWwBnArqRacw9grbxvQVJN8GJJC0fEh8DPgavyeW+k6fnqz42Ia4DuwFsA+XgfAwsB4wtpXwROl7QQ8AXwE9JUvOQg/U6eCKd4/G+Om72dty0J/DcXSnoADwInRERDRJw8nWvzfWBTUnB7Gtg1Io7LhZ3tgDsKaXsB3fMiQkiq1HCvB/4cEbcrTb87G7BI4X0fAFvmx/hWAAYAvYGfAfdFxOk5gHZt7hySjuXbcwMAPBYRffM1GFTYXrkuRQsB/80FrKbS7CppY9KMgUdFxFukIH+KpHNz/jYFKq0eRwADI2Lct8tzZvXJQd/aQ19JlRruUqSm64VJX+hvAETER3n/FqRmXPL2CS04/s15mlpIU8FenYNPALMXjntZJSBUzifpWmAfSVcBGzB12ttv9e020lQUmOb514h4WdJZwAOk5+eHA5OVFrI5kTSNbUuP25k0T/9apO6AG4EDSNP/Ts89EfG1pBFAJ+DevH0E6Rn7oteB5SRdCNwF3K80sU/3iLg9f6Yv4ZvZBytmBy5SmmmwgVQIA3gO+EduUbgjIoYpTeE7zTnycc8h1aqbM8PrPYM0/wIGRMQkSYeSWgE2i4j7Jf0AeIo0z8DTpL/REsDuTDsroVndc/O+VZWkTUgBd4O8kMtQYE7SF3RTk0Q0t724bc5G+4pT3/4ReDjXJHcopG3uuFcB+5Bmxbu5UihQGgQ2rImf/fL73iYVYCpjCeYDPmp88Ii4MiLWjoiN8/7XSLXvHsBwSWNJtfghkhYrHjdbEng3bx8aEa/nPN5B6ouekUk5H1OAr2PqxBxTaFTozwWsNYFHgMOBK2g6kDZ2FPB+fm9v0hoDRMRjwMbAO8C1kvZr5hyVlQKbut4X5HM0d12KxpO6Qzo3ThMR/4mISXl7P2Cdwuc+Pffzb5k/72ukwtXywOj8N+oqaXQLroXZLM01fau2+UgD3j6XtBJpERpINaqLJfWoNO/n2vf9pGbVX0Nq3s+B4n2lmedGkfrFP53O+d7Jvx9Q2H4/cKikRyrN+xHxUUS8K+ld0mp4W1YSt6CmPxDYP3+O3YB/FwLqNyQtEhEfSFqatNrdBvnzLFJIMxboHRHjlRbGOULSDaSBZR/n5uUPgAUKXRGbMbWr4Ezg2UptfGYpPUHwVUTcKmkM0D8iPpH0tqSdI+IOpZkFGw+gmw94OyKmSNq/sl/SMqQujH5K0/OuLenuxueAFtX0BwL/zM3wS5Bai54tJoiIUBrbsRtwA+nv8785L4tHxLicdEfSKoKVfvv5I+I/ktYA1gDuzwWrb1Y/lDQxD9o0q2uu6Vu13Qt0lvQCqRY+CNIUtaR+9tskDSc1VwP8iRTcXszbN83bTwDuBP4NjKN5ZwNnSnqSaYPTFaRm8RfycX9W2Hc98FYrR7BfCSyUa3+/yflD0hI5sFXcKuklUvPy4S3orrib1Mw+mlQj/R/4ZpW9Y4CHclO98n6A1YH3WpH35nQHHlFaPbA/8Nu8fV9SF80LpGbwxksBXwLsL2kQqWm/0vKyCTBM0lDSuIq/Tecc0xURI4GbSP3t95KuZQOApLtzczzA8aTli0eT+vgr3R99lQaSDgf6MrVAODvweP4bXU6awrgyJsCsw/E0vFZ6ki4iNZ3PqH98liTpvoho8rl1M7MiB30rNUmDSTXTLQt9vmZmHZKDvpmZWUm4T9/MzKwkHPTNzMxKwkHfzMysJBz0zczMSsJB38zMrCQc9M3MzEri/wGfpxswV1hotAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Confusion Matrix of classification Results\n",
    "plot_confusion_matrix(cm= confusion_matrix(y_test, pred_gs), \n",
    "                      normalize    = False,\n",
    "                      target_names = np.unique(y_test),\n",
    "                      title        = \"Confusion Matrix for Grid Searched XGB Classifier\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=StratifiedShuffleSplit(labels=['RW27R' 'RW09L' ... 'RW27R' 'RW27R'], n_iter=5, test_size=0.3, random_state=42),\n",
      "       error_score='raise',\n",
      "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
      "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
      "       n_jobs=1, nthread=None, num_class=4, objective='multi:softmax',\n",
      "       random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
      "       seed=None, silent=1, subsample=1),\n",
      "       fit_params=None, iid=False, n_jobs=4,\n",
      "       param_grid={'max_depth': [8, 16], 'n_estimators': [500, 1000], 'learning_rate': [0.01, 0.0075], 'subsample': [0.5, 0.75]},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
      "       scoring='accuracy', verbose=0)\n"
     ]
    }
   ],
   "source": [
    "#Print the results of the grid search\n",
    "print(gsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Best Score: 0.9400318979266347\n"
     ]
    }
   ],
   "source": [
    "# Print the best score of the grid search\n",
    "print('\\n Best Score:',gsearch.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.9400318979266347\n",
      "Best params: \n",
      "learning_rate: 0.0075\n",
      "max_depth: 16\n",
      "n_estimators: 1000\n",
      "subsample: 0.5\n"
     ]
    }
   ],
   "source": [
    "best_score = train_model_gs.best_score_\n",
    "best_params = train_model_gs.best_params_\n",
    "print(\"Best score: {}\".format(best_score))\n",
    "print(\"Best params: \")\n",
    "for param_name in sorted(best_params.keys()):\n",
    "    print('%s: %r' % (param_name, best_params[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted correctly: 62342/66282\n",
      "Accuracy for model 3: 94.06 %\n",
      "Error: 0.0594\n"
     ]
    }
   ],
   "source": [
    "# Calculate Obtained Error\n",
    "print('Predicted correctly: {0}/{1}'.format(correct_predictions_counter(y_test, pred_gs), len(pred_gs)))\n",
    "print(\"Accuracy for best grid searched model : %.2f\" % (accuracy_score(y_test, pred_gs) * 100),'%')\n",
    "print('Error: {0:.4f}'.format(1-(accuracy_score(y_test, pred_gs))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.940032 using {'learning_rate': 0.0075, 'max_depth': 16, 'n_estimators': 1000, 'subsample': 0.5} \n",
      "\n",
      "TestScore_Mean: 0.868068 and Std_Dev: (0.001662) with Parameters: {'learning_rate': 0.01, 'max_depth': 8, 'n_estimators': 500, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.858873 and Std_Dev: (0.000994) with Parameters: {'learning_rate': 0.01, 'max_depth': 8, 'n_estimators': 500, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.898203 and Std_Dev: (0.002377) with Parameters: {'learning_rate': 0.01, 'max_depth': 8, 'n_estimators': 1000, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.890344 and Std_Dev: (0.002291) with Parameters: {'learning_rate': 0.01, 'max_depth': 8, 'n_estimators': 1000, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.939691 and Std_Dev: (0.000972) with Parameters: {'learning_rate': 0.01, 'max_depth': 16, 'n_estimators': 500, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.939847 and Std_Dev: (0.000951) with Parameters: {'learning_rate': 0.01, 'max_depth': 16, 'n_estimators': 500, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.939799 and Std_Dev: (0.000995) with Parameters: {'learning_rate': 0.01, 'max_depth': 16, 'n_estimators': 1000, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.939825 and Std_Dev: (0.001042) with Parameters: {'learning_rate': 0.01, 'max_depth': 16, 'n_estimators': 1000, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.856278 and Std_Dev: (0.001861) with Parameters: {'learning_rate': 0.0075, 'max_depth': 8, 'n_estimators': 500, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.848127 and Std_Dev: (0.001007) with Parameters: {'learning_rate': 0.0075, 'max_depth': 8, 'n_estimators': 500, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.887284 and Std_Dev: (0.002110) with Parameters: {'learning_rate': 0.0075, 'max_depth': 8, 'n_estimators': 1000, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.877236 and Std_Dev: (0.002363) with Parameters: {'learning_rate': 0.0075, 'max_depth': 8, 'n_estimators': 1000, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.939256 and Std_Dev: (0.000756) with Parameters: {'learning_rate': 0.0075, 'max_depth': 16, 'n_estimators': 500, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.939079 and Std_Dev: (0.000967) with Parameters: {'learning_rate': 0.0075, 'max_depth': 16, 'n_estimators': 500, 'subsample': 0.75}\n",
      "TestScore_Mean: 0.940032 and Std_Dev: (0.000982) with Parameters: {'learning_rate': 0.0075, 'max_depth': 16, 'n_estimators': 1000, 'subsample': 0.5}\n",
      "TestScore_Mean: 0.939976 and Std_Dev: (0.001102) with Parameters: {'learning_rate': 0.0075, 'max_depth': 16, 'n_estimators': 1000, 'subsample': 0.75}\n"
     ]
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (train_model_gs.best_score_, train_model_gs.best_params_),'\\n')\n",
    "means = train_model_gs.cv_results_['mean_test_score']\n",
    "stds = train_model_gs.cv_results_['std_test_score']\n",
    "params = train_model_gs.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"TestScore_Mean: %f and Std_Dev: (%f) with Parameters: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notes:\n",
    "- Specifying `num_class` parameter for multiclass grid search seems to be [optional](https://stackoverflow.com/questions/39386966/multiclass-classification-in-xgboost-python) as there is no difference in accuracy score.\n",
    "- In grid-search using `scoring = f1` gives error as it is meant for binary classification. Need to use appropriate [scoring metric](http://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter) such as `f1_weighted` for multi-class classification.\n",
    "- Using `scoring = f1_samples` with non encoded raw **y** gives error as f-score is not meaningful outside multilabel classification. we have to use numerically encoded y for each class (eg: 1,2,3,4 ). `f1_weighted` and `accuracy` can handle raw *y* ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resources:\n",
    "- https://www.kaggle.com/dkim1992/grid-search-vs-random-search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}