evaluating-ml-model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Evaluating a machine learning model\n",
    "\n",
    "Three ways to evaluate Scikit-Learn models:\n",
    "1. Estimator `score` method\n",
    "2. The `scoring` parameter\n",
    "3. Problem-specific metric functions\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing essentials\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Evaluating a model with `score` method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For Classification Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading the data\n",
    "heart_disease = pd.read_csv('data/heart-disease.csv')\n",
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7868852459016393"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing RandomForestClassifier predictor class\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Setting a random seed\n",
    "np.random.seed(27)\n",
    "\n",
    "# Getting the data ready\n",
    "X = heart_disease.drop('target', axis = 1)\n",
    "y = heart_disease['target']\n",
    "\n",
    "# Splitting data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "# Instantiating and fitting the model\n",
    "model = RandomForestClassifier().fit(X_train, y_train)\n",
    "\n",
    "# Checking the accuracy\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 1], dtype=int64)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds = model.predict(X_test)\n",
    "y_preds[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7868852459016393"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(y_preds == y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Regression Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing Boston housing dataset\n",
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "boston;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston_df = pd.DataFrame(boston['data'], columns = boston['feature_names'])\n",
    "boston_df['target'] = pd.Series(boston['target'])\n",
    "boston_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9033142910210461"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setting a random seed\n",
    "np.random.seed(27)\n",
    "\n",
    "# Getting the data ready\n",
    "X = boston_df.drop('target', axis = 1)\n",
    "y = boston_df['target']\n",
    "\n",
    "# Splitting data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "# Instantiating and fitting the model\n",
    "model = RandomForestRegressor().fit(X_train, y_train)\n",
    "\n",
    "# Checking the accuracy\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`score()` uses different evaluation metrices depending on the model\n",
    "\n",
    "* For `Classification`: mean accuracy\n",
    "* For `Regression`: coefficient of determination or mean absolute error    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Evaluating a model using the `scoring` parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(27)\n",
    "\n",
    "X = heart_disease.drop('target', axis = 1)\n",
    "y = heart_disease['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "clf = RandomForestClassifier().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7868852459016393"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.80327869, 0.90163934, 0.80327869, 0.83333333, 0.78333333])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reason behind five values in the output:\n",
    "\n",
    "`cross_val_score()` creates n number of train and test sets (n can be specified by passing a number in cv parameter, by default it is 5) and evaluates the accuracy for each split. \n",
    "\n",
    "When using just one split, sometimes we may be lucky or unlucky and may get some accuracy that would show wrong results. So just to be sure this function creates n splits and evaluate the model on all these splits.\n",
    "\n",
    "Finally we can take mean of all the accuracies and get a more precise accuracy for the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7868852459016393, 0.8283060109289618)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(27)\n",
    "\n",
    "# Single training and test split score\n",
    "clf_single_score = clf.score(X_test, y_test)\n",
    "\n",
    "# Take the mean of 5-fold cross-validation score\n",
    "clf_cross_val_score = np.mean(cross_val_score(clf, X, y))\n",
    "\n",
    "# Comparing the two\n",
    "clf_single_score, clf_cross_val_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `scoring` parameter is by default set to `none` and can be set as per the need.\n",
    "So in case of classifier it is the mean accuracy by defalut"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.1 Classification model evaluation metrics\n",
    "1. Accuracy\n",
    "2. Area under ROC curve\n",
    "3. Confusion matrix\n",
    "4. Classification report\n",
    "\n",
    "**Accuracy** : How likely our model is going to predict a correct answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(27)\n",
    "\n",
    "X = heart_disease.drop('target', axis = 1)\n",
    "y = heart_disease['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "clf = RandomForestClassifier().fit(X_train, y_train)\n",
    "cross_val_score = cross_val_score(clf, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Heart disease classifier cross-Validated accuracy : 82.50%\n"
     ]
    }
   ],
   "source": [
    "print(f'The Heart disease classifier cross-Validated accuracy : {np.mean(cross_val_score) * 100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Area under the reciever operating characteristic curve (AUC/ROC)**\n",
    "* Area under curve (AUC)\n",
    "* ROC curve\n",
    "\n",
    "ROC curves are a comparison of model's true positive rate (tpr) versus model's false positive rate (fpr).\n",
    "\n",
    "* True positive : model predicts 1 when the truth is 1\n",
    "* False positive : model predicts 1 when the truth is 0\n",
    "* True negative : model predicts 0 when the truth is 0\n",
    "* Flase negative : model predicts 0 when the truth is 1\n",
    "\n",
    "**https://towardsdatascience.com/understanding-auc-roc-curve-68b2303cc9c5**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.62, 0.38],\n",
       "       [0.03, 0.97],\n",
       "       [0.05, 0.95],\n",
       "       [0.52, 0.48],\n",
       "       [0.02, 0.98],\n",
       "       [0.  , 1.  ],\n",
       "       [0.87, 0.13],\n",
       "       [0.03, 0.97],\n",
       "       [0.63, 0.37],\n",
       "       [0.88, 0.12]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "# Make predictions with probabilities\n",
    "y_probs = clf.predict_proba(X_test)\n",
    "\n",
    "y_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.38, 0.97, 0.95, 0.48, 0.98, 1.  , 0.13, 0.97, 0.37, 0.12])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probs_positive = y_probs[:, 1]\n",
    "y_probs_positive[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03703704, 0.03703704, 0.07407407,\n",
       "       0.07407407, 0.14814815, 0.14814815, 0.18518519, 0.22222222,\n",
       "       0.22222222, 0.22222222, 0.33333333, 0.40740741, 0.44444444,\n",
       "       0.59259259, 0.62962963, 0.62962963, 0.7037037 , 0.77777778,\n",
       "       0.88888889, 0.96296296, 1.        ])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate fpr, tpr and thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_probs_positive)\n",
    "\n",
    "# Check the false positive rate\n",
    "fpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxN9f/A8dfbMmYwdkp2sg7Zt0iEEhWV/EgLJSnqW74t2qRIKUWylBZayLdUsoUoCUmUnSSFKWrs24zZ3r8/zhld052ZizlzZ+a+n4/HPOaeez7nnPfnLud9zuec+/mIqmKMMSZ05Ql2AMYYY4LLEoExxoQ4SwTGGBPiLBEYY0yIs0RgjDEhzhKBMcaEOEsE2YyIbBaRtsGOI7sQkcdF5K0gbXuqiIwIxrYzm4j0FpFF57jsOX8mRWSFiDQ8l2XPlYjcLyIvZOU2czpLBOkQkd9FJFZEjovIPnfHUNjLbapqlKou9XIbKUSkgIg8LyK73Xr+IiIPi4hkxfb9xNNWRKJ9n1PVkaraz6PtibvT2CQiJ0QkWkQ+FpF6XmzvXInIMBH54HzWoarTVPXKALb1r+R3rp9JEbkWOKaqP7nTw0Qkwf0+HRaRlSLSMtUyxURkkvt9OykiG0Wkr5913ywia9x17RWRL0SktTt7MnCLiJRJJ7Yc8d5nFUsEGbtWVQsDDYCGwGNBjuesiUi+NGZ9DLQHOgORwK1Af+BVD2IQEclun7dXgf8A9wMlgBrALKBLZm8onffAc0Hc9gDg/VTP/c/9PpUCvsb5DAIgImHAYqAS0BIoCjwMvCAig33KDQbGAiOBC4CKwESgK4CqxgFfALelE1umvffBfG8zjaraXxp/wO9AB5/pF4F5PtMtgJXAYWA90NZnXglgCvAncAiY5TPvGmCdu9xK4JLU2wQuAmKBEj7zGgL7gfzu9B3AVnf9C4FKPmUVGAj8Avzmp27tgTigQqrnmwNJwMXu9FLgeWA1cAT4PFVM6b0GS4HngBVuXS4G+roxHwN2Ane7ZQu5ZZKB4+7fRcAw4AO3TGW3XrcDu93X4gmf7UUA77qvx1bgESA6jfe2ulvPZum8/1OBCcA8N97vgWo+818F9gBHgbXAZT7zhgEzgQ/c+f2AZsB37mu1FxgPhPksEwV8CRwE/gIeBzoB8UCC+5qsd8sWBd521/MHMALI687r477mY9x1jXCfW+7OF3fe3+57ugGoi3MQkOBu7zgwJ/X3AMjrxvWr+5qsJdVnyC0X5r6f5VO9Jh/4TNdx38/S7vSdbkyFUq3r/9x4irj1Pg7clMF3tzfw9Xm890uBfj7Tp18/f98v4HVgdKp1fA4Mdh9fBHwCxLjl7w/2/u2MWIMdQHb+S/UFKA9sBF51p8sBB3COpvMAHd3plA/1POB/QHEgP3C5+3wj98Pe3P1S3e5up4CfbX4F3OUTz0vA6+7jbsAOoDaQD3gSWJnqg/olTkKK8FO3F4Bv0qj3Lv7ZQS/F2dHUxdlZf8I/O+aMXoOlODvsKDfG/DhHXNVwdkaXAyeBRm75tqTaceM/EbyJs9OvD5wCavvWyX3Ny+Ps4NJKBAOAXRm8/1NxdqTN3PinATN85t8ClHTn/RfYB4T7xJ3gvk953Hgb4yTOfG5dtgIPuOUjcXbq/wXC3enmqV8Dn23PAt5w35MyOIk65T3rAyQC97nbiuDMRHAVzg68mPs+1AbK+tR5RDrfg4dxvgc13WXrAyX9vHZRwIl03ssw9/3aD+Rzn5sBvOtnXfnc+lyFkxgTU5ZJ571rBBw8j/d+KRkngtPfL6ANzkGBuPOL4yTCi9z3fy0w1K13VZyDoKuCvY9L+ctup+rZ0SwROYbzJv8NPO0+fwswX1Xnq2qyqn4JrAE6i0hZ4GpggKoeUtUEVf3GXe4u4A1V/V5Vk1T1XZydWQs/254O9AKnaQXo6T4HcDfwvKpuVdVEnNPkBiJSyWf551X1oKrG+ll3KZwdjz973fkp3lfVTap6AngK6CEiedN7DXyWnaqqm1U10X0d5qnqr+r4BlgEXJZGHGl5RlVjVXU9zllIfff5HsBI9zWPBsals46S6dTf16equtp9jafhNBECoKofqOoBt24vAwVwdpApvlPVWe5rE6uqa1V1lVv+d5wd+eVu2WuAfar6sqrGqeoxVf3eX0AicgHO5+sBVT2hqn/jHOH39Cn2p6q+5m4r9fufgJNoauHsuLaqaiCvBThnNk+q6s/ue7heVQ/4KVcM54whtR4ichhnJ3kX0N19bSGNz6Q7f787vySw32eZtBzDOXvwJ9D3PiO+369vcZJDyme5O877/yfQFOfg6FlVjVfVnTgHMz39rjUILBFkrJuqRuIcrdbinx1kJeAm96LXYffD3RooC1TAORo55Gd9lYD/plquAs6RQ2ozgZYichHOEYfifOBS1vOqzzoO4hyhlfNZfk869drvxupPWXe+v/XswjmyL0X6r4HfGETkahFZJSIH3fKdOTPpBGKfz+OTQMoF/ItSbS+9+h8g7foHsi1E5L8islVEjrh1KcqZdUld9xoiMte9EHoUJ3mnlK+A09wSiEo478Fen9f9DZwzA7/b9qWqX+E0S00A/hKRySJSJMBtBxrnIZxkk9pHqloMp21/E85ZUgq/n0m3Db6UO/8AUCqAdvlInGYvfwJ97zNy+jVW5zRgBu6BG3AzzoEDOO/XRam+J4/jvAbZgiWCALlHr1OB0e5Te3COlIv5/BVS1RfceSVEpJifVe0Bnku1XEFV/dDPNg/jHDH3wPlgfeh+4FLWc3eq9USo6krfVaRTpcVAcxGp4PukiDTD+bJ/5fO0b5mKOEeU+zN4Df4Vg4gUwGlaGg1c4O4Q5uMksIziDcRenCYhf3GntgQoLyJNzmVDInIZ8CjOe1PcrcsR/qkL/Ls+k4BtQHVVLYKzM0gpvwenycyf1OvZg3MWWcrndS+iqlHpLHPmClXHqWpjnCacGjhNPhkul0Gcvn7BOZEt52+mqu7HOasd5p5Bg/OZvFpECqUqfiNOfVfhXGOJw2lyS09tnLNFfwJ5708ABX2mL/RTJvVr9SHQ3T0rb47zWQfnNfst1fckUlU7k01YIjg7Y4GOItIA5yLgtSJylYjkFZFw9/bH8u5p9hfARBEpLiL5RaSNu443gQEi0ty9k6aQiHQREX9HT+A0Bd2G82WY7vP868BjIhIFICJFReSmQCuiqotxvhCfiEiUW4cWOEcxk1T1F5/it4hIHREpCDwLzFTVpPRegzQ2G4bTfBIDJIrI1YDvLY1/ASVFJK1T+ox8hPOaFHd3QIPSKujWbyLwoRtzmBt/TxEZEsC2InHaqmOAfCIyFOdiZkbLHAWOi0gt4B6feXOBC0XkAXFu640UkebuvL+Ayil3Xbmfr0XAyyJSRETyiEg1EbmcAIhIU/fzlx9nhxeHc/E0ZVtV01n8LWC4iFR3P7+XiEjJ1IVUNQFnx55mTKq6Decmh0fcp94HooGPRaSy+725CqeJb5iqHlHVIzht7RNEpJuIFHTLXS0iL/qs/nKc76C/7Qby3q8DbnDXfzHOhex0qXObbIz7Gi10D+TAuX5zVEQeFZEI97tSV0SaZrTOrGKJ4CyoagzwHvCUqu7BuV3tcZw3fw/OUVXKa3orzpHzNpxrCw+461iD0zY6Huf0eQfOhai0zMa5y+Evt008JZbPgFHADLeZYRNOu/HZuBHnFr4FOHdifIBzJ8p9qcq9j3M2tA/nQub9bgwZvQZnUNVj7rIf4dT9Zrd+KfO34RxV7XRPof01l6XnWZwdyW84O6GZOEeSabmff5pIDuM0eVwPzAlgWwtxdjTbcZrL4ki/KQrgIZw6H8M5IPhfygz3tekIXIvzOv8CtHNnp9xieUBEfnQf34aTWLfgvJYzCby5o4i7/UNu7Af450z3baCO+/rP8rPsKzjv3yKcpPY2zsVSf97A+R6k5yWgv4iUUdVTOHfM7cG5Q+uou70nVPWllAVU9RVgMM4NEimfu0E4F9ARkXCcJsd309luRu/9GJy7p/5y1zPNzzr8+dCtw+mDNveg6Vqc60u/4ZxNv0Xa1zCyXMoVbmP8EpGlOHd6BOXXvedDRO4BeqpqQEfKJvOJyHLgPvdoOau2eR/OLa2PZFjYAM5tWcbkCm5bc1WcduTqOLdijg9qUCFOVVtnXCrTt/laVm8zp7NEYHKTMJzmiCo4p/szcNqCjTHpsKYhY4wJcXax2BhjQlyOaxoqVaqUVq5cOdhhGGNMjrJ27dr9qlra37wclwgqV67MmjVrgh2GMcbkKCKyK6151jRkjDEhzhKBMcaEOEsExhgT4iwRGGNMiLNEYIwxIc6zRCAi74jI3yKyKY35IiLjRGSHiGwQkUZexWKMMSZtXp4RTMUZVi4tV+P0B1MdZ6zUSR7GYowxJg2e/Y5AVZeJSOV0inQF3nMHWlklIsVEpOxZDJlnjAnUgTXwx+yMy5lsKSEBfvsjDzWaXgplr8x4gbMUzB+UlePM/tuj3ef+lQhEpD/OWQMVK1bMkuCMyVU2PwfRszhzADWTE/z0+0XcMfkm/j5amO3zvqZQLksE/j6RfnvAU9XJwGSAJk2aWC95xpwtTYLiDeHqHzMua7KFuLhEnnlmJS+99AOlSkUw8a0OFGpZw5NtBTMRRHPmmLLlgT+DFIsxxmQr3brNYuHC3+nbty4vv9yW4sXDPdtWMG8fnQ3c5t491AI4YtcHjDGh7NixeOLiEgEYMqQZixZ15513OnmaBMDDMwIR+RBoC5QSkWjgaSA/gKq+DszHGVd0B3AS6OtVLMYYk90tXPgb/fsv4pZb6vDcc5fRtm3WXQ/18q6hXhnMV2CgV9s3GdjzKexMb2xvk6scWA0RFwU7CuPHwYOxDB68lHff3UytWiXo0qVqlseQ47qhNplk51TY9yUUqRXsSExWiCgL5a8PdhQmlSVLdtG79zwOHIjjiSda8OSTLQgPz/rdsiWCUFaktt1FYkwQlSlTkCpVirJgQXcaNCgTtDisryFjjMkiqsrUqZu4//4lANSrV5qVK28OahIASwTGGJMlfvvtMFddNZO+fRewbl0MsbEJAIgE/0d+1jRkjDEeSkpKZsKEdTz22DLy5BEmTuzA3XfXJ0+e4CeAFJYIjDHGQ/v3xzJ06Aouv7wCr7/ekYoViwQ7pH+xRGCMMZksISGJadO2ctttUVxwQSF+/PFWqlQpmi2agfyxRGCMMZlo7dp93HHHQjZsiKFs2UJcdVUVqlYtFuyw0mUXi40xJhPExiYwZMgymjefRkzMST77rCtXXVUl2GEFxM4IjDEmE3Tr9jmLFv1Ov371eOmlyylWzNv+gTKTJQJjjDlHR4+eIiwsL+Hh+Xj88eY88khT2revFOywzpo1DRljzDmYP38ndetO5dlnvwPg8ssr5MgkAJYIjDHmrOzff5Jbb51Ply6fEhkZxnXXVQt2SOfNmoaMMSZAX375O717z+PQoVMMHdqSxx9vToECOX83mvNrYIwxWaRs2ULUqFGCSZM6UK9e6WCHk2msacgYY9Kgqrz11gYGDlwMQN26pfn22565KgmAJQJjjPFr587DdOjwMXfdtYgtWw5kq07iMpslAmOM8ZGUlMyYMWuoW3cqP/ywjzfe6MiSJT2IiMgf7NA8Y9cIjDHGx/79sTzzzHe0b1+RSZM6Ur58ZLBD8pwlAmNMyIuPT+KDD7bQp09dLrigEOvW3UalSkVyZTOQP5YIjDEh7Ycf9nLHHQvZtGk/5ctHcuWVlalcuWiww8pSdo3AGBOSTp5M4KGHltKixXQOHYpj9uzrufLKysEOKyjsjCBUJccHOwJjgqpr11ksXryL/v0v4cUXL6do0QLBDilo7Iwg1OxfBV91hL0LIaJssKMxJksdOXKKuLhEAJ56qgVffdWDN964MqSTAFgiCB0Hf4Kl18KilnBoPTR6BVrPDHZUxmSZuXN/JSpqCs88sxKANm0q0K5dxSBHlT1Y01Bud2QLbHga9syE/MWg/kiocR/kLxzsyIzJEjExJ/nPf77iww+3Ua9eKW64oXqwQ8p2LBHkVsd+hY3D4PdpkK8w1B0KtR6EsOw9ZJ4xmWnRIqeTuCNHTvHMM5cyZEhzwsLyBjusbMcSQW5zYjdsGg47p0CeMKj9sPMXXirYkRmT5cqVK0zt2iWZNKkDUVH2HUiLJYLsLPEErHsMEo8FXj76c+dx9YEQ9RhEXOhdfMZkM8nJTidxP/30N5MmdSQqqhTLlvUMdljZniWC7OzgWtj+GoSXgTwBjH8qAlX7QNSTUKiC5+EZk53s2HGIu+5axNKle2jXrgKxsQm5un+gzGSJICe49EO48IpgR2FMtpSUlMzYsWt56qkV5M+fhzffvJI776wXMt1DZAZPbx8VkU4i8rOI7BCRIX7mFxWROSKyXkQ2i0hfL+MxxuQ++/fHMmLEKjp2rMSWLX3p1+8SSwJnybNEICJ5gQnA1UAdoJeI1ElVbCCwRVXrA22Bl0UkzKuYjDG5w6lTibz55gaSk/V0J3GzZnWjXLnc31OoF7w8I2gG7FDVnaoaD8wAuqYqo0CkOOm7MHAQSPQwJmNMDvf993tp3Ph9+vdfxOLFuwCoVKmonQWcBy8TQTlgj890tPucr/FAbeBPYCPwH1VNTr0iEekvImtEZE1MTIxX8RpjsrETJ+IZPPhrWracxpEj8cybd0PIdhKX2bxMBP7Ss6aavgpYB1wENADGi0iRfy2kOllVm6hqk9Klc9dYocaYwHTr9jljxqxlwID6bN7ch86dqwY7pFzDy0QQDfjew1ge58jfV1/gU3XsAH4DankYkzEmBzl8OO70WMFDh7bkm2/+j4kTO1KkSGh3EpfZvEwEPwDVRaSKewG4JzA7VZndQHsAEbkAqAns9DAmY0wOMXv2DqKipvLMM98BcNll5WnTxn4f4wXPEoGqJgKDgIXAVuAjVd0sIgNEZIBbbDhwqYhsBJYAj6rqfq9iMsZkf3//fYKePefQtessSpWKoHv3GsEOKdfz9AdlqjofmJ/qudd9Hv8JXOllDMaYnGPBgt/o3Xsex48nMHx4Kx59tBn581sncV6zXxYbY7KNChUiqVevFBMndqBOHeskLqvYwDTGmKBJTlYmTVrH3XcvAiAqqhRLl/a0JJDFLBEYY4Ji+/aDtG37P+69dzG//Xbk9BCSJutZIjDGZKnExGRGjfqeSy55l40bY5gypRMLF3YnPNxaqoPFXnljTJY6cCCWUaN+oHPnqkyY0J6yZW3Y1GCzRJAZEo7DcQ9+/nDs18xfpzFBcOpUIlOnbuauuy7hggsKsX79bVSo8K9OBEyQWCLIDMu7w96F3q0/b4R36zbGY9999yd33rmArVsPUq1aMTp0qGRJIJuxRJAZ4g9B8QZQ96nMX3e+SCjVPPPXa4zHjh+P58knlzNu3I9UqBDJggU30qFDpWCHZfywRJBZwi+ECjcEOwpjso1u3WaxZMluBg1qyMiRlxEZaUONZFeWCIwxmebQoTjCw/MSEZGfYcMuZdiwS2ndunywwzIZCPj2UREp5GUgxpic7dNPt1OnzhSGDVsJQOvW5S0J5BAZnhGIyKXAWzgjiFUUkfrA3ap6r9fBBdW+r+BkdGBl42IgrIS38RiTTe3bd4JBgxbzySe/0KBBGXr2tJ7kc5pAmobG4AwgMxtAVdeLSBtPowq2xBPwVQf+PY5OOspc5lk4xmRXX3yxk96953PyZAIjR17GQw81sU7icqCArhGo6p5U44EmeRNONpGcCChEPQHV7ghsmYLWT7oJPZUqFaFhwzJMmNCeWrVKBjscc44CSQR73OYhdQeYuR9nfIHcr0BJKGzD4RmTIjlZmTjxJ9avj+HNN6+iTp1SLFnSI9hhmfMUyMXiAcBAnIHno3HGFs7d1weMMf/y888HadNmBvfd9xV79hyzTuJykUDOCGqqam/fJ0SkFbDCm5CMMdlJQkISo0ev4ZlnVlKwYH6mTu3EbbdFkaq52ORggZwRvBbgc8aYXOjQoTheeukHrr22Glu29OX22+taEshl0jwjEJGWwKVAaREZ7DOrCGC3BRiTi8XFJfLOOxsZMKABZcoUYsOG2ylfPjLYYRmPpNc0FIbz24F8gO8n4CjQ3cugjDHBs3x5NHfeuZDt2w9Ro0YJOnSoZEkgl0szEajqN8A3IjJVVXdlYUzGmCA4diyexx5bxoQJ66hcuQiLFnW3TuJCRCAXi0+KyEtAFBCe8qSqXuFZVMaYLNet2yy+/no3//lPI0aMaE3hwtZJXKgIJBFMA/4HXINzK+ntQIyXQRljssbBg7GEh+ejYMH8DB/eCpHWtGx5UbDDMlkskLuGSqrq20CCqn6jqncALTyOyxjjsZkzf6Z27X86ibv00nKWBEJUIIkgwf2/V0S6iEhDwLoUNCaH2rv3ODfc8Dk33TSHChUi6d27drBDMkEWSNPQCBEpCvwX5/cDRYAHPI3KGOOJefN+5ZZb5hMXl8SoUW0YPLgJ+fIF3Bu9yaUyTASqOtd9eARoB6d/WWyMyWGqVi1G06YXMn58e2rUsK7TjSO9H5TlBXrg9DG0QFU3icg1wONABNAwa0I0xpyrpKRkxo//iQ0bYnj77U7Url2SRYtuCnZYJptJ74zgbaACsBoYJyK7gJbAEFWdlRXBGWPO3ZYt++nXbxHfffcnnTtXIS4ukfBwG53W/Ft6n4omwCWqmiwi4cB+4GJV3Zc1oWWyQxtgaWdIis24rCa7D6w/FZPzxMcn8eKLqxk+fBWRkWF88EFnbr65tvUPZNKUXiKIV3X2iKoaJyLbzzYJiEgn4FWcvoneUtUX/JRpC4wF8gP7VfXys9lGwI79DLF/QKVezjgDGZH8UOEGT0IxxkuHD8cxZsxarr/+YsaNu4IyZWy4cZO+9BJBLRHZ4D4WoJo7LYCq6iXprdi9xjAB6IgzjsEPIjJbVbf4lCkGTAQ6qepuESlzHnUJTNQTUCzK880Yk5ViYxN4++2N3HtvQ8qUKcTGjX246KLCwQ7L5BDpJYLzvbm4GbBDVXcCiMgMoCuwxafMzcCnqrobQFX/Ps9tGhNyli3bQ79+i/jll0PUrl2S9u0rWRIwZyXNG4hVdVd6fwGsuxywx2c62n3OVw2guIgsFZG1InKbvxWJSH8RWSMia2JirHcLYwCOHj3Fvfd+yeWX/4/ExGQWL76J9u2tkzhz9ry8hcDflSn1s/3GQHucW1K/E5FVqrr9jIVUJwOTAZo0aZJ6HcaEpG7dZrF06R4efLAxw4e3olAh6yTOnBsvE0E0zu2nKcoDf/ops19VTwAnRGQZUB/YjjHmX/bvP0nBgvkpWDA/zz13GSLQooX1D2TOT0C/LReRCBGpeZbr/gGoLiJVRCQM6AnMTlXmc+AyEcknIgWB5sDWs9yOMbmeqjJjxjZq157C0087w4W3bHmRJQGTKTJMBCJyLbAOWOBONxCR1Dv0f1HVRGAQsBBn5/6Rqm4WkQEiMsAts9Vd7wacH669paqbzrUyxuRGf/xxjG7dZtGr11yqVCnKbbfZXW8mcwXSNDQM5w6gpQCquk5EKgeyclWdD8xP9dzrqaZfAl4KZH3GhJq5c3+ld+95JCQkM3r05TzwQGPy5rVO4kzmCiQRJKrqEftVojFZ7+KLi3HppRfx2mvtufji4sEOx+RSgRxabBKRm4G8IlJdRF4DVnoclzEhKSkpmTFj1tCnzxcA1KpVki++6G5JwHgqkERwH854xaeA6TjdUdt4BMZkss2b99Oq1YcMHryU/ftjiYtLDHZIJkQE0jRUU1WfAJ7wOhhjQlF8fBIvvPA9I0asomjRAkyf3oWePWtZJ3EmywSSCF4RkbLAx8AMVd3scUzGhJTDh+MYN+4nbrqpJmPHtqN06YLBDsmEmAybhlS1HdAWiAEmi8hGEXnS68CMyc1Onkzg1VfXkpSU7HYSdzvTpnWxJGCCIqD70FR1n6qOAwbg/KZgqKdRGZOLff31burVm8oDD3zN0qVOd1xly1oncSZ4AvlBWW0RGSYim4DxOHcMlfc8MmNymSNHTnH33Yu44oqPEBG+/rqHdRJnsoVArhFMAT4ErlTV1H0FGWMC1K3bLJYti+bhh5sybNilFCyYP9ghGQMEkAhUtUVWBGJMbhQTc5JChZxO4p5//jLy5hWaNi0b7LCMOUOaTUMi8pH7f6OIbPD52+gzcpkxxg9VZfr0rWd0EteixUWWBEy2lN4ZwX/c/9dkRSDG5BbR0ce4554vmTt3J82bl6VPn7rBDsmYdKU3Qtle9+G9fkYnuzdrwjMmZ5k9ewd16kzhq692M2ZMO1as6EVUVKlgh2VMugK5fbSjn+euzuxAjMkNatQoTuvW5di4sY/1FGpyjDSbhkTkHpwj/6qprglEAiu8DsyYnCAxMZmxY9eyYUMM773XmVq1SjJ//o3BDsuYs5LeNYLpwBfA88AQn+ePqepBT6MyJgfYsCGGO+9cwJo1f9G168XExSUSHu7l6K/GeCO9T62q6u8iMjD1DBEpYcnAhKpTpxIZOfJ7Ro78nhIlwvnoo2vp3r2GdRJncqyMzgiuAdYCCvh+yhWo6mFcxmRbR4/GM3HiOnr1qsWYMe0oWTIi2CEZc17STASqeo37v0rWhWNM9nTiRDyTJ2/g/vsbUbp0QTZt6sMFFxQKdljGZIpA+hpqJSKF3Me3iMgrIlLR+9CMyR6WLNlFvXrvMnjwUr75JhrAkoDJVQK5t20ScFJE6gOPALuA9z2Nyphs4PDhOPr1W0iHDh+TL18evvnm/7jiCjsGMrlPoIPXq4h0BV5V1bdF5HavAzMm2K6//nO+/TaaRx9txtNPtyQiwjqJM7lTIIngmIg8BtwKXCYieQH7Rphc6a+/TlC4cH4KFQrjhRfakC+f0LjxhcEOyxhPBdI09H84A9ffoar7gHLAS55GZUwWU1Xef38zdepM4emnVwLQvHlZSwImJAQyVOU+YBpQVESuAeJU9T3PIzMmi+zefZQuXT7lttu+oGbNEtx5Z71gh2RMlgrkrqEewGrgJqAH8L2IdPc6MGOywuef7yAqagrLlkUzbtwVfPttT2rXLhnssIzJUoFcI3gCaKDBlnYAABvmSURBVKqqfwOISGlgMTDTy8CM8ZKqIiLUqlWCtm0r8Npr7alcuWiwwzImKAK5RpAnJQm4DgS4nDHZTmJiMqNGfc+tt84HoGbNEsyZc4MlARPSAjkjWCAiC3HGLQbn4vF870Iyxhvr1//NHXcs5Mcf/+L666tbJ3HGuAIZs/hhEbkBaI3T39BkVf3M88iMySRxcYmMGLGKUaNWU7JkODNnXseNN9YIdljGZBvpjUdQHRgNVAM2Ag+p6h9ZFZgxmeXYsXjeeGM9vXvX5pVX2lKihHUSZ4yv9Nr63wHmAjfi9ED62tmuXEQ6icjPIrJDRIakU66piCTZ3Ugmsxw/Hs/o0T+QlJRM6dIF2bKlL1OnXm1JwBg/0msailTVN93HP4vIj2ezYvcXyBNwhrqMBn4QkdmqusVPuVHAwrNZvzFpWbTod/r3X8Tu3Udp3PgC2rWrSOnSBYMdljHZVnpnBOEi0lBEGolIIyAi1XRGmgE7VHWnqsYDM4CufsrdB3wC/O1nnjEBO3gwlr59v+Cqq2YSHp6Pb7/tRbt21kmcMRlJ74xgL/CKz/Q+n2kFrshg3eWAPT7T0UBz3wIiUg643l1X07RWJCL9gf4AFSvaF9v4d/31n7NixR88/nhznnqqpd0RZEyA0huYpt15rtvfuH2aanos8KiqJqU3zJ+qTgYmAzRp0iT1OkwI27fvBJGRTidxL710OWFheWnQoEywwzImR/Hyh2HRQAWf6fLAn6nKNAFmiMjvQHdgooh08zAmk0uoKlOnbqJOnSkMHboCgGbNyloSMOYceHnu/ANQXUSqAH8APYGbfQv4DoMpIlOBuao6y8OYTC7w++9HuPvuL1m06Hdaty5H//71gx2SMTmaZ4lAVRNFZBDO3UB5gXdUdbOIDHDnv+7Vtk3u9dlnv3DrrfMRgfHj23PPPQ3IkyftZkVjTMYyTATiNN73Bqqq6rPueMUXqurqjJZV1fmk6o4irQSgqn0CitiEpJRO4qKiStKhQyVefbUdlSpZ/0DGZIZArhFMBFoCvdzpYzi/DzDGcwkJSYwcuYrevecBUKNGCWbN6mZJwJhMFEgiaK6qA4E4AFU9BIR5GpUxwI8//kWzZtN44onlJCUpp04lBjskY3KlQBJBgvvrX4XT4xEkexqVCWmxsQk89tgymjX7gH37TvDZZ1353/+upUAB+12AMV4I5Js1DvgMKCMiz+Hc5vmkp1GZkHbiRAJvv72R22+PYvTothQvHh7skIzJ1QLphnqaiKwF2uP8SKybqm71PDITUo4di2fSpHX8979NKFXK6SSuVCnrH8iYrBDIXUMVgZPAHN/nVHW3l4GZ0LFgwW/cffci9uw5RrNmF9K2bUVLAsZkoUCahubhXB8QIByoAvwMRHkYlwkBBw7EMnjw17z33hZq1y7BihU307LlRcEOy5iQE0jTUD3fabfn0bs9i8iEjBtu+JyVK//kqada8MQTLexisDFBctbfPFX9UUTS7CnUmPTs3XucyMgwChcOY/Rop5O4+vWtfyBjgimQawSDfSbzAI2AGM8iMrmSqjJlyiYGD17KHXfU5ZVX2tG0adlgh2WMIbAzgkifx4k41ww+8SYckxvt3HmYu+/+ksWLd9GmTXkGDLBO4ozJTtJNBO4PyQqr6sNZFI/JZT79dDu33jqfvHnzMGlSB/r3r2+dxBmTzaSZCEQkn9uDaCDDUhpzhpRO4urVK02nTlUYO7YdFSoUCXZYxhg/0jsjWI1zPWCdiMwGPgZOpMxU1U89js3kQPHxSbz44mo2bz7A9OldqF69OJ984m+oamNMdhHINYISwAGccYVTfk+ggCUCc4Y1a/Zx550L2bAhhp49axEfn2S3hBqTA6T3LS3j3jG0iX8SQAobN9icFhubwNNPr+Tll9dw4YWF+Pzzblx33cXBDssYE6D0EkFeoDCBDUJvQtiJEwlMnbqJO++sx4svtqFYMeskzpicJL1EsFdVn82ySEyOcvToKSZOXMfDDzelVKmCbN16ByVLRgQ7LGPMOUgvEdg9fsavefN+ZcCAxfz553FatChL27YVLQkYk4OlNzBN+yyLwuQIMTEn6d17Htdc8xlFi4axcuXNtG1bMdhhGWPOU5pnBKp6MCsDMdnfjTfOZtWqPxk27FIee6w5YWF5gx2SMSYT2L19Jl1//HGMokULULhwGGPGtKVAgbzUrVs62GEZYzJRIGMWmxCkqrz55gbq1JnC0KErAGjc+EJLAsbkQnZGYP7l118Pc9ddC/n66z20a1eBgQMbBjskY4yHLBGYM8yc+TO33fYF+fPnYfLkK+nXrx4idgOZMbmZJQID/NNJXP36ZejSpSpjxrSjfPnIjBc0xuR4do0gxMXHJ/HMMyvp2XMuqkr16sX5+OPrLAkYE0IsEYSw1av30rjx+wwbtpJ8+fIQH58U7JCMMUFgiSAEnTyZwEMPLaVly+kcOhTHnDnXM21aF+sp1JgQZd/8EBQbm8gHH2yhf/9LGDWqDUWKFAh2SMaYIPL0jEBEOonIzyKyQ0SG+JnfW0Q2uH8rRcQGs/XIkSOneO65VSQmJlOyZARbt97BpEkdLQkYY7xLBO54xxOAq4E6QC8RqZOq2G/A5ap6CTAcmOxVPKFszpxfT/8wbPnyaACKF7euoo0xDi/PCJoBO1R1p6rGAzOAM8YsVNWVqnrInVwFlPcwnpATE3OSXr3mct11n1GyZDjff9/bOokzxvyLl9cIygF7fKajgebplL8T+MLfDBHpD/QHqFjRdmSBSukk7tlnW/Hoo82skzhjjF9eJoKARzYTkXY4iaC1v/mqOhm32ahJkyY2Olo6oqOPUayY00nc2LHtKFAgL1FRpYIdljEmG/OyaSgaqOAzXR74M3UhEbkEeAvoqqoHPIwnV0tOVt54Yz116kzhqaecTuIaNbrAkoAxJkNenhH8AFQXkSrAH0BP4GbfAiJSEfgUuFVVt3sYS672yy+HuOuuhXzzTTTt21fkvvuskzhjTOA8SwSqmigig4CFQF7gHVXdLCID3PmvA0OBksBEt2OzRFVt4lVMudHHHzudxBUokJe3376Kvn3rWidxxpiz4ukPylR1PjA/1XOv+zzuB/TzMobcKqWTuIYNy9C1azVeeaUdF11UONhhGWNyIOtiIoc5dSqRoUOX06PHHFSViy8uzowZ11oSMMacM0sEOciqVX/SqNH7DB++ioiIfNZJnDEmU1giyAFOnIjnwQe/5tJLp3PsWDzz59/Ae+91tk7ijDGZwvYkOUBcXBIzZmzj3nsb8PzzbYiMDAt2SMaYXMQSQTZ1+HAcr732E4891tztJK4vxYpZ/0DGmMxnTUPZ0KxZv1CnzhSeeWYlK1f+AWBJwBjjGTsjyEb++usE9923hI8/3k79+qWZM+d6Gje+MNhhGZNlEhISiI6OJi4uLtih5Fjh4eGUL1+e/PnzB7yMJYJspHv32axevY8RI1rzyCNNyZ/fOokzoSU6OprIyEgqV65sP4w8B6rKgQMHiI6OpkqVKgEvZ4kgyHbvPkrx4uFERoYxbtwVFCiQlzp1rH8gE5ri4uIsCZwHEaFkyZLExMSc1XJ2jSBIkpOVCRN+IirKGTAGoGHDCywJmJBnSeD8nMvrZ2cEQfDzzwfp128hy5f/QceOlfjPfxoFOyRjTAizM4Is9tFH26hf/102bdrPlCmdWLiwO5UrFw12WMYYV968eWnQoAF169bl2muv5fDhw6fnbd68mSuuuIIaNWpQvXp1hg8fjuo/Q6R88cUXNGnShNq1a1OrVi0eeuihYFThrFkiyCIpH5bGjS/khhuqs3XrHfTpYz2FGpPdREREsG7dOjZt2kSJEiWYMGECALGxsVx33XUMGTKE7du3s379elauXMnEiRMB2LRpE4MGDeKDDz5g69atbNq0iapVqwazKgGzpiGPxcUlMnz4d2zbdpCZM6+jWrViTJ9+TbDDMib7W/sAHFqXuess3gAajw24eMuWLdmwYQMA06dPp1WrVlx55ZUAFCxYkPHjx9O2bVsGDhzIiy++yBNPPEGtWrUAyJcvH/fee2/mxu8ROyPw0MqVf9Cw4XuMHPk9kZFh1kmcMTlIUlISS5Ys4brrrgOcZqHGjRufUaZatWocP36co0ePsmnTpn/NzynsjMADx4/H8/jj3zJ+/E9UqBDJggU3ctVVgd/Ta4zhrI7cM1NsbCwNGjTg999/p3HjxnTs2BH4ZwwQf3J6E6+dEXggPj6JmTO3M3BgQzZt6mtJwJgcJOUawa5du4iPjz99jSAqKoo1a9acUXbnzp0ULlyYyMhIoqKiWLt2bTBCPm+WCDLJwYOxDBu2gsTEZEqUiGDr1jt47bX21lOoMTlU0aJFGTduHKNHjyYhIYHevXuzfPlyFi9eDDhnDvfffz+PPPIIAA8//DAjR45k+3Zn+PXk5GReeeWVoMV/NiwRZIJPPtlOnTpTGDFi1elO4ooWLRDkqIwx56thw4bUr1+fGTNmEBERweeff86IESOoWbMm9erVo2nTpgwaNAiASy65hLFjx9KrVy9q165N3bp12bt3b5BrEBi7RnAe9u49zqBBS/j0019o2LAMCxZ0p0GDMsEOyxhzHo4fP37G9Jw5c04/rlevHkuXLk1z2WuuuYZrrsl5dwVaIjgPPXrM4Ycf9vHCC5fx3/82JV8+O8EyxuQ8lgjO0q5dRyhRIoLIyDBee609ERH5qFmzRLDDMsaYc2aHsAFKTlZee+1HoqKm8tRTywFo0KCMJQFjTI5nZwQB2LbtAP36LWLFij/o1KkyDz6YM380Yowx/lgiyMCMGdu4/fYvKFw4P++9dzW33FInx/94xBhjfFkiSENyspInj9C06YXcdFMNXn65LRdcUCjYYRljTKazawSpxMYmMGTIMm688XNUlWrVivHBB10sCRgTIv766y9uvvlmqlatSuPGjWnZsiWfffaZp9tcs2YN999/v6fbSI8lAh/ffhtNgwbvMWrUakqWjCAhITnYIRljspCq0q1bN9q0acPOnTtZu3YtM2bMIDo62tPtNmnShHHjxnm6jfRY0xBw7Fg8Q4YsY+LEdVSpUpQvv7yJDh0qBTssY0Je27Yz/vVcjx41uffehpw8mUDnzp/8a36fPnXp06cu+/efpHv32WfMW7q0Z7rb++qrrwgLC2PAgAGnn6tUqRL33XcfU6dOZc2aNYwfPx5wfjz20EMP0bZtWxYtWsTTTz/NqVOnqFatGlOmTKFw4cIMGTKE2bNnky9fPq688kpGjx7Nxx9/zDPPPEPevHkpWrQoy5YtY+nSpYwePZq5c+cybNgwdu/ezc6dO9m9ezcPPPDA6bOF4cOHM23aNCpUqECpUqVo3Lhxpgx+Y4kASEhIYtasHTzwQGNGjGhFoULWP5AxoWjz5s00anR2Q8fu37+fESNGsHjxYgoVKsSoUaN45ZVXGDRoEJ999hnbtm1DRE6PdPbss8+ycOFCypUrd8boZ762bdvG119/zbFjx6hZsyb33HMP69ev55NPPuGnn34iMTGRRo0aZVq31yGbCA4ciOXVV9cydOillCgRwbZtd1gHccZkM+kdwRcsmD/d+aVKFczwDCAjAwcOZPny5YSFhTFw4EC/ZVatWsWWLVto1aoVAPHx8bRs2ZIiRYoQHh5Ov3796NKly+muJ1q1akWfPn3o0aMHN9xwg991dunShQIFClCgQAHKlCnDX3/9xfLly+natSsREREAXHvttedVN1+eXiMQkU4i8rOI7BCRIX7mi4iMc+dvEBHPR3FXVT7++Gfq1JnC88+v5rvv/gSwJGCMISoqih9//PH09IQJE1iyZAkxMTHky5eP5OR/rhvGxcUBzj6lY8eOrFu3jnXr1rFlyxbefvtt8uXLx+rVq7nxxhuZNWsWnTp1AuD1119nxIgR7NmzhwYNGnDgwIF/xVGgwD+dVubNm5fExMQzxkbObJ4lAhHJC0wArgbqAL1EpE6qYlcD1d2//sAkr+IB+PNQEW649Sd69JhDhQqRrFlzC5ddVt7LTRpjcpArrriCuLg4Jk36Z1d08uRJACpXrsy6detITk5mz549rF69GoAWLVqwYsUKduzYcbr89u3bOX78OEeOHKFz586MHTuWdeucYTd//fVXmjdvzrPPPkupUqXYs2dPQLG1bt2aOXPmEBcXx/Hjx5k3b16m1dvLpqFmwA5V3QkgIjOArsAWnzJdgffUSXWrRKSYiJRVVU/6bu0x7hbW7o7hxRfb8OCDTayTOGPMGUSEWbNm8eCDD/Liiy9SunTp0+3+rVq1okqVKtSrV4+6deuevpZQunRppk6dSq9evTh16hQAI0aMIDIykq5duxIXF4eqMmbMGMAZt+CXX35BVWnfvj3169fnm2++yTC2pk2bct1111G/fn0qVapEkyZNKFq0aObU26vTDRHpDnRS1X7u9K1Ac1Ud5FNmLvCCqi53p5cAj6rqmlTr6o9zxkDFihUb79q16+wDivmO9XNfJ6Lho9RokPrExBiTHWzdupXatWsHO4xs6/jx4xQuXJiTJ0/Spk0bJk+e7Pfitr/XUUTWqmoTf+v18ozAXz8MqbNOIGVQ1cnAZIAmTZqcW+Yq3ZL6fVue06LGGJMd9O/fny1bthAXF8ftt99+1nc4pcXLRBANVPCZLg/8eQ5ljDHGANOnT/dkvV42kv8AVBeRKiISBvQEZqcqMxu4zb17qAVwxKvrA8aYnMHLu2NCwbm8fp6dEahqoogMAhYCeYF3VHWziAxw578OzAc6AzuAk0Bfr+IxxmR/4eHhHDhwgJIlS1ovv+dAVTlw4ADh4eFntZxnF4u90qRJE12zZk3GBY0xOU5CQgLR0dGn79E3Zy88PJzy5cuTP3/+M54P1sViY4w5K/nz56dKlSrBDiPk2I30xhgT4iwRGGNMiLNEYIwxIS7HXSwWkRjgHH5aDEApYH8mhpMTWJ1Dg9U5NJxPnSupaml/M3JcIjgfIrImravmuZXVOTRYnUODV3W2piFjjAlxlgiMMSbEhVoimBzsAILA6hwarM6hwZM6h9Q1AmOMMf8WamcExhhjUrFEYIwxIS5XJgIR6SQiP4vIDhEZ4me+iMg4d/4GEcmc0R2CKIA693brukFEVopI/WDEmZkyqrNPuaYikuSOmpejBVJnEWkrIutEZLOIZDwGYjYXwGe7qIjMEZH1bp1zdC/GIvKOiPwtIpvSmJ/5+y9VzVV/OF1e/wpUBcKA9UCdVGU6A1/gjJDWAvg+2HFnQZ0vBYq7j68OhTr7lPsKp8vz7sGOOwve52I444JXdKfLBDvuLKjz48Ao93Fp4CAQFuzYz6PObYBGwKY05mf6/is3nhE0A3ao6k5VjQdmAF1TlekKvKeOVUAxESmb1YFmogzrrKorVfWQO7kKZzS4nCyQ9xngPuAT4O+sDM4jgdT5ZuBTVd0NoKo5vd6B1FmBSHEGMCiMkwgSszbMzKOqy3DqkJZM33/lxkRQDtjjMx3tPne2ZXKSs63PnThHFDlZhnUWkXLA9cDrWRiXlwJ5n2sAxUVkqYisFZHbsiw6bwRS5/FAbZxhbjcC/1HV5KwJLygyff+VG8cj8DesUep7ZAMpk5MEXB8RaYeTCFp7GpH3AqnzWOBRVU3KJaNdBVLnfEBjoD0QAXwnIqtUdbvXwXkkkDpfBawDrgCqAV+KyLeqetTr4IIk0/dfuTERRAMVfKbL4xwpnG2ZnCSg+ojIJcBbwNWqeiCLYvNKIHVuAsxwk0ApoLOIJKrqrKwJMdMF+tner6ongBMisgyoD+TURBBInfsCL6jTgL5DRH4DagGrsybELJfp+6/c2DT0A1BdRKqISBjQE5idqsxs4Db36nsL4Iiq7s3qQDNRhnUWkYrAp8CtOfjo0FeGdVbVKqpaWVUrAzOBe3NwEoDAPtufA5eJSD4RKQg0B7ZmcZyZKZA678Y5A0JELgBqAjuzNMqslen7r1x3RqCqiSIyCFiIc8fBO6q6WUQGuPNfx7mDpDOwAziJc0SRYwVY56FASWCie4ScqDm458YA65yrBFJnVd0qIguADUAy8Jaq+r0NMScI8H0eDkwVkY04zSaPqmqO7Z5aRD4E2gKlRCQaeBrID97tv6yLCWOMCXG5sWnIGGPMWbBEYIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRGCyJbe30HU+f5XTKXs8E7Y3VUR+c7f1o4i0PId1vCUiddzHj6eat/J8Y3TXk/K6bHJ73CyWQfkGItI5M7Ztci+7fdRkSyJyXFULZ3bZdNYxFZirqjNF5EpgtKpech7rO++YMlqviLwLbFfV59Ip3wdooqqDMjsWk3vYGYHJEUSksIgscY/WN4rIv3oaFZGyIrLM54j5Mvf5K0XkO3fZj0Ukox30MuBid9nB7ro2icgD7nOFRGSe2//9JhH5P/f5pSLSREReACLcOKa58467///ne4TunoncKCJ5ReQlEflBnD7m7w7gZfkOt7MxEWkmzjgTP7n/a7q/xH0W+D83lv9zY3/H3c5P/l5HE4KC3fe2/dmfvz8gCacjsXXAZzi/gi/iziuF86vKlDPa4+7//wJPuI/zApFu2WVAIff5R4GhfrY3FXe8AuAm4Huczts2AoVwujfeDDQEbgTe9Fm2qPt/Kc7R9+mYfMqkxHg98K77OAynF8kIoD/wpPt8AWANUMVPnMd96vcx0MmdLgLkcx93AD5xH/cBxvssPxK4xX1cDKcPokLBfr/tL7h/ua6LCZNrxKpqg5QJEckPjBSRNjhdJ5QDLgD2+SzzA/COW3aWqq4TkcuBOsAKt2uNMJwjaX9eEpEngRicHlrbA5+p04EbIvIpcBmwABgtIqNwmpO+PYt6fQGME5ECQCdgmarGus1Rl8g/o6gVBaoDv6VaPkJE1gGVgbXAlz7l3xWR6jg9UeZPY/tXAteJyEPudDhQkZzdH5E5T5YITE7RG2f0qcaqmiAiv+PsxE5T1WVuougCvC8iLwGHgC9VtVcA23hYVWemTIhIB3+FVHW7iDTG6e/leRFZpKrPBlIJVY0TkaU4XSf/H/BhyuaA+1R1YQariFXVBiJSFJgLDATG4fS387WqXu9eWF+axvIC3KiqPwcSrwkNdo3A5BRFgb/dJNAOqJS6gIhUcsu8CbyNM9zfKqCViKS0+RcUkRoBbnMZ0M1dphBOs863InIRcFJVPwBGu9tJLcE9M/FnBk5HYZfhdKaG+/+elGVEpIa7Tb9U9QhwP/CQu0xR4A93dh+fosdwmshSLATuE/f0SEQaprUNEzosEZicYhrQRETW4JwdbPNTpi2wTkR+wmnHf1VVY3B2jB+KyAacxFArkA2q6o841w5W41wzeEtVfwLqAavdJpongBF+Fp8MbEi5WJzKIpxxaRerM/wiOONEbAF+FGfQ8jfI4IzdjWU9TtfML+KcnazAuX6Q4mugTsrFYpwzh/xubJvcaRPi7PZRY4wJcXZGYIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRGCMMSHOEoExxoQ4SwTGGBPi/h9M+UYTiAnsPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a function for plotting ROC curve\n",
    "def plot_roc_curve(fpr, tpr):\n",
    "    \"\"\"\n",
    "    Plots a ROC curve given false positive rate (fpr)\n",
    "    and true positive rate (tpr) of a model.\n",
    "    \"\"\"\n",
    "    # Plot ROC curve\n",
    "    plt.plot(fpr, tpr, color='orange', label = 'ROC')\n",
    "    \n",
    "    # Plot line with no predictive power (baseline)\n",
    "    plt.plot([0,1], [0,1], color='darkblue', linestyle='--', label='Guessing')\n",
    "    \n",
    "    # Customize the label\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "plot_roc_curve(fpr, tpr)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The more the area under ROC curve the better is the classifying power of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9308278867102396"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_test, y_probs_positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.roc_curve.RocCurveDisplay at 0x1f9533bb070>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV5b3H8c9PFgm7rEWRghJlCRAlFXCFUtyBq1YRrRbbvqgL2t57XeuOdWnVqqgtxQXUUuC6VVTU4lZUXNgCBFCMyhJFRQTKKtvv/jGT9BCyTEzmhGS+79frvHJm5pmZ35PA+Z1nnpnnMXdHRESSa5/qDkBERKqXEoGISMIpEYiIJJwSgYhIwikRiIgkXN3qDqCiWrVq5R07dqzuMEREapQ5c+Z84+6tS9pW4xJBx44dmT17dnWHISJSo5jZ8tK26dKQiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwsWWCMzsUTP72szyStluZjbGzPLNbIGZHR5XLCIiUro4WwQTgBPL2H4SkBm+RgJ/iTEWEREpRWzPEbj7DDPrWEaRocDjHoyD/Z6ZNTezdu6+Kq6YRJJqQcE6Xl38VXWHIZWU07EFxx5S4jNhlVKdD5QdAKxMWS4I1+2RCMxsJEGrgQ4dOqQlOJHa5IHX8/nn4q8wq+5IpDIuPO7gWpcISvonWeIsOe4+DhgHkJOTo5l0RCpolzvd92/Ki5cdU92hyF6oOu8aKgAOTFluD3xRTbGIiCRWdSaCqcD54d1DfYH16h8QEUm/2C4NmdkkoD/QyswKgBuBegDuPhaYBpwM5AObgQviikVEREoX511Dw8vZ7sAlcZ1fyvZy3iqemvN5dYchaZK7cj1tm+5b3WHIXqrGDUMtVeOpOQW89fE3HNy6cXWHImnQpsm+nND9B9UdhuyllAgSrHObxrqLREQ01pCISNIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSQUJt2+nVHYKI7CWUCBJm7oq1/Ozh95mxdDVtmuxb3eGIyF6gbnUHIOmR9/l67pm+lNc+/JqWjepz3Sld+VnfH1Z3WCKyF1AiqOU+/moD97y6lGkLv6Rpg7pcccKhjDiyI4321Z9eRAL6NKillq/ZxL2vfsw/cj+nUf26XDYwk18e3YlmGfWqOzQR2csoEdQyn6/bwv2vfcyTcwqoV8cYeexB/PrYg2nRqH51hyYieyklgr3Y5m07+OPLH7Hxux2Rym/ZtpPpi78C4Ly+P+TiAQfTpkmDOEMUkVpAiWAvtrBgPRNmLqNV4/rsW7dOpH3O6N2eS3/cmf2bZ8QcnYjUFkoENcCYsw/jyM6tqjsMEamlYn2OwMxONLOPzCzfzK4uYXszM3vezOab2SIzuyDOeEREZE+xJQIzqwM8CJwEdAOGm1m3YsUuARa7ey+gP3C3malXU0QkjeJsERwB5Lv7p+6+DZgMDC1WxoEmZmZAY+BbIFrPqIiIVIk4E8EBwMqU5YJwXaoHgK7AF8BC4Dfuvqv4gcxspJnNNrPZq1evjiteEZFEijMRWAnrio90dgKQC+wPZAMPmFnTPXZyH+fuOe6e07p166qPVEQkweJMBAXAgSnL7Qm++ae6AHjGA/nAZ0CXGGMSEZFi4kwEs4BMM+sUdgCfDUwtVmYFMBDAzNoChwKfxhiTiIgUE9tzBO6+w8xGAa8AdYBH3X2RmV0Ybh8L3AJMMLOFBJeSrnL3b+KKSURE9hTrA2XuPg2YVmzd2JT3XwDHxxmDiIiUTRPTiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJyGoa4Cm77bwYpvN1f5cZfHcEwRkeKUCKrARRPnMmNpfGMg7Vsv2qQ0IiLfhxJBFVi/ZTvd2jXlsoGdq/zYjfaty2EHNq/y44qIFFIiqCKtm+zLiVntqjsMEZEKU2exiEjCRU4EZtYozkBERKR6lHtpyMyOBB4mmEGsg5n1An7t7hfHHVx1mpn/DavWb41U9ttN39E8o17MEYmIxCNKH8E9BBPITAVw9/lmdmysUVWzzdt2cO4j7+PFp9Epw486togvIBGRGEXqLHb3lcG0wkV2xhPO3mHHLscdRg3ozFk5B5a/A9CueYOYoxIRiUeURLAyvDzk4QQzlwFL4g1r79C8YT06tGxY3WGIiMQqSmfxhcAlBBPPFxDMLVyr+wdERJIkSovgUHc/N3WFmR0FvBNPSCIikk5RWgT3R1wnIiI1UKktAjPrBxwJtDaz/0nZ1JRgDmIREakFyro0VJ/g2YG6QJOU9f8GfhpnUCIikj6lJgJ3/xfwLzOb4O7L0xiTiIikUZTO4s1mdifQHSi6Wd7dfxxbVCIikjZROosnAh8CnYCbgWXArBhjEhGRNIqSCFq6+yPAdnf/l7v/Augbc1wiIpImUS4NbQ9/rjKzU4AvgPbxhSQiIukUJRH83syaAf9L8PxAU+C3sUYlIiJpU24icPcXwrfrgQFQ9GSxiIjUAmU9UFYHOItgjKGX3T3PzE4FfgdkAIelJ0QREYlTWS2CR4ADgQ+AMWa2HOgHXO3u/0hHcCIiEr+yEkEO0NPdd5lZA+AboLO7f5me0KrWklX/5oLxs9i6o/ypFHbtCmakKTYHg4hIrVRWItjm7rsA3H2rmS2taBIwsxOB+wjGJnrY3e8ooUx/4F6gHvCNux9XkXNE9enqTXz5760M6bU/+zUsf1rJunX24cSsH8QRiojIXqWsRNDFzBaE7w04OFw2wN29Z1kHDvsYHgQGEcxjMMvMprr74pQyzYE/Aye6+woza1OJukQy6sedOaRtk/ILiogkRFmJoGslj30EkO/unwKY2WRgKLA4pcw5wDPuvgLA3b+u5DlFRKSCyhp0rrIDzR0ArExZLgD6FCtzCFDPzN4kGOH0Pnd/vPiBzGwkMBKgQ4cOlQxLRERSRRli4vsqqafViy3XBXoDpwAnANeb2SF77OQ+zt1z3D2ndevWVR+piEiCRXmy+PsqILj9tFB7guEpipf5xt03AZvMbAbQC1gaY1wiIpIiUovAzDLM7NAKHnsWkGlmncysPnA2MLVYmeeAY8ysrpk1JLh0tKSC5xERkUooNxGY2WAgF3g5XM42s+If6Htw9x3AKOAVgg/3/3P3RWZ2oZldGJZZEh53AcGDaw+7e973rYyIiFRclEtDNxHcAfQmgLvnmlnHKAd392nAtGLrxhZbvhO4M8rxRESk6kW5NLTD3dfHHomIiFSLKC2CPDM7B6hjZpnAZcDMeMMSEZF0idIiuJRgvuLvgL8TDEet+QhERGqJKC2CQ939WuDauIMREZH0i9Ii+JOZfWhmt5hZ99gjEhGRtCo3Ebj7AKA/sBoYZ2YLzey6uAMTEZH0iPRAmbt/6e5jgAsJnim4IdaoREQkbaI8UNbVzG4yszzgAYI7htrHHpmIiKRFlM7i8cAk4Hh3Lz5WkIiI1HDlJgJ375uOQEREpHqUmgjM7P/c/SwzW8juw0dHmqFMRERqhrJaBL8Jf56ajkBERKR6lNpZ7O6rwrcXu/vy1BdwcXrCExGRuEW5fXRQCetOqupARESkepTVR3ARwTf/g8xsQcqmJsA7cQcmIiLpUVYfwd+Bl4DbgatT1m9w929jjUpERNKmrETg7r7MzC4pvsHMWigZiIjUDuW1CE4F5hDcPmop2xw4KMa4REQkTUpNBO5+avizU/rCERGRdIsy1tBRZtYofP8zM/uTmXWIPzQREUmHKLeP/gXYbGa9gCuB5cATsUYlIiJpE3XyegeGAve5+30Et5CKiEgtEGX00Q1mdg1wHnCMmdUB6sUbloiIpEuUFsEwgonrf+HuXwIHAHfGGpWIiKRNlKkqvwQmAs3M7FRgq7s/HntkIiKSFlHuGjoL+AA4EzgLeN/Mfhp3YCIikh5R+giuBX7k7l8DmFlr4FXgqTgDExGR9IjSR7BPYRIIrYm4n4iI1ABRWgQvm9krBPMWQ9B5PC2+kEREJJ2izFl8hZmdDhxNMN7QOHd/NvbIREQkLcqajyATuAs4GFgIXO7un6crMBERSY+yrvU/CrwAnEEwAun9FT24mZ1oZh+ZWb6ZXV1GuR+Z2U7djSQikn5lXRpq4u4Phe8/MrO5FTlw+ATygwRTXRYAs8xsqrsvLqHcH4BXKnJ8ERGpGmUlggZmdhj/mYcgI3XZ3ctLDEcA+e7+KYCZTSYYr2hxsXKXAk8DP6pg7CIiUgXKSgSrgD+lLH+ZsuzAj8s59gHAypTlAqBPagEzOwA4LTxWqYnAzEYCIwE6dNAI2CIiVamsiWkGVPLYVsI6L7Z8L3CVu+80K6l4USzjgHEAOTk5xY8hIiKVEOU5gu+rADgwZbk98EWxMjnA5DAJtAJONrMd7v6PGOMSEZEUcSaCWUCmmXUCPgfOBs5JLZA6DaaZTQBeUBIQEUmv2BKBu+8ws1EEdwPVAR5190VmdmG4fWxc5xYRkejKTQQWXLc5FzjI3UeH8xX/wN0/KG9fd59GseEoSksA7j4iUsQiIlKlogwe92egHzA8XN5A8HyAiIjUAlEuDfVx98PNbB6Au681s/oxxyUiImkSpUWwPXz616FoPoJdsUYlIiJpEyURjAGeBdqY2a3A28BtsUYlIiJpE2UY6olmNgcYSPCQ2H+5+5LYIxMRkbSIctdQB2Az8HzqOndfEWdgIiKSHlE6i18k6B8woAHQCfgI6B5jXCIikiZRLg31SF02s8OBX8cWkYiIpFWFJ6EPh5/WkNEiIrVElD6C/0lZ3Ac4HFgdW0QiIpJWUfoImqS830HQZ/B0POGIiEi6lZkIwgfJGrv7FWmKR0RE0qzUPgIzq+vuOwkuBYmISC1VVovgA4IkkGtmU4EngU2FG939mZhjExGRNIjSR9ACWEMwr3Dh8wQOKBGIiNQCZSWCNuEdQ3n8JwEU0rzBIiK1RFmJoA7QmGiT0IuISA1VViJY5e6j0xaJiIhUi7KeLC6pJSAiIrVMWYlgYNqiEBGRalNqInD3b9MZiIiIVI8KDzonIiK1ixKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCRcrInAzE40s4/MLN/Mri5h+7lmtiB8zTSzXnHGIyIie4otEYTzHT8InAR0A4abWbdixT4DjnP3nsAtwLi44hERkZLF2SI4Ash390/dfRswGRiaWsDdZ7r72nDxPaB9jPGIiEgJ4kwEBwArU5YLwnWl+SXwUkkbzGykmc02s9mrV6+uwhBFRCTORBB5ZjMzG0CQCK4qabu7j3P3HHfPad26dRWGKCIiUSav/74KgANTltsDXxQvZGY9gYeBk9x9TYzxiIhICeJsEcwCMs2sk5nVB84GpqYWMLMOwDPAee6+NMZYRESkFLG1CNx9h5mNAl4B6gCPuvsiM7sw3D4WuAFoCfzZzAB2uHtOXDGJiMie4rw0hLtPA6YVWzc25f2vgF/FGYOIiJRNTxaLiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJFzd6g5Aaq/t27dTUFDA1q1bqzsUkcRo0KAB7du3p169epH3USKQ2BQUFNCkSRM6duyImVV3OCK1nruzZs0aCgoK6NSpU+T9dGlIYrN161ZatmypJCCSJmZGy5YtK9wKVyKQWCkJiKTX9/k/p0QgIpJwSgRSq9WpU4fs7GyysrIYPHgw69atq5LjTpgwgVGjRlXJsTp27EiPHj3Izs4mOzubmTNnVslxi8vNzWXatGm7rXvppZfIycmha9eudOnShcsvvxyAm266ibvuuqvKzn3kkUcWvb/iiivo3r07V1xxBWPHjuXxxx+v1LHnzZvHr371q93WDR06lH79+u22bsSIETz11FO7rWvcuHHR+6VLl3LyySfTuXNnunbtyllnncVXX31Vqdi+/fZbBg0aRGZmJoMGDWLt2rUllrvvvvvIysqie/fu3HvvvUXrr7/+enr27El2djbHH388X3zxBQALFy5kxIgRlYotlRKB1GoZGRnk5uaSl5dHixYtePDBB6s7pBK98cYb5Obmkpubu9uHZll27NhRoXMUTwR5eXmMGjWKv/3tbyxZsoS8vDwOOuigCh0zqtTk9te//pW5c+dy5513cuGFF3L++edHPk5Jdb7tttu49NJLi5bXrVvH3LlzWbduHZ999lmk427dupVTTjmFiy66iPz8fJYsWcJFF13E6tWrI8dWkjvuuIOBAwfy8ccfM3DgQO644449yuTl5fHQQw/xwQcfMH/+fF544QU+/vhjIEiaCxYsIDc3l1NPPZXRo0cD0KNHDwoKClixYkWl4iuku4YkLW5+fhGLv/h3lR6z2/5NuXFw98jl+/Xrx4IFCwD44IMP+O1vf8uWLVvIyMhg/PjxHHrooUyYMIGpU6eyefNmPvnkE0477TT++Mc/AjB+/Hhuv/122rVrxyGHHMK+++4LwPLly/nFL37B6tWrad26NePHj6dDhw6MGDGCjIwMPvzwQ5YvX8748eN57LHHePfdd+nTpw8TJkwoNdayjtmiRQvmzZvH4YcfzsUXX8wll1zC6tWradiwIQ899BBdunThySef5Oabb6ZOnTo0a9aMV199lRtuuIEtW7bw9ttvc8011/Diiy9y7bXX0qVLFwDq1q3LxRdfvEcsDz30EOPGjWPbtm107tyZJ554goYNG+5xjhkzZrBo0SIuuOACtm3bxq5du3j66afJzMykcePGbNy4kSFDhrBp0yb69OnDNddcw5IlS2jcuDGXX345n3zySYl1KV7nu+++uyi2DRs2sGDBAnr16lW07umnn2bw4MG0bduWyZMnc80115T7b+Pvf/87/fr1Y/DgwUXrBgwYUO5+5Xnuued48803Afj5z39O//79+cMf/rBbmSVLltC3b18aNmwIwHHHHcezzz7LlVdeSdOmTYvKbdq0abfr/4MHD2by5MlceeWVlY5TLQJJhJ07d/Laa68xZMgQALp06cKMGTOYN28eo0eP5ne/+11R2dzcXKZMmcLChQuZMmUKK1euZNWqVdx444288847TJ8+ncWLFxeVHzVqFOeffz4LFizg3HPP5bLLLivatnbtWl5//XXuueceBg8ezH//93+zaNEiFi5cSG5ublG5AQMGkJ2dTZ8+fco95tKlS3n11Ve5++67GTlyJPfffz9z5szhrrvuKvogHz16NK+88grz589n6tSp1K9fn9GjRzNs2DByc3MZNmwYeXl59O7du9zf3emnn86sWbOYP38+Xbt25ZFHHinxHABjx47lN7/5Dbm5ucyePZv27dvvdqypU6cWtdKGDRu227bS6lK8zqlmz55NVlbWbusmTZrE8OHDGT58OJMmTSq3fkDk38WGDRuKLuEVf6X+myj01Vdf0a5dOwDatWvH119/vUeZrKwsZsyYwZo1a9i8eTPTpk1j5cqVRduvvfZaDjzwQCZOnFjUIgDIycnhrbfeilS/8qhFIGlRkW/uVWnLli1kZ2ezbNkyevfuzaBBgwBYv349P//5z/n4448xM7Zv3160z8CBA2nWrBkA3bp1Y/ny5XzzzTf079+f1q1bAzBs2DCWLl0KwLvvvsszzzwDwHnnnbfbN7TBgwdjZvTo0YO2bdvSo0cPALp3786yZcvIzs4GgktDrVq1KtqvrGOeeeaZ1KlTh40bNzJz5kzOPPPMom3fffcdAEcddRQjRozgrLPO4vTTT6/U7zAvL4/rrruOdevWsXHjRk444YRSz9GvXz9uvfVWCgoKOP3008nMzIx0jrLqklrn4latWlX0N4Hggzc/P5+jjz4aM6Nu3brk5eWRlZVV4t00Fb3DpkmTJrsl8KrQtWtXrrrqKgYNGkTjxo3p1asXdev+56P51ltv5dZbb+X222/ngQce4OabbwagTZs2RX0GlRVri8DMTjSzj8ws38yuLmG7mdmYcPsCMzs8zngkeQq/fS5fvpxt27YV9RFcf/31DBgwgLy8PJ5//vnd7rsuvOQDQWdz4XXpqB8aqeUKj7XPPvvsdtx99tmnQtf4U4/ZqFEjAHbt2kXz5s2L+hZyc3NZsmQJEHwz//3vf8/KlSvJzs5mzZo1exyze/fuzJkzp9xzjxgxggceeICFCxdy4403Fv2uSjrHOeecU/St/4QTTuD111+PVL+y6pJa5+IyMjJ2+9tNmTKFtWvX0qlTJzp27MiyZcuYPHkyAC1bttyts/bbb78tSr5RfxcVbRG0bduWVatWAUHSatOmTYnH/eUvf8ncuXOZMWMGLVq0KDGBnnPOOTz99NNFy1u3biUjI6PcmKOILRGYWR3gQeAkoBsw3My6FSt2EpAZvkYCf4krHkm2Zs2aMWbMGO666y62b9/O+vXrOeCAAwDKvFZfqE+fPrz55pusWbOG7du38+STTxZtO/LII4s+bCZOnMjRRx9d6XijHLNp06Z06tSpKBZ3Z/78+QB88skn9OnTh9GjR9OqVStWrlxJkyZN2LBhQ9H+V1xxBbfddltRy2bXrl386U9/2uM8GzZsoF27dmzfvp2JEycWrS/pHJ9++ikHHXQQl112GUOGDCnqkylPWXUpS9euXcnPzy9anjRpEi+//DLLli1j2bJlzJkzp+j32L9/f6ZMmcK2bduA4O9e2A9wzjnnMHPmTF588cWiY7388sssXLhwt/MVtghKenXrVvzjDYYMGcJjjz0GwGOPPcbQoUNLrEfhJaMVK1bwzDPPMHz4cICiTmMILqsV9udAcLms+GWx7yvOFsERQL67f+ru24DJQPHfwlDgcQ+8BzQ3s3YxxiQJdthhh9GrV6+iDrZrrrmGo446ip07d5a7b7t27bjpppvo168fP/nJTzj88P80XseMGcP48ePp2bMnTzzxBPfdd1+lY416zIkTJ/LII4/Qq1cvunfvznPPPQcEH/I9evQgKyuLY489ll69ejFgwAAWL15MdnY2U6ZMoWfPntx7770MHz6crl27kpWVVfTtNdUtt9xCnz59GDRo0G4fRCWdY8qUKWRlZZGdnc2HH35YoTuCSqtLWbp06cL69evZsGEDy5YtY8WKFfTt27doe6dOnWjatCnvv/8+p556Kscccwy9e/cmOzubd955p6jjNiMjgxdeeIH777+fzMxMunXrxoQJE0r9Bh/V1VdfzfTp08nMzGT69OlcfXVwYeSLL77g5JNPLip3xhln0K1bNwYPHsyDDz7IfvvtV7R/VlYWPXv25J///Odu/w7eeOMNTjnllErFV8jcvUoOtMeBzX4KnOjuvwqXzwP6uPuolDIvAHe4+9vh8mvAVe4+u9ixRhK0GOjQoUPv5cuXVzieOcvX8sjbn3LdKd3Yv3nVNKekbEuWLKFr167VHYbUcvfccw9NmjTZ41mC2uy7777juOOO4+23396tP6FQSf/3zGyOu+eUdLw4WwQlXVAtnnWilMHdx7l7jrvnpHYMVUTvH+7Hn8/trSQgUstcdNFFu/W/JMGKFSu44447SkwC30ecdw0VAAemLLcHindxRykjIlKqBg0acN5551V3GGmVmZkZ+Y6sKOJsEcwCMs2sk5nVB84GphYrMxU4P7x7qC+w3t33vEgpNVZclx5FpGTf5/9cbC0Cd99hZqOAV4A6wKPuvsjMLgy3jwWmAScD+cBm4IK44pH0a9CgAWvWrNFQ1CJpUjgfQYMGDSq0X2ydxXHJycnx2bNnl19Qqp1mKBNJv9JmKCurs1hPFkts6tWrV6FZkkSkemisIRGRhFMiEBFJOCUCEZGEq3GdxWa2Gqj4o8WBVsA3VRhOTaA6J4PqnAyVqfMP3b3EJ3JrXCKoDDObXVqveW2lOieD6pwMcdVZl4ZERBJOiUBEJOGSlgjGVXcA1UB1TgbVORliqXOi+ghERGRPSWsRiIhIMUoEIiIJVysTgZmdaGYfmVm+mV1dwnYzszHh9gVmdnhJx6lJItT53LCuC8xsppn1qo44q1J5dU4p9yMz2xnOmlejRamzmfU3s1wzW2Rm/0p3jFUtwr/tZmb2vJnND+tco0cxNrNHzexrM8srZXvVf365e616EQx5/QlwEFAfmA90K1bmZOAlghnS+gLvV3fcaajzkcB+4fuTklDnlHKvEwx5/tPqjjsNf+fmwGKgQ7jcprrjTkOdfwf8IXzfGvgWqF/dsVeizscChwN5pWyv8s+v2tgiOALId/dP3X0bMBkYWqzMUOBxD7wHNDezdukOtAqVW2d3n+nua8PF9whmg6vJovydAS4Fnga+TmdwMYlS53OAZ9x9BYC71/R6R6mzA00smPSiMUEi2JHeMKuOu88gqENpqvzzqzYmggOAlSnLBeG6ipapSSpan18SfKOoycqts5kdAJwGjE1jXHGK8nc+BNjPzN40szlmdn7aootHlDo/AHQlmOZ2IfAbd9+VnvCqRZV/ftXG+QhKmgqr+D2yUcrUJJHrY2YDCBLB0bFGFL8odb4XuMrdd9aSGdKi1Lku0BsYCGQA75rZe+6+NO7gYhKlzicAucCPgYOB6Wb2lrv/O+7gqkmVf37VxkRQAByYstye4JtCRcvUJJHqY2Y9gYeBk9x9TZpii0uUOucAk8Mk0Ao42cx2uPs/0hNilYv6b/sbd98EbDKzGUAvoKYmgih1vgC4w4ML6Plm9hnQBfggPSGmXZV/ftXGS0OzgEwz62Rm9YGzganFykwFzg973/sC6919VboDrULl1tnMOgDPAOfV4G+Hqcqts7t3cveO7t4ReAq4uAYnAYj2b/s54Bgzq2tmDYE+wJI0x1mVotR5BUELCDNrCxwKfJrWKNOryj+/al2LwN13mNko4BWCOw4edfdFZnZhuH0swR0kJwP5wGaCbxQ1VsQ63wC0BP4cfkPe4TV45MaIda5VotTZ3ZeY2cvAAmAX8LC7l3gbYsmD0ooAAARCSURBVE0Q8e98CzDBzBYSXDa5yt1r7PDUZjYJ6A+0MrMC4EagHsT3+aUhJkREEq42XhoSEZEKUCIQEUk4JQIRkYRTIhARSTglAhGRhFMikL1SOFpobsqrYxllN1bB+SaY2WfhueaaWb/vcYyHzaxb+P53xbbNrGyM4XEKfy954Yibzcspn21mJ1fFuaX20u2jslcys43u3riqy5ZxjAnAC+7+lJkdD9zl7j0rcbxKx1Tecc3sMWCpu99aRvkRQI67j6rqWKT2UItAagQza2xmr4Xf1hea2R4jjZpZOzObkfKN+Zhw/fFm9m6475NmVt4H9Aygc7jv/4THyjOz34brGpnZi+H493lmNixc/6aZ5ZjZHUBGGMfEcNvG8OeU1G/oYUvkDDOrY2Z3mtksC8aY/3WEX8u7hIONmdkRFswzMS/8eWj4JO5oYFgYy7Aw9kfD88wr6fcoCVTdY2/rpVdJL2AnwUBiucCzBE/BNw23tSJ4qrKwRbsx/Pm/wLXh+zpAk7DsDKBRuP4q4IYSzjeBcL4C4EzgfYLB2xYCjQiGN14EHAacATyUsm+z8OebBN++i2JKKVMY42nAY+H7+gSjSGYAI4HrwvX7ArOBTiXEuTGlfk8CJ4bLTYG64fufAE+H70cAD6Tsfxvws/B9c4IxiBpV999br+p91bohJqTW2OLu2YULZlYPuM3MjiUYOuEAoC3wZco+s4BHw7L/cPdcMzsO6Aa8Ew6tUZ/gm3RJ7jSz64DVBCO0DgSe9WAAN8zsGeAY4GXgLjP7A8HlpLcqUK+XgDFmti9wIjDD3beEl6N62n9mUWsGZAKfFds/w8xygY7AHGB6SvnHzCyTYCTKeqWc/3hgiJldHi43ADpQs8cjkkpSIpCa4lyC2ad6u/t2M1tG8CFWxN1nhIniFOAJM7sTWAtMd/fhEc5xhbs/VbhgZj8pqZC7LzWz3gTjvdxuZv9099FRKuHuW83sTYKhk4cBkwpPB1zq7q+Uc4gt7p5tZs2AF4BLgDEE4+284e6nhR3rb5ayvwFnuPtHUeKVZFAfgdQUzYCvwyQwAPhh8QJm9sOwzEPAIwTT/b0HHGVmhdf8G5rZIRHPOQP4r3CfRgSXdd4ys/2Bze7+N+Cu8DzFbQ9bJiWZTDBQ2DEEg6kR/ryocB8zOyQ8Z4ncfT1wGXB5uE8z4PNw84iUohsILpEVegW41MLmkZkdVto5JDmUCKSmmAjkmNlsgtbBhyWU6Q/kmtk8guv497n7aoIPxklmtoAgMXSJckJ3n0vQd/ABQZ/Bw+4+D+gBfBBeorkW+H0Ju48DFhR2FhfzT4J5aV/1YPpFCOaJWAzMtWDS8r9STos9jGU+wdDMfyRonbxD0H9Q6A2gW2FnMUHLoV4YW164LAmn20dFRBJOLQIRkYRTIhARSTglAhGRhFMiEBFJOCUCEZGEUyIQEUk4JQIRkYT7f4WrkTVCuODUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting ROC curve using Scikit-Learn\n",
    "from sklearn.metrics import plot_roc_curve\n",
    "\n",
    "plot_roc_curve(clf, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This score represents the area under curve. \n",
    "\n",
    "The maximum score we can get is 1.0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Confusion Matrix**\n",
    "\n",
    "A confusion matrix is a quick way to compare the labels that our model predicts versus what it was supposed to predict.\n",
    "\n",
    "In essence giving us an idea where our model is getting confused."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[23,  9],\n",
       "       [ 4, 25]], dtype=int64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "confusion_matrix(y_preds, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Predicted Labels</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual Labels</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Predicted Labels   0   1\n",
       "Actual Labels           \n",
       "0                 23   4\n",
       "1                  9  25"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualize confusion matrix with pd.corosstab\n",
    "pd.crosstab(y_test,\n",
    "            y_preds,\n",
    "            rownames=['Actual Labels'],\n",
    "            colnames=['Predicted Labels'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above table shows the number of times predicted values matched with the actual labels and also the times when they didn't match.\n",
    "\n",
    "- The model predicted 0 when the actual label was also 0 23 times (True negative) whereas it predicted 0 when the actual label was 1 9 times (False negative).\n",
    "\n",
    "- The model predicted 1 when the actual label was also 1 25 times (True positive) whereas it predicted 1 when the actual label was 0 4 times (False positive).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making our confusion matrix more visual using Seaborn's heatmap()\n",
    "import seaborn as sns\n",
    "\n",
    "# Set the font scale\n",
    "sns.set(font_scale=1.5)\n",
    "\n",
    "# Create a confusion matrix\n",
    "conf_mat = confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAEWCAYAAAC0byiGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deViU5cIG8HtAQQEVMBQVRHDB2NTjcUEUSUTEfcElRDPrUzMVt0ys1LLj0tEMtTyokeKSRJgGKqIpVpSaVMfMHVBW0WSRRdaZ7w+PkziOvODwDDPev+vqupr3eWfmJvTumWfeRaZQKBQgIhLAQNsBiOj5wcIhImFYOEQkDAuHiIRh4RCRMCwcIhKmgbYD1JWS0xHajkC1ZOa5QNsR6BlUlGWoHeMMh4iEYeEQkTAsHCIShoVDRMKwcIhIGBYOEQnDwiEiYVg4RCQMC4eIhGHhEJEwLBwiEoaFQ0TCsHCISBgWDhEJw8IhImFYOEQkDAuHiIRh4RCRMCwcIhKGhUNEwrBwiEgYFg4RCcPCISJhWDhEJAwLh4iEYeEQkTAsHCIShoVDRMKwcIhIGBYOEQnDwiEiYVg4RCQMC4eIhGHhEJEwLBwiEoaFQ0TCsHCISBgWDhEJw8IhImFYOEQkDAuHiIRh4RCRMCwcIhKGhUNEwrBwiEiYBtoOQE+W8Mc1bPv2FC7dyIJMJoNbexvMHusNtw62yn3OXEzGZ/tP4GraLZg2Msagns6YPdYbJo2MtZicquPq+iLO/HwYa9ZuwgcrP9Z2HKE4w6mHzl1OwZvrd6OguASzx3pj5igvpN3OwbTVYfgjKR0AcPZSMmZ+tBPlFZUIGueDYR5d8PXJc3hj3S7I5XIt/wSkjqGhIT7fvgFGRkbajqIVnOHUQx/tOQJry6bYvWw6Ghs/+IM53KMrRgVvwuao4whdPBUff3kU1s2bIWzpNDQyaggAaNXcHKvCY/DTH9fRt0snbf4IpMaSt+fA2en5/d1whlPP3Cu6j6tp2RjU00VZNgDQvJkZuju2w+/X0lBaVg6LpqYY69VdWTYA0N2xHQDganq26NgkgYtLZywNnot/rQrRdhSt0foMJyMjAykpKSgsLISBgQGaNGkCe3t7WFtbazuaVpg2NsbBNXPR2LihylheYREaGBrA2KghtiyaojJ+JTULANDKslmd56SaMTQ0xPZtH+O7737Enr1R+OD9xdqOpBVaK5y4uDiEhIQgOTkZCoWiyphMJoOdnR3mzZuHwYMHaymhdhgaGMDOurnK9qupt/D7tTT0cWmvMpb5Vx5+uZSC9V/GooNNCwzo/qKIqFQDi996Ex072GOs/2to0MBQ23G0Rm3hdO7cGTKZrEYvJpPJcPHixWr3O3DgAJYsWQI/Pz/MmTMHdnZ2MDU1hUKhQFFREW7evImjR49i/vz5KC8vx/Dhw2uUQ98Ul5Ti3W37AQDThvWrMpZfWAy/hQ++6Whk1BBLAofC2Eh1dkTa4+TUCe++Mw9zg95FRkYW7OxstB1Ja9QWzqhRo2pcOFJt3boVL7/8MpYvX/7EcScnJ/j5+WHFihUIDQ19rgvnfmkZ5n6yF1dSb+G1Yf3wz872VcZlMhnWzhqH8opKfHnsNGZ8tBNrZ42DTw9nLSWmRxkYGODzbR8jIeEXfB62V9txtE5t4axZs6bO3jQjIwMDBw6sdj9vb2988803dZajvrtXdB9zNuzB79dSMcrzH5jjr/rfrKlpYwzu5QoA8OnhjLFLN2Pd3lgWTj2xaOEbcHNzQn+v0Wje3AIAYGHxYI3NxKQxmje3QE5Onsqygr6q8bdUOTk5iImJwdatW5Geno6cnBwkJSXV6DVsbW3x448/VrtffHz8c7t4fPdeIV5f8wV+v5aKsV7/xIppI6udcTYyagjPro64lZOP3IIiQUnpaXwHecHY2Binfz6M7KwLyM66gHNn4wAAixbOQnbWBbRt20bLKcWp0aJxWFgYQkJCUFpaCplMBldXVxQVFWHOnDmYOHEili1bJulj2MyZM/HWW2/h9u3bGDRoEOzt7WFmZgaZTIbCwkLlGk5MTAzef//9Wv9wuqrofine+Hc4rqTeQqCvO94K8KsynpJ5B7PW78LUIX0xwbtn1eeWPPjdGDXQ+heQBOCtxR/AwsK8yrYWLV/Arp2bsWv319i9+2vcunVHS+nEk/ynMjo6Gh999BGGDRsGHx8fBAUFAQCcnZ3h4+ODffv2wd7eHlOmqH5d+7hhw4bB0NAQGzZswKFDh1RKSqFQwMbGBqtWrcLo0aNr+CPpvlXhMbiSeguTBvVWKRsAsG1piYLiEkSe+AVj+v8DDf9XLpl/5eG7cxfR3dEOpo15ekN98Otvf6hse7honJJyE9+d+EF0JK2SXDhhYWHw8PDAunXrkJubq9zeqlUrbNy4ETNnzkRkZKSkwgEAPz8/+Pn5IS0tDcnJySgsLIRCoVAeh9O2bdua/zR6IDnzDmJ++i+amDSCY9tWiEn4r8o+wzy6YEngULyzNQrTVoVhaJ8uyCssxr7jZyGTybBk8lAtJCeqnuTCSUpKgr+/v9rxl156CatXr65xAFtbW9ja2la/43Pi3OUbAICC4hIs2/7kBfNhHl0wzKMLGjYwxBeHfsC6L2PR2NgIvZwcMNvfG+2sXxCYmEg6yYVjamqKgoICteOZmZkwMTHRSKjn2fgBPTB+QA9J+/r2coFvL5c6TkSadvNmOhoYPT8LxY+S/C1Vv379sHfvXty9e1dl7PLly9izZw/69Omj0XBEpF9kCokHAGRnZ8Pf3x/l5eXo0aMHjh8/Dl9fX1RUVCA+Ph5mZmaIjIysNx+PSk5HaDsC1ZKZ5wJtR6BnUFGWoXZM8gynZcuWiIqKgpeXF06fPg2FQoHY2FgkJCTA29u7XpUNEdVPkmc4j1IoFMjNzUVlZSUsLS1haFj/TkbjDEd3cYaj2542w6nx0WH5+flISEhARkYGDA0N0bZtW/Tu3RtmZmbPFJKI9F+NCmfz5s3Ytm0bSktLq2w3NTXF4sWLMWHCBI2GIyL9Irlwdu7cic2bN6N3794IDAyEra0tFAoFUlJSsHPnTqxYsQJmZmYYOpQHnRHRk0lew/H19YWNjQ0+//xzlbHKykpMmTIF9+7dQ3R0tMZD1gbXcHQX13B0m0a+pcrKyoK3t/cTxwwNDTF06FDcvHmz5umI6LkhuXA6duyI//5X9byeh1JTU9GuXTtNZCIiPSW5cN577z3ExcVh/fr1yMvLU24vLi5GeHg4vv76ayxdurROQhKRflC7hvOkaxorFArlNnNzc8hkMuTn50Mul6NRo0YwNzfHyZMn6z61BFzD0V1cw9FttToOpy6vaUxEzyetXNOYiJ5PGrnzplwuR0FBAY4dO6aJlyMiPSX5wL+CggKsXr0acXFxKC4uVnuV+UuXLmksHBHpF8kznH//+9/Yv38/OnToAHd3dygUCgwbNgzu7u5o0KABjI2NsWnTprrMSkQ6TvIMJz4+Hj4+Pti0aRNyc3Ph7u6OyZMnw83NDZcuXcKkSZOQnJxcl1mJSMdJnuHk5OTAw8MDAGBhYYGWLVvi/PnzAIAXX3wR/v7+9ea0BiKqnyQXjqmpKeRyufJx27ZtcfXqVeXjTp06ITMzU7PpiEivSC4cNzc3HDlyBJWVlQCADh064Ny5c8rF45SUFBgZGdVNSiLSC5ILZ9q0aUhMTISvry/y8/MxevRoJCcnY9q0aVixYgXCw8PRq1evusxKRDpOcuG4u7tj69atsLe3R9OmTeHm5oYVK1bg999/x759++Dk5ITg4OC6zEpEOq5W1zR+VFlZGUpKStC0aVNNZdIInkulu3gulW7T6DWNH2dkZMS1GyKSRG3hPOls8erIZDJcvHjxmUMRkX7i2eJEJAzPFiciYTRytjgRkRQsHCIShoVDRMKwcIhIGBYOEQnDwiEiYTR64B/AS4wSkXo1OvDv+PHjKC0tRd++feHg4AC5XI60tDScOnUKZmZmGDduXJ0HJiLdJfnAv127duHkyZM4ePAg7O3tq4ylp6cjICCARyYT0VNJXsPZvn07pk6dqlI2AGBjY4PAwEBERkZqNBwR6RfJhVNQUPDUs8LlcjnKyso0EoqI9JPkwunatSt27dqF7OxslbHr169jx44d6Nmzp0bDEZF+kXwBrgsXLmDy5MkwMDBA//79YWtri7KyMqSkpODHH39EkyZNsG/fPtjZ2dV1Zkl4AS7dxQtw6TaNXIDLxcUFkZGR2LhxI+Lj41FcXAwAMDMzw/DhwxEUFARra+tnT0tEeqtWlxhVKBTIzc2FTCaDhYVFXeR6Zpzh6C7OcHSbRi8xmpOTg59++gmZmZkYMmSIsnzat2//TCGJSP/VqHDCwsIQEhKC0tJSyGQyuLq6oqioCHPmzMHEiROxbNkyHotDRGpJ/pYqOjoaH330EXx8fBASEqK8AZ6zszN8fHywb98+7Nq1q86CEpHuk1w4YWFh8PDwwLp166p8/d2qVSts3LgR/fv354F/RPRUkj9SJSUlwd/fX+34Sy+9hNWrV2sklCa8PoGLxrrqfuYP2o5AdUTyDMfU1BQFBQVqxzMzM2FiYqKRUESknyQXTr9+/bB3717cvXtXZezy5cvYs2cP+vTpo9FwRKRfJB+Hk52dDX9/f5SXl6NHjx44fvw4fH19UVFRgfj4eJiZmSEyMhK2trZ1nVmSQLsx2o5AtfRF4jptR6Bn0PAFB7Vjkmc4LVu2RFRUFLy8vHD69GkoFArExsYiISEB3t7e9apsiKh+qtFxOC1atMCaNWuUB/tVVlbC0tIShoaGAICysjLeZ5yI1JI8w/H29sZ3330H4ME9xC0tLWFlZaUsm5iYGPTr169uUhKRXlA7w8nJyUFSUpLycUZGBv744w80bdpUZV+5XI5jx47xejhE9FRqC8fY2BgLFy7EnTt3ADyY1YSGhiI0NPSJ+ysUCgwZMqRuUhKRXlBbOKamptiyZQuuXr0KhUKBpUuXYvz48ejWrZvKvgYGBrC0tIS7u3udhiUi3fbURWNnZ2c4OzsDeHBg36BBg9CpUychwYhI/0heNJ49ezbKysowf/78Kgf/rV27FnPnzq2y3kNE9CSSC+fcuXMICAhAQkICcnNzldutrKyQmJgIf39/XL58uU5CEpF+kFw4ISEhsLe3R1xcHDp06KDcPm3aNBw+fBi2trZYv359nYQkIv0guXAuXbqECRMmwNzcXGWsWbNmGD9+PM6fP6/RcESkXyQXToMGDap8lHpcYWEh5HK5RkIRkX6SXDi9evXC7t27kZaWpjKWnZ2N3bt3875URPRUks+lCgoKwrhx4zBixAh4enqiXbt2kMlkSE1NxalTpyCTybBgAa+2T0TqSS4cBwcH7N+/Hxs2bMD333+Po0ePAgAaNWoEDw8PLFiwgHduIKKnqtHZ4nZ2dvjkk0+UZ4vL5XJYWFgoT+AkInqaGt+XCvj7bHEioppQWzje3t5YunQpvL29lY+rI5PJcPz4cc2lIyK9orZwWrduXeWi6K1btxYSiIj0V63uLa4LeE1j3cVrGus2jVzTmIjoWan9SDVlypRavWB4eHitwxCRflNbOOnp6Srb7t69i9LSUjRr1gx2dnaQy+XIyMhAbm4uzM3NeRwOET2V2sI5ceJElcdnzpzBzJkzsWbNGowYMQIGBn9/GouJicG7776LSZMm1V1SItJ5ktdwPvzwQ/j7+2PUqFFVygYAhg0bhoCAAISEhGg8IBHpD8mFk5qainbt2qkdt7a2xu3btzWRiYj0lOTCsbe3x6FDh1BZWakyVlpaiqioKDg6Omo0HBHpF8mnNkyfPh0LFixAQEAAxowZA1tbW5SWluLGjRv48ssvkZmZqfYWMkREQA0KZ8iQISgpKcH69euxfPlyyGQyAA/uR9WmTRts3rwZHh4edRaUiHRfjU7eHDNmDEaNGoU///wTGRkZkMlksLW1hZOTU13lIyI9UuOzxQ0MDNCiRQvI5XI4ODjA2NgYcrlc5ZsrIqLH1aglEhMTMWbMGHh5eWHixIm4cOECzp49Cy8vLxw+fLiuMhKRnpBcOOfPn8err76KoqIivPLKK3h4zmezZs3QoEEDLFq0CKdOnaqzoESk+2p0XyobGxscPHgQ06dPV253dXXFt99+i/bt2/NbKiJ6KsmF89tvv2HMmDFo1KiR8huqh8zMzDB+/Hhcu3ZN4wGJSH/UaA3HyMhI7VhpaSnvS0VETyW5cLp06YKYmJgnjhUXFyMyMhKurq4aC0ZE+kdy4cydOxcXL15EYGAgDhw4AJlMhvPnzyM8PBwjR45Eeno6Zs6cWZdZiUjH1egSowkJCVi+fLnKtXKsrKzw3nvvYdCgQRoPWFu8xKju4iVGddvTLjEq+cC/3NxceHh44NixY7h48SJSU1Mhl8vRpk0buLi4oEGDWt1xhoieI5JbYvTo0Rg3bhzefPNNODs7w9nZuS5zEZEekryGk5OTAysrq7rMQkR6TvIMZ/jw4YiIiECfPn1gY2NTl5lIjU7/7IzxiyfB3q0DivILkRh3FlEf70NhboG2o9FjEs4kInTHl7h45TpkBjJ0ce6MOf83BV1cXlTuM/H1IFy4dFXluT5eHtjwr3dFxhVGcuEYGBggOTkZvr6+aNu2LZo3b65ywqZMJsPOnTs1HpKAF3s7Y3H4MhTfK8K3n0ZBXinH4GnD4NTHFe+PDkbxvSJtR6T/+eW385i58D10sLfD3OmvoKKyEhHfxGDq7MUI/2wdXJ0coVAokHwjDQM83eHj1bfK81u3bKGl5HVPcuEkJCTAwsICwIOD/DIzM+ssFKma8v7rkMvleH9MMG6nZgMAzh09g9WxH2PkHH98+S8WfX2xNiQU1i2ssHfbBjRu1AgAMMLPGyMCpiMkdCe2h6xCRlY2iu/fx4C+7hjuO0DLicWRXDiP38WBxHnBxgq2ne1wYk+csmwAICspA79+dw79xnqxcOqJ/HsFuHI9Ba9MHKMsGwB4wdIC/+zmip/P/goAuJ5yEwDg0M5WKzm1pdrCKS8vx/Xr11FRUYEOHTqgcePGInLRIyysmwMA0q7cVBnLvnELPf3cYdmqOXKy7oqORo8xMzVBzJfbqpTNQ3l592BoaAgASHqscIrvl8Cksepz9M1TC2fHjh349NNPUVhYCODBuVQBAQFYuHAhj7sRqLS4BADQyFS17JtYNAEAmFtZsHDqAUNDQ9jZtlHZfuV6Cn774yI8enUHAFxLvglTk8b4aOM2xH73PYrv34dNa2vMnfEKhgz0EpxaHLWtceDAAaxZswZt2rTByJEjYWBggDNnzmDHjh2orKzE0qVLReZ8rmVcS0fxvSL08OuN6M/2K7c3NG4IV8+uyn+n+qm4+D6Wrnxw9PRrgeMAPJjhFBXfR0FhIVa9txAFhUXY/dVBLF6+FhUVlRgx2FubkeuM2sLZu3cvunbtip07d8LY2BjAgwumz58/HxEREVi0aNFTzx6vTnZ2dvU7PaJly5a1fi9dV1legSPbozF2wUTM2jgP3366HwaGBhi3KADGJg9+N0+6fQ9p3/2SEsx++31cuZ6M1ydPQI9ubgAA/xF+kMvleHnscOW+fgP7Y1TgTKz/9HMM9fFSfvzSJ2oLJykpCQsWLFCWDfDga++pU6fi6NGjSE5ORufOnWv9xt7e3jX6S3Lp0qVav5c+OLAxEiZNTeH76hD0GekJAPj12C+I2XIAE4MnoyivUMsJ6XH3Cgrx5uLl+O38RYweNghBM15Rjk0YPVRl/0bGxhg+2BtbwvYg6UYqOrW3FxlXCLWFc//+fTRp0kRlu42NDRQKBe7du/dMbxwZGYkZM2agrKyMa0ISKBQK7Fn5BaK37Id1u1a4m3UXdzPuYNxbAaisqMRfGXe0HZEecTc3DzPmv4PL15IxbqQflr01R+XCdU/S3MIcwINFZH2k9m+5XC5/4n+gh9O8Z53Cv/jii9ixYwfGjRuHO3fuYNasWc/0evrOfURf5N3OxaXTf+LeX/nK7Z17OiHljySUl5ZrMR09qqioWFk2UyaMxuK506uMZ9/5C9PnvYPB3p54Y9qkKmPJN9MAADat9HMJQav3dnFwcMCCBQuwfft25OTkaDNKvTf4teGY8sH/wcDw719Z1wHd4djTCcfDY7WYjB734cef4fK1ZASOG6lSNgDQ0uoFFBYV4evoWBQW/X2EeFb2HRw8fAw9/9EFLzS3FBlZmKd+jsnLy1M5ojg//8H/XXNycp54tHHr1q1rFGDixIno2LFjjZ7zPIr5zzcI+s9iLAp7B7/EnoaVTQsMfn04zsf/hoQD32s7Hv1P0o1URMd+hyZmpujcsT2ij6oeMDvcdwCWLpiFoOCVCJyxEGNHDEZx8X3sjYqGoaEh3lmov7N9tRfg6ty5s9rPnAqF4oljMpkMFy9e1GzCWtLHC3D1Hu6B4W+MQUv7Vrj3Vx5+OvADvv00CmUlZdqOplG6fAGuiG8OYeW6zU/d50LCEQDAiR9+xrbwCFy5noxGxsb4ZzdXzJv5KhzsdPvo46ddgEtt4QQHB9fqzVavXl2r52maPhbO80KXC4dqecW/+lIcRKQ/eENwIhKGhUNEwrBwiEgYFg4RCcPCISJhWDhEJAwLh4iEYeEQkTAsHCIShoVDRMKwcIhIGBYOEQnDwiEiYVg4RCQMC4eIhGHhEJEwLBwiEoaFQ0TCsHCISBgWDhEJw8IhImFYOEQkDAuHiIRh4RCRMCwcIhKGhUNEwrBwiEgYFg4RCcPCISJhWDhEJAwLh4iEYeEQkTAsHCIShoVDRMKwcIhIGBYOEQnDwiEiYVg4RCQMC4eIhGHhEJEwLBwiEoaFQ0TCsHCISBgWDhEJw8IhImFYOEQkDAuHiISRKRQKhbZDENHzgTMcIhKGhUNEwrBwiEgYFg4RCcPCISJhWDhEJAwLh4iEYeEQkTAsHCIShoWjY2JiYjB06FC4ubnBz88PBw4c0HYkqqFLly7B2dkZt27d0nYU4Vg4OuTIkSNYtGgRPDw88Omnn6Jnz554++23ERsbq+1oJFFycjJmzJiBiooKbUfRCp5LpUN8fHzg4uKCDRs2KLfNmzcPV65cwZEjR7SYjKpTUVGBiIgIrF+/Hg0bNkReXh5OnToFa2trbUcTijMcHZGWlobU1FQMGjSoynZfX18kJycjLS1NS8lIisTERKxbtw7Tpk3DokWLtB1Ha1g4OiI5ORkAYG9vX2W7nZ0dACAlJUV4JpKuffv2OH78OGbPng1DQ0Ntx9GaBtoOQNIUFBQAAMzMzKpsNzU1BQAUFhYKz0TSvfDCC9qOUC9whqMjHi61yWSyJ243MOCvkuo//inVEU2aNAGgOpMpKiqqMk5Un7FwdMTDtZvU1NQq22/evFllnKg+Y+HoCDs7O9jY2KgccxMXF4d27dqhdevWWkpGJB0XjXXIm2++ieDgYDRr1gxeXl44ceIEjhw5UuW4HKL6jIWjQ8aMGYOysjKEhYUhMjIStra2WLt2LYYMGaLtaESS8EhjIhKGazhEJAwLh4iEYeEQkTAsHCIShoVDRMKwcIhIGBbOc2rJkiVwdHSs9p8lS5ZoNeemTZvg6OiI9PR0Ic8T9XrPKx7495yaMGEC3N3dlY8TExMRERGBCRMmoHv37srtbdu21UY80lMsnOdUt27d0K1bN+XjyspKREREoGvXrhg5cqQWk5E+40cqIhKGhUPV2rRpE1xdXXHs2DF4eHigW7duiIyMVLuu8aTt+fn5WLlyJfr16wcXFxf4+flh586dqM2ZNX/++SfmzJmDPn36wNnZGe7u7li4cOETb7uSnJyMKVOmwM3NDV5eXggJCUF5eXmVfTSZjZ6OH6lIkoqKCrz77rt47bXXUFZWhu7du+PQoUOSnltcXIzAwEBkZWUhICAA1tbWOH36NFatWoUbN25g+fLlknNcuXIFAQEBsLOzw/Tp09G4cWP8+uuvOHjwIG7fvo1du3ZV2T8oKAi9evXC22+/jbNnz+Kzzz5DVlYW1qxZo/FsVD0WDkkil8sRGBiI6dOn1/i5n3/+OVJSUhAVFQVHR0cAQEBAAD7++GOEhoZiwoQJ6Ny5s6TX2rt3L2QyGcLDw2Fubg7gwQJ4eXk5Dh06hLy8POV2AOjfvz8++eQTAMCkSZMQHByM/fv349VXX4Wjo6NGs1H1+JGKJOvbt2+tnhcXF4dOnTrBysoKOTk5yn8GDhwIADh58qTk11qxYgVOnDhRpVQKCwthbGwM4MGM5VGvvfZalceTJ08GAJw6dUrj2ah6nOGQZM2bN6/V81JTU1FSUlLla/hHZWVlSX4tmUyG3NxchIaG4sqVK0hNTUVmZqZyvUUul1fZ38HBocrjh1/zP1xf0mQ2qh4LhySTemeIyspKlcfdu3fH7Nmzn7h/ixYtJGeIj4/HrFmz0KJFC/Tu3Ruenp5wcXHBjz/+iNDQUJX91d3l4uG9oTSZjarHwqFae1hAZWVlVbb/9ddfVR63adMGRUVF6NOnT5Xt+fn5+Pnnn5U385Ni5cqVsLOzQ1RUFExMTJTbo6Ojn7h/RkYGOnbsqHz88IaBD2c6msxG1eMaDtWalZUVAODy5cvKbYWFhcr1kYcGDBiAy5cvIz4+vsr2LVu2ICgoCNeuXZP8nnl5eWjdunWVssnKykJcXBwA1dnVV199VeXxF198AZlMhgEDBmg8G1WPMxyqtYEDB+LDDz/EBx98gIyMDBgZGeGrr76qUgYAMGPGDMTFxWH27NmYOHEiOnbsiMTERBw8eBCenp7w9PSU/J6enp44fPgwli1bBldXV6Snp+Orr77C/fv3Afx9n66HoqOjUVhYCDc3N5w6dQonT57E66+/rpy5aDIbVY+FQ7VmaWmJbdu2Yf369di4cSMsLCwwfvx4ODg4YP78+cr9zM3NERERgY0bNyI2NhYRERFo3bo1Zs2ahenTp9forqErVqyAiYkJTpw4gYMHD8La2hqjRo2Cj48PXn75ZZw+fRpOTk7K/bdt24YPP/wQMTExaNmyJYKDgzF16tQ6yUbV40XUiUgY1jcRCcPCISJhWDhEJAwLh4iEYeEQkTAsHCIShoVDRMKwcIhIGBYOEQnDwqORwwcAAAAJSURBVCEiYf4fjTF8vvrJNMMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting it using Seaborn's heatmap()\n",
    "def plot_conf_mat(conf_mat):\n",
    "    \"\"\"\n",
    "    Plots a confusion matrix using Seaborn's heatmap().\n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(4,4))\n",
    "    ax = sns.heatmap(conf_mat,\n",
    "                annot=True,\n",
    "                cbar = False);\n",
    "    plt.xlabel('True label')\n",
    "    plt.ylabel('Predicted label');\n",
    "    \n",
    "plot_conf_mat(conf_mat)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x1f95366eb20>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAAEcCAYAAABUAHZfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de1hU1d4H8O/MCAgOIKBHEUWUvCumkoqmooiopSKGWgivoCimSIo3Omadx0rxkinaMUPr6MEE8/J6TMuDha+VZl5OHhNQ8wIiYFwEuTMz+/2DmBpnYGYUZobh+3me/fSw1tqL35D8WHuvtfcSCYIggIiIVIiNHQARkSliciQi0oDJkYhIAyZHIiINmByJiDRgciQi0qCFsQMgInpaCoUCiYmJ2L9/P+7fvw8nJyf4+PggMjISUqkUAODr64uMjAy1c8+dOwdHR8c6+2ZyJKImKz4+Hh9++CHmzJkDLy8v3LlzB9u2bcOtW7ewe/dulJaWIjMzE9HR0Rg8eLDKuXZ2dvX2zeRIRE2SIAiIj4/HjBkzEB0dDQAYNmwYHBwcsGTJEqSmpqK8vByCIMDHxwfu7u569c/kSERNUmlpKSZPnowJEyaolHft2hUAkJGRgby8PFhZWcHNzU3v/jkhQ0RNklQqxerVqzFo0CCV8uTkZADAc889h/T0dLRu3RpLly6Fp6cnBgwYgCVLluC3337T2j9HjkRkUoqLi1FcXKxWbmdnp/U+4c8//4xdu3Zh7NixcHd3R1paGvLy8tCtWzcEBwfj9u3b2LZtG0JCQnDkyBG0bNmyzr5EfPEEETU2QfEIInFrndpWVFRg5MiRKCoqUilftGgRIiMj6zzv0qVLiIiIQNu2bZGQkAAHBwdcvXoVgiCgf//+Ku1ee+01rF27FtOnT6+zP7MYOV7JnolKeY6xw2hQQzum4Px9b2OH0Si2j+ht7BAaxT/vfIRZXV43dhgNqo2LIz787t1n7kckbg1Z/gxAoeX3VNweLZ0S8b//+7+Qy+UqVfWNGk+cOIFVq1bBzc0N8fHxcHBwAAB4eHiotR00aBBsbW2RlpZWbyhmkRwr5TmokGUZO4wGZ46fCQBy77U1dgiNJvee9ntZzZVc/gCQa/k3LVGgBQBnZ2ed+/30008RGxuLwYMHY8eOHbC1tQUAlJWV4eTJk+jTpw969uypbC8IAqqrq5UJtC6ckCEigxAgQKHlEKDfXb6DBw9i/fr1mDBhAuLj45WJEQCsrKwQGxuL7du3q5xz+vRpVFRUqK17fJJZjByJyPQpfk+P9RHpkRzz8/Px3nvvwcXFBUFBQbh+/bpKvaurKxYsWID169fj3XffxZgxY3Djxg3ExcXBx8cHQ4YMqbd/JkciMgiZoIBCqD85irXU/9nZs2dRXl6OrKwsBAUFqdVv2LABoaGhkEql2Lt3Lw4ePAh7e3vMnDmz3omdWkyORGQQ8t8vneujz2W1v78//P39tbYLDAxEYGCgzv3WYnIkIoNQ6JAcoec9x8bE5EhEBqEQBMi1Las2oWXXTI5EZBCK34/6iAwRiI6YHInIIOQQIOdlNRGRKplQc9THhK6qmRyJyDDkEEGu5cJZZEIX1kyORGQQCqHm0NbGVDA5EpFBKHQYOYo5ciSi5kaXy2omRyJqdmSCGNVC/e+6EWmpNyQmRyIyCDnEkGt5EZi2ekNiciQig6iZkKn/spkTMkTU7OgyIaPgPUciam7kEEOu5Z4iL6uJqNlRQAyFluSnrd6QmByJyCCqBTGqBEm9bcScrSai5kYBkdZ7irznSETNjkKHpTy8rCaiZkcu6DAhw8tqImpuOCFDRKSBQgDkXARORKSqWmiBaqH+lKOt3pBMJxIiMmuckCEi0kAuiLReVmurNyQmRyIyiJp1jtpGjkyORNTMKHRYyqPgUh4iam6qBQmqtTw+qK3ekJgcicggal5ZxstqIiIVCoi0v+yWyZGImhtuk0BEpIEgiLVOuAickCGi5kaXrVm11RsSkyMRGUTN1qz1z0bLOHIkouZGocNlNdc5ElGzw/c5EhFpIOiwTYLAe45E1NzUvHhC28iRyZGImhmFoMMicCZHImpuZDo8Wy3js9UEAL/+ny2+394O2ddsIBILcHm+DN7R2XAZUKZsc/cHKc5saY+HadawlMrRa0IRvKOzYdlKYcTISROhOg3H717Fgbi/4J+b2xs7HJPTGHvIKBQKJCYmYv/+/bh//z6cnJzg4+ODyMhISKVSAMB3332HLVu24NatW3BycsKsWbMQFhamtW/TmRpqZu792AoHQruiolgC7+hsjFicg8IMS+x79Tlk/WwDALh7Tor9Ie6QV4swenk2+vkX4srnTvh8dlcIzI0mRSwRIBStgoWlCW2CYmLkEClfeFvnoeeETHx8PNauXQtvb2/s2LEDoaGhOHr0KKKiogAAly9fRkREBLp27Yq4uDhMmjQJGzZswO7du7X2zZGjkfx7rQvsnKsReuQGLKxrfqH6TS3Ex+N64swmZwRNAU6v6wC7DlUIPnALFi1r2th1qMLXb3fCr/9ni+e8HxvzI9CfzIx8CMjyjR2GSRME7fcUBT3+tgiCgPj4eMyYMQPR0dEAgGHDhsHBwQFLlixBamoqtm3bht69e2Pjxo0AgJEjR0Imk2Hnzp0IDg6GpaVlnf0bfeR4/PhxvPTSS/Dw8MCECRNw9OhRY4fU6MqLJMhNtUbvlx4pEyMASNvK4DqkBPcv26Cqogo2jjIMmJGvTIwA0HlIKQDgYZq1weMmzdx6luPVqFyIpAuNHYpJq10Eru3QVWlpKSZPnoyXX35Zpbxr164AgJs3b+LixYsYN26cSr2fnx+Ki4tx+fLlevs36sjx5MmTWLZsGUJCQjBixAgkJydj5cqVaNmyJcaPH2/M0BqVlVSOBcmpsLBRvzYuL2gBsQSwbGmJVz+7rVafe70mKdp3qGr0OEk7sURA9JZMXDkrxdCZk4GSD40dksmqeXyw/uSnz+ODUqkUq1evVitPTk4GAPTu3RvV1dXo0qWLSn3nzp0BAHfu3MHQoUPr7N+oyfGDDz7AhAkT8OabbwIARowYgaKiImzdutWsk6NYAjh2UU9uuaktkXmpFbqOVL9cLsqywN1ztjj9fge07V6OHn5FhgiVtJix8CFculTib2FuGDrT2NGYNn0eH8zOzoZcLleps7Ozg52dXb3n//zzz9i1axfGjh2Lx49rfo9qJ2ZqtWrVCgBQUlJSb19GS46ZmZnIyMjA0qVLVcr9/Pxw8uRJZGZmolOnTkaKzvCqSsU4tqzmL9qwiFyVuvJHEmwf0QcAYGEth987WWhhxRv/xta5ewVeW5KLj/7qgrzsuu9dUQ2FDk/I1NYHBQUhKytLpW7RokWIjIys89xLly4hIiICHTt2xLvvvos7d+4AAEQizd9TLK4/URstOd6+XXPJWN+Qt7kkx+pyEZLmdcHDVGsMW5CrvK+oJAKmbrsLebUIP33WFgnB7pi69S56TeTo0VjEYgHRWzLwy4VWOLnfydjhNAlyQfsTMPLf/+YnJCRoHDnW5cSJE1i1ahXc3NwQHx8PBwcH5OXlAVAfIdZ+bWtrW28sRkuOzzrk/bOhHVMaLC5DK3lUitXB63DvXDrGh47G0u0LlH/pvN1u/tGwf81/KiMqEd4vGmdj22DB6zuNEPGz8zaDZUhCyccQSlIhctqHU7KONYXyHABA8JpJCImdC4haQyQy+pynydDnZbfOzs469/vpp58iNjYWgwcPxo4dO5RJz9XVFRKJBBkZGSrta79+cmD2JKMlR+H3Ofsnh7y15dqGvH92/r43KmRZ2huamNK8Fvh8dlfkXrfBgFfzMHD1Fpy5twVATWJMudtN43kdR7rgp8/a4sTlnrBxlGtsY8re6/q8sUN4Zhu+uIX+w6oh5L+iXlkaD6E0HiGDeyH3ftO+3G7XuS3+eeejBumrMR4fPHjwINavX4+JEyciNjZWZWmOlZUVPD09cerUKfzP//yPMtd8/fXXsLW1Rd++fevt22jJsTa7PzlCLC0tVak3V5UlYmViHBz2EL6rH6jUZ6RlYbtvbwydlwvP4Hy1cyESIOGCY6PZ9bcOsG2t+ocp9qvXIRQtQ/JBByR/4YCC37iM+M9kEGudjZbpsbowPz8f7733HlxcXBAUFITr16+r1Lu6umLBggUIDQ3FkiVLMHXqVFy5cgW7d+9GdHQ0rK3rXw5ntP97tUPajIwM9OjRQ1l+7949lXpz9fXbHZF73QYvzP5NLTECgMtz7VH5WIwr+9tgwIwCZSIsyrJA+let0XlICaykZnB92kTd+q+NeqHFQABAdoYlrpw17z/uT6Oh95A5e/YsysvLkZWVhaCgILX6DRs2YMqUKYiLi8O2bduwcOFCtGvXDitWrNDp8UGjJcfOnTujY8eO+Oqrr+Dr66ssP3XqFNzc3NChQwdjhdbo8m5Z4b9HHGFlK0e73uX471EHtTbeb0gw7p0sHFvaGftefQ59pxSi/JEEF/e1AUTAuLeb3m0Eat4UOjwho9DjYsjf3x/+/v5a2/n6+qrkGF0Zddy/cOFCxMTEwN7eHt7e3vjmm29w8uRJbNmyxZhhNbp7P9ZMQlU+luD4CleNbSLfAPr5F0JiIeDcx39B8vsdYGGtgNuwEnhHZ8Opa6UhQyZ6Zvos5TEFRk2OAQEBqKqqwp49e3Dw4EF06tQJsbGxmDhxojHDanSDgvIxKEi353B7v/QIvV961MgRUUMQteiIcR36GzsMk8X3Oepp5syZmDmTjxYQmTtBh+QoMDkSUXMjU4ghU2iZrdZSb0hMjkRkELznSESkAS+riYg0UECHpTyGCUUnTI5EZBCcrSYi0kChEEOuZcJFwQkZImpuOCFDRKQBL6uJiDQQBJHW2egmMVv94IH6m2J0Yc4vjCCip2c2I8cxY8bUufdCfVJTU58pICIyU4IOI0MTekVpnclx4cKFT5UciYg0kQsiyBXa9pAxnZxTZ3Ksb5cvIiJ9mf1sdXp6OlJSUvDgwQOEhITAxsYGN27cwKhRoxojPiIyE2YzIaPJ2rVrsX//fgiCAJFIhPHjx6O4uBhRUVHw9vbG1q1bYWVl1VixElET1tSerdZ5OfrevXuRkJCAefPmISkpSblLoJeXF2bPno2UlBR88sknjRYoETVtgqDbYSp0To4HDhzA+PHjsWTJEnTq1ElZbmdnh1WrVmHy5Mk4fvx4owRJRE1f7WW1tsNU6JwcMzMzMXTo0DrrPT09kZ2d3SBBEZH5kf/+bLW2w1TofM/RwcEBOTk5ddbfvHkT9vb2DRIUEZkfAdovm03oqlr3kaOvry/279+PW7duKctq10GeOXMGiYmJGD16dMNHSERmQRB0ubQ2dpR/0HnkGBUVhQsXLiAgIADdunWDSCTC9u3bERsbi7S0NLi4uCAqKqoxYyWipkyXe4pN8Z6jnZ0dkpKSEB4ejqqqKlhZWeHnn39GeXk5QkNDcejQITg6OjZmrETUhAk6HqZCr3WO1tbWiIyM5NMzRKQ3QSGCoOXxQW31hqT3EzI3b95ESkoKsrKyIJFI4OrqijFjxqgs7yEiepLZPiEjk8nw1ltv4ejRo8oF4LViY2Mxd+5cLF26tMEDJCLzoMsi7yY5IfPRRx/hyJEjmDp1KkJCQpQjxdu3b+PTTz/FJ598grZt2yI4OLjRgiWipstsR45HjhzBhAkTsG7dOpVyDw8PbNmyBeXl5di3bx+TIxHVQaTDbLTpJEedZ6sLCgrwwgsv1Fnv7e2N3NzcBgmKiMyP2T5b3b9/f5w9e7bO+qtXr6JXr14NEhQRmZ/a2Wpth6nQeQ+Z8PBwLF68GNHR0ZgzZw66dOkCkUiErKwsJCUl8a08RFQ/XRYymtDIUa89ZARBwJdffokTJ06olQPAK6+8wj1kiEizJvaEDPeQISLDMJeRI5+CIaKG13QGXHo/IVNcXIyysjIoFAplmVwuR2lpKc6fP4/Zs2c3ZHxEZC4EAAod2pgInZNjbm4uVqxYgQsXLtTbjsmRiDQSdFjn2BTuOT5pw4YNuHDhAiZOnAhLS0scOXIE8+fPR0FBAU6dOoXKykp89tlnjRgqETVlTe3xQZ3XOZ47dw7+/v7YvHkz/vrXv0IkEmHEiBFYu3Ytjh49ChsbG/z73/9uzFiJqClr5HeWpaamok+fPmo7Fvj6+qJHjx5qR0FBQb396TxyLC4uxsCBAwEAUqkUHTp0wLVr1+Dp6QlnZ2cEBgbi1KlTWL58+VN8LCIye414WX379m3Mnz8fMplMpby0tBSZmZmIjo7G4MGDVers7Ozq7VPn5Ghvb4/y8nLl166urkhPT1d+3alTp3r3mCGi5k0k1Bza2uhDJpMhMTERmzdvhoWFhVp9eno6BEGAj48P3N3d9epb58vqgQMH4vDhw3j8+DEAoHv37vjxxx9RWVkJAPjvf/8LqVSq1zcnomZEIdLt0MOlS5ewadMmhIWFYdmyZWr1qampsLKygpubm97h6pwcFyxYgDt37mDUqFEoLCzE9OnTkZubi4CAAISHhyMpKQne3t56B0BEzYiO9xuzs7Nx//59laO4uFitO3d3dyQnJ2PRokWQSCRq9enp6WjdujWWLl0KT09PDBgwAEuWLMFvv/2mNVSdk2Pv3r2RlJSEyZMnw8HBAe7u7tixYwcqKipw5coVTJgwAStWrNC1OyJqbvSYkAkKCoKPj4/K8Y9//EOtyzZt2sDJyanOb5mWloa8vDx069YNO3fuRExMDH766SeEhISgoqKi3nD1WgTeo0cPvPPOO8qvvb29OVokIt3o8fhgQkIC5HK5SpW2CRRNVq9eDUEQ0L9/fwCAp6cn3N3d8dprr+HYsWOYPn16nefq/FYeXXXo0OGpziMiM6fHbLWzs3ODfEsPDw+1skGDBsHW1hZpaWn1nqvXW3l0wbfyEJFGOsxWN+Tjg2VlZTh58iT69OmDnj17/vEtBAHV1dVwcHCo93y+lYeIDMPAb+WxsrJCbGwsBg8ejO3btyvLT58+jYqKCrV1j08yi7fybH7nVeQ8VJ/JasrOHgPeWjzP2GE0ijMPdhk7hEbz9YP/GDuEhiVxabCuGmOdY30kEgkWLFiA9evX491338WYMWNw48YNxMXFwcfHB0OGDKn3fL3fykNE9FSM8OKJ0NBQSKVS7N27FwcPHoS9vT1mzpyp0+CPyZGIDKcRXywREBCAgIAAtfLAwEAEBgbq3R+TIxEZhrm8CZyIqCGJFDWHtjamgsmRiAzD3EeO6enpSElJwYMHDxASEgIbGxvcuHEDo0aNaoz4iMhMGHq2+lnplRzXrl2L/fv3QxAEiEQijB8/HsXFxYiKioK3tze2bt0KKyurxoqViJqyJrZNgs4vnti7dy8SEhIwb948JCUlKfeq9vLywuzZs5GSkoJPPvmk0QIloiaukd8E3tB0To4HDhzA+PHjsWTJEnTq1ElZbmdnh1WrVmHy5Mk4fvx4owRJRE2fCH9cWtd5GDvIP9E5OWZmZmLo0KF11nt6eiI7O7tBgiIi81M7W63tMBU633N0cHCodxuEmzdvwt7evkGCIiIz1MRmq3UeOfr6+mL//v24deuWsqz2xRRnzpxBYmIiRo8e3fAREpF5aGL3HHUeOUZFReHChQsICAhAt27dIBKJsH37dsTGxiItLQ0uLi6IiopqzFiJqAlrakt5dB452tnZISkpCeHh4aiqqoKVlRV+/vlnlJeXIzQ0FIcOHYKjo2NjxkpEZDB6rXO0trZGZGRkk3qdGRGZiCZ2z1Hn5KjrtgncJoGINBEJOjxb3RSTo67bJnCbBCLSyFxHjpq2TZDL5cjLy8OZM2fQqlUrXm4TUd0MvIfMs9I5OdaX+EpKSjBz5kzcu3evQYIiIjPUxEaOOs9W10cqlSIwMBCJiYkN0R0RmSGtjw7qMrI0oAZ7n2N1dTUKCwsbqjsiMjeK3w9tbUzEM89WV1VVITU1FXv27EGvXr0aLDAiMi9NbRF4g8xWC4IAKysrREdHN1hgRGSGTCj5aaNzcly0aJHGcrFYjLZt28LHx4dPyBBR3ZrYhIzOydHZ2RmDBg2Cm5tbI4ZDROaqqV1W6zxb/f777+PLL79szFiIyJyZ61t5rK2tuT8MET01s92a9Z133sHq1atRWVmJF198EY6OjpBIJGrt+Gw1EWlkrvccly5dCplMhri4OGzfvr3Odny2mog0EUH7HjGmtIeMzskxPDxcpxdPEBFpZC4jx5iYGMycORP9+/cHUP+z1URE2tTuPqitjamoc7b6yJEjyMjIMGQsRGTOzHW2mojoWZjtbDUR0TMxl3uOAHDx4kXI5XK9OvT393+mgIjITJnTy26TkpKQlJSkU0eCIEAkEjE5EpFm5jRynD59Op5//nlDxUJEZqypPVtdb3L09PTEpEmTDBULEZkzAdpfZttUkiMRUUMxq5EjEVGDMZd7jlOnToWrq6shYyEiMyYSBIiE+rOftnpDqvMJmXXr1ikfHSQiemaN/IRMamoq+vTpg5ycHJXy7777DtOmTUP//v0xZswY7NmzR6f+GmRrViIibRpza9bbt29j/vz5kMlkKuWXL19GREQEunbtiri4OEyaNAkbNmzA7t27tfbJe45EZBAiQYfHB/VMjjKZDImJidi8eTMsLCzU6rdt24bevXtj48aNAICRI0dCJpNh586dCA4OhqWlZZ19c+RIRIbRCJfVly5dwqZNmxAWFoZly5ap1FVWVuLixYsYN26cSrmfnx+Ki4tx+fLlevvmyNHE9HsuB+H+P0GRsw9fxIrx3X/c8Om/BqGopKWxQ2v2LqbYYv+H7XDrqg1EYgE9B5Zh9sps9BpUpmwTObEbbvynldq5L770CG99cteA0ZoefZbyZGdnqz26bGdnBzs7O5Uyd3d3JCcnw8nJCYcPH1apy8zMRHV1Nbp06aJS3rlzZwDAnTt3MHTo0DpjYXI0Ic93f4CNUSdRUm4FkTQCRw5/j1fGXsPAng/weuxklJRxDx9juXquFVYHdUXnHhWYvSobchnwr3+0wfJpz2HTkVvoOaAMgiAg82ZLDBv/CC++VKRy/l86VhkpchOix1KeoKAgZGVlqVQtWrRI7b2ybdq0qbOrx48fAwCkUqlKeatWNX+8SkpK6g2FydGERL36AxSCCAtjJyNx9+tI+KoUZ//jhj1rDiF44hX8/Yu6/8pR49q5xgVtO1Rj6/EbaGlT8xs8NrAQc0f1xGfrnbE+8Vfk3H2I8lIJvPyK4TOt0MgRmx59Ro4JCQkaR476EH5fFlTXDgZicf13FU3mnmNd0/DNRXunx+jqUohT57vhwW9//CPIyGmNH666YrzXTSNG17w9fiTB7evWGDnpkTIxAoBDWxk8hpbg+kUbAMC9X+4DAFy7VRglTpOnECDSckBR8/N1dnZGx44dVQ59k6OtrS0A9RFi7de19XUxiZFjXdPwzUmb1qUAgNtZjmp1WQ/tMGrgXbR1KMFvhVK1empcNrZy7D6bipY26lOtRQUtIPn9t+juL5kAgE6/J8eKMrHGc5otAz8h4+rqColEorajQe3XT96LfJJRR44ymQwJCQl45ZVXUFlZacxQjK6ismYZgo1VtVqdXauan42TXblBY6IaEgng0rUKTu1V/3jfvt4S139qhd6eNX/Y7v6SARupHLvecYF/t36Y8pwHZnv1QsrR1sYI2+TULuWp92jA5GhlZQVPT0+cOnVKeYkNAF9//TVsbW3Rt2/fes83anKsbxq+ubmb3Rol5RYYOfAO/vzn07KFDIP71FyuWVo035G1qSkvFWPj4ppZzxmLcgHUXFaXlUhQUizB8m0ZWPpBBqxbKbDudTckf+FgzHBNgxH2kFmwYAEuX76MJUuW4MyZM/jwww+xe/duzJ8/H9bW1vWea9TL6vqm4ZsbmVyCpH/3Q9jky3hrzrcQqqfguY75mOP/E1pa1SRFucJkbhE3axVlIrw9uwtuX7fGjMhceHjVjBwnho+FvPBtTA7NU7b1nlKI+WN6In5tB4yeWgiJxFhRG58x3srj5eWFuLg4bNu2DQsXLkS7du2wYsUKhIWFaT3XqMmxvmn45mjvlwMhtanCtDG/QMifhN1rgO9/dsX+rzwQMe0nFJdyKY+xlRRJsCakC375SQq/mfkIXZWtrJsUMQ6KnIUq7a2sBfhMK8Q/P2iPjBst0aVXM56sEYSaQ1ubpxQQEICAgAC1cl9fX/j6+urdn0lMyDyrg/HzjR1CgxLkeYD8LiBxxgg/Fwwf9gFQehkJu9ZAJDKHBLnc2AE8lcKHRYh59V38+p+7eCl8LKJ2zlNbJiJur76qwKHr1wDiUdnyC4jb9zBQtKaHuw8aQeDcj5HzsNjYYTwznxduIb/IBv+50QFnjy3HiMk1z4PGLT+GFi0csWDeNiNH2DDO7Npl7BD0VlYiRszU5/DrLzYImPcQ89/ZBCF3k/IWWV62Bd4M9sPIiT9h1tJclXMzLrsAaIu/tPKHIqeJ3TeWuEDcNqVBumpqL7vlTSwTEjj2GqJe/QES8R9/Pof2y4BHt1wc/ba3ESOj7W92xK+/2MB/7m+Y/84Dtfo2ztUoLSrDyQQnlD7+49fqYZYF/p3kiP7DH8PxL00sMTY44Y9L67oOE3rbrVmMHM3F/q/7Y21EMtYt+hpCWWeE+/+EQN//4sdrHfHvH58zdnjNVsZNK5z+whGt7ORw71OO04fUZ559phViUdwcvBOwEUsmd8OEoHyUl0hw7NM2ELcQsOj9+0aI3LQ0tZEjk6MJ+b/LXfC3T8bgtfH/gfB4Hca80AIHvvZAwlfPQyFwkG8sV8/VLLwvLZZg8xLNb8f3mVaI4f6D8fae2zgQ1w673+sAq5YKeHiVIDQmG67dmvc6XgDms00CGcc3P7njm5/ccfbYcrwavtHY4RCAl0Py8XJIvk5th40vxrDxTf/+d2NoaiNHkxmOBAQEID09He3btzd2KETUGOSCboeJ4MiRiAyiqY0cmRyJyEB0WARuQjcdmRyJyDB02UDLdHIjkyMRGQhnq4mI1InkgEjLhItIXm+1QTE5EpFBiAQBIi33HLXVGxKTIxEZBi+riYg04Ww1EZEarnMkIve3vHYAAA4iSURBVNKkkV9229CYHInIIERyQYfZaiZHImpuOCFDRKSOS3mIiDTibDURkTrF74e2NiaCyZGIDIKX1UREmigEQKFlaKhgciSi5oaX1URE6kTQ4bKaEzJE1OzwCRkiIg2YHImINNBld0E+PkhEzY4OS3k4ciSi5oeX1UREGgjQvo7RdHIjkyMRGQhHjkREGjA5EhFpIFfUHNramAgmRyIyDEFRc2hrYyKYHInIQPg+RyIidQpon602nYEjkyMRGUgjTMjIZDIMHDgQlZWVKuU2Nja4cuWKvhGqYHIkIsNohOR4584dVFZWIjY2Fm5ubspysVj8FAGqYnIkIsOQy2sObW30kJaWBrFYDD8/P1hbWz9DcOqYHInIQBp+QiY1NRWurq4NnhgB4NnHnkREuqi9rNZ26CE9PR2WlpaYM2cOBgwYgBdeeAFr1qxBSUnJM4fLkSMRGYYes9XZ2dmQP3GJbWdnBzs7O5WytLQ0lJSUIDAwEBEREbh27Rri4uJw584d7N27FyKR6KnDZXIkIsMQFBB0XAQeFBSErKwslapFixYhMjJSpWzLli2wt7dHjx49AAAvvPACnJycsHz5cvzwww8YPnz4U4fL5EhEhqHH44MJCQkaR45PGjx4sFqZt7c3gJpRJZMjEZk+QaF9a9bfR47Ozs5au8vPz8c333yDoUOHolOnTsryiooKAICDg8PTxwpOyBCRoTTwhIxIJMKaNWvwz3/+U6X8xIkTkEgkGDRo0DOFy5EjERmEoBAgaBk5CtombP7E0dERQUFB2LdvH6RSKTw9PXHp0iXs3LkTQUFB6Ny58zPFy+RIRIbRCE/IrFy5Eu3atcOhQ4ewa9cutGvXDosXL8bcuXOfIdAaTI5EZBgKQYelPPolRwsLC4SHhyM8PPwZAtOMyZGIDEJQyCFoeTxQUOj3+GBjYnIkIsMQBB1edsv3OTaotk5SY4fQKNr/RX1dl1mQuBg7gsZjbp9N3L7BunLq4KB1wsWpw7Mtv2lIIkEwoVRNRGQiuM6RiEgDJkciIg2YHImINGByJCLSgMmRiEgDJkciIg2YHImINGByJCLSgMmRiEgDJkcTc/z4cbz00kvw8PDAhAkTcPToUWOHRDpKTU1Fnz59kJOTY+xQqAEwOZqQkydPYtmyZRg+fDh27NiBwYMHY+XKlfjqq6+MHRppcfv2bcyfPx8ymczYoVAD4bPVJsTX1xd9+/bFli1blGVvvPEG0tPTcfLkSSNGRnWRyWRITEzE5s2bYWFhgUePHuHMmTNo377hXthAxsGRo4nIzMxERkYGxo0bp1Lu5+eH27dvIzMz00iRUX0uXbqETZs2ISwsDMuWLTN2ONSAmBxNxO3btwEAXbp0USmv3Qfjzp07Bo+JtHN3d0dycjIWLVoEiURi7HCoAZnF+xzNwePHjwEAUqnquylbtWoFACgpKTF4TKRdmzZtjB0CNRKOHE1E7a1fkUiksVws5v8qIkPib5yJsLW1BaA+QiwtLVWpJyLDYHI0EbX3GjMyMlTK7927p1JPRIbB5GgiOnfujI4dO6qtaTx16hTc3NzQoUMHI0VG1DxxQsaELFy4EDExMbC3t4e3tze++eYbnDx5UmXdIxEZBpOjCQkICEBVVRX27NmDgwcPolOnToiNjcXEiRONHRpRs8MnZIiINOA9RyIiDZgciYg0YHIkItKAyZGISAMmRyIiDZgciYg0YHI0EatWrUKPHj1Ujl69emHgwIEIDAzEkSNHDBLHmDFjEBwcrPw6ODgYY8aM0bufkpISFBQUNFhctT+fZ23TkOcZqj8yDi4CNzExMTFwcHAAUPNGnpKSEhw7dgyrVq1CYWEhwsLCDBpPREQEysvL9Trn2rVrWLBgATZt2oQhQ4Y0UmREjYvJ0cSMHTsWHTt2VCl75ZVXMHHiROzYsQOzZs2CpaWlweIZPny43ufcuHEDDx8+bIRoiAyHl9VNQMuWLTFmzBiUlJTg5s2bxg6HqFlgcmwial+CK5fLAdTcG1y9ejXefPNN9OvXDyNHjlTe47ty5QpCQ0MxYMAADBgwAGFhYbh69apanydOnMCUKVPg4eGBl19+GefPn1dro+me46+//oqoqCgMGTIEgwYNQnBwMC5evAgAiIuLQ0xMDAAgJCRE5dycnBysWLECQ4cORb9+/eDv749jx46pfc9r164hLCwMAwYMwIgRI7B3796n+ZEBAM6dO4e5c+diyJAh6NOnD0aMGIE1a9aguLhYre2VK1cwbdo09OvXD+PGjcNnn32m1kbXz0BNHy+rmwCFQoELFy7A0tIS7u7uyvIvv/wSXbp0wV//+lfk5eXB0dER33//PebPn4+ePXsiKioKVVVVOHz4MIKCgvDpp5/C09MTAHD48GHExMRgwIABWL58Oe7du4eIiAgoFAq4uLjUGcvdu3cxffp0tGjRArNmzYKjoyMOHDiA0NBQJCQkwNfXF7/99hsSExMRERGBfv36AQByc3MRGBgIQRAQHBwMe3t7nD59GsuXL8fDhw8xd+5cAMDNmzcRHBwMOzs7vP7666iursaOHTuUfxT08d133yE8PBwDBw7E4sWLIRKJ8P333yMxMRHV1dVYt26dSvuwsDCMHTsWAQEBSE5Oxrp16/D48WNERkbq9RnITAhkElauXCl0795d+OWXX4T8/HwhPz9fePjwoXDlyhUhKipK6N69u/D+++8r248ePVro2bOncO/ePWWZXC4XfHx8hJkzZwoymUxZXlpaKvj6+gpTpkwRBEEQZDKZ4OXlJUybNk2oqqpStjt06JDQvXt3YdasWcqyWbNmCaNHj1Z+HRUVJXh4eAh3795VlhUUFAiDBg0SFi9erNLP+fPnVT7f4MGDhdzcXJXPvXTpUqFv375CXl6eIAiCEBkZKTz//PPCgwcPlG1u3bol9O3bV+jevbtOP8Nac+bMEUaPHi1UVlaqtJs+fbowYMAAtfNiY2NVfpYhISFC3759hYKCAr0+w5NxUNPEy2oTM3XqVHh5ecHLywsvvvgiZsyYgdOnTyM4OBjR0dEqbV1dXeHq6qr8+vr168jMzMTYsWNRVFSEgoICFBQUoKKiAqNHj0ZqaipycnLwyy+/ID8/HwEBAbCwsFCeP2XKFNjb29cZm0KhwJkzZzBq1CjlrogA4ODggP3792P16tV1npecnAxPT0+0aNFCGVdBQQHGjRuHqqoqfP/991AoFDh79ixGjRoFZ2dn5fnu7u548cUX9f5Zfvzxxzh06JDKBFZhYSGkUinKysrU2v955CcWizFr1ixUVVXhhx9+0PkzkPngZbWJ2bhxo3JHO7FYDDs7O7i7u8PKykqtrZOTk8rXtVssbNiwARs2bNDYf3Z2NnJycgBAJbECgEQiUUl6T3r06BHKyso0tunevXud5xUWFuLx48dITk5GcnJynXHV9v9kXADQtWtXfPPNN3V+D00kEgkyMzOxdetW3Lp1CxkZGcjNzdXYtnXr1nB0dFQp69SpEwAgKytL589A5oPJ0cQMHDhQbSlPXZ7cJ1mhUAAAoqKi8Pzzz2s8p2vXrsoEUVlZqVZf24cmtff99N0JsfY8Pz8/zJw5U2Ob2kT0NHHV5cCBA3j77bfRpUsXeHp6Yty4cejfvz/27duHf/3rXyptn9z1EVDd+VHfz0BNH5OjGamdSLGxscGwYcNU6q5evYqioiK0bNlS+Ut89+5dlTaCICArKwvdunXT2L+DgwNatmyp3PTrz3bv3o28vDysXLlSrc7R0RHW1taQyWRqcT148ADXr1+HtbU1HBwcIJVK1eICgPv379f5uTWprKzE+vXrMWTIEOzZswctWvzxT33r1q1q7YuKilBSUqKyb3htHK6urjp/BjIfvOdoRvr27Yu2bdti3759yi1dgZpH+d544w3ExMRAIpGgd+/ecHFxweeff67y9MuXX36JwsLCOvtv0aIFhg8fjjNnzqhcQhYVFWH37t3Ky/rakWXtaK9FixYYOXIkzpw5g7S0NJU+169fj4ULF6KwsBAikQi+vr44e/Ysbty4oWxz//59pKSk6PWzqKioQHl5Odzc3FQSY2pqKi5cuAAAkMlkynKFQoEvvvhC+bVMJsM//vEP2NjYwMvLS+fPQOaDI0czYmFhgbfeegtvvPEGAgIC8Morr8DKygoHDx7EgwcPsGnTJmWieOutt7Bw4ULMmDED06ZNQ25uLhISEtC6det6v0d0dDQCAwMRGBiIoKAgSKVSJCUloaysDG+88QYAKO/dff7558jLy8OkSZOwbNky/PjjjwgKCkJQUBA6dOiAlJQUfPvtt5gxY4ZytBoVFYWUlBQEBwdj9uzZkEgk2LdvH1q1aoWqqiqdfxb29vbo378/Dh8+DKlUii5duuDmzZs4ePCgMnmXlpYqJ6Csra2xbds2ZGdnw9XVFSdOnMCVK1fw9ttvK/cM1/UzkHlgcjQzfn5+2LNnD/7+97/jo48+glgsRrdu3fD3v/8do0ePVrYbPXo0Pv74Y8TFxeGDDz5Au3bt8N577yEhIaHe/t3d3ZGYmIgPPvgA8fHxEIvF8PDwQGxsrDI5eHl5YcKECfj2229x/vx5jBs3Dq6urkhKSsK2bduUybRTp06IiYlRedGFs7MzPv/8c2zYsAHx8fGwtLREYGAggJrZZ31s3boV69atw6FDh1BVVQUXFxfMmzcP7u7uiIyMxPnz5+Hn5wcAsLOzQ2xsLN5//30kJCSgc+fO2LhxIyZPnqzsT9fPQOaBG2wREWnAe45ERBowORIRacDkSESkAZMjEZEGTI5ERBowORIRacDkSESkAZMjEZEGTI5ERBowORIRafD/iVqt2i0qrJwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting confusion matrix using Scikit-Learn\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "plot_confusion_matrix(clf, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification Report**\n",
    "\n",
    "The classification report is about key metrics in a classification problem.\n",
    "\n",
    "https://medium.com/@kohlishivam5522/understanding-a-classification-report-for-your-machine-learning-model-88815e2ce397"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.85      0.78        27\n",
      "           1       0.86      0.74      0.79        34\n",
      "\n",
      "    accuracy                           0.79        61\n",
      "   macro avg       0.79      0.79      0.79        61\n",
      "weighted avg       0.80      0.79      0.79        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.2 Regression model evaluation metrics\n",
    "\n",
    "1. R^2 (pronounced r-squared) or coefficient of determination.\n",
    "2. Mean absolute error (MAE)\n",
    "3. Mean squared error (MSE)\n",
    "\n",
    "**R^2**\n",
    "\n",
    "Compares the model's predictions with the mean of the targets. Values can range from negative infinity (very poor model) to 1. For example, if all our model predicts is mean of targets, it's R^2 value would be 0. And if our model perfectly predicts a range of numbers its R^2 is 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9033142910210461"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setting a random seed\n",
    "np.random.seed(27)\n",
    "\n",
    "# Getting the data ready\n",
    "X = boston_df.drop('target', axis = 1)\n",
    "y = boston_df['target']\n",
    "\n",
    "# Splitting data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "# Instantiating and fitting the model\n",
    "model = RandomForestRegressor().fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9033142910210461"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluating the model (by defalut it is measured using R^2)\n",
    "model.score(X_test, y_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9033142910210461"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using R^2 metric\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "r2_score(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean absolute error (MAE)**\n",
    "\n",
    "MAE is the average of the absolute difference between the predicted values and actual values. It gives an idea of how wrong our model's predictions are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0334705882352933"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "mean_absolute_error(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual values</th>\n",
       "      <th>Predicted values</th>\n",
       "      <th>Difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>427</th>\n",
       "      <td>10.9</td>\n",
       "      <td>16.750</td>\n",
       "      <td>-5.850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>33.8</td>\n",
       "      <td>32.239</td>\n",
       "      <td>1.561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>18.3</td>\n",
       "      <td>18.817</td>\n",
       "      <td>-0.517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>5.0</td>\n",
       "      <td>7.622</td>\n",
       "      <td>-2.622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>19.6</td>\n",
       "      <td>19.076</td>\n",
       "      <td>0.524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>27.5</td>\n",
       "      <td>17.039</td>\n",
       "      <td>10.461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>20.1</td>\n",
       "      <td>21.437</td>\n",
       "      <td>-1.337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>23.5</td>\n",
       "      <td>23.847</td>\n",
       "      <td>-0.347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>10.5</td>\n",
       "      <td>8.923</td>\n",
       "      <td>1.577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19.602</td>\n",
       "      <td>3.398</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Actual values  Predicted values  Difference\n",
       "427           10.9            16.750      -5.850\n",
       "260           33.8            32.239       1.561\n",
       "115           18.3            18.817      -0.517\n",
       "398            5.0             7.622      -2.622\n",
       "131           19.6            19.076       0.524\n",
       "..             ...               ...         ...\n",
       "409           27.5            17.039      10.461\n",
       "241           20.1            21.437      -1.337\n",
       "65            23.5            23.847      -0.347\n",
       "386           10.5             8.923       1.577\n",
       "132           23.0            19.602       3.398\n",
       "\n",
       "[102 rows x 3 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data = {'Actual values': y_test,\n",
    "                   'Predicted values': y_preds,\n",
    "                   'Difference': y_test - y_preds})\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean squared error (MSE)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.759936745098026"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "mean_squared_error(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.759936745098026"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate MSE by hand\n",
    "square = np.square(df['Difference']) # Squaring the difference \n",
    "square.mean() # Taking the mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When to use which parameter :\n",
    "https://towardsdatascience.com/metrics-to-understand-regression-models-in-plain-english-part-1-c902b2f4156f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.3 Using `scoring` parameter\n",
    "\n",
    "#### On classification model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(27)\n",
    "\n",
    "X = heart_disease.drop('target', axis=1)\n",
    "y = heart_disease['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)\n",
    "\n",
    "clf = RandomForestClassifier().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7868852459016393"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'accuracy'**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.80327869, 0.90163934, 0.80327869, 0.83333333, 0.78333333])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using scoring parameter\n",
    "cv_acc = cross_val_score(clf, X, y, scoring = 'accuracy')\n",
    "cv_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"accuracy\" as scoring parameter is: 82.50%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"accuracy\" as scoring parameter is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'precision'**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_precision = cross_val_score(clf, X_test, y_test, scoring = 'precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"precision\" as scoring parameter is: 82.14%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"precision\" as scoring parameter is: {np.mean(cv_precision)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'recall'**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_recall = cross_val_score(clf, X, y, scoring = 'recall')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"recall\" as scoring parameter is: 86.67%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"recall\" as scoring parameter is: {np.mean(cv_recall)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'f1'**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_acc = cross_val_score(clf, X, y, scoring = 'f1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"f1\" as scoring parameter is: 84.65%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"f1\" as scoring parameter is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### On regression model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "np.random.seed(27)\n",
    "\n",
    "X = boston_df.drop('target', axis = 1)\n",
    "y = boston_df['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\n",
    "\n",
    "model = RandomForestRegressor().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9033142910210461"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'r2' (r-squared)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_r2 = cross_val_score(model, X, y, scoring = 'r2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"r2\" as scoring parameter is: 62.98%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"r2\" as scoring parameter is: {np.mean(cv_r2)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'neg_mean_absolute_error' (MAE)**\n",
    "\n",
    "All scorer objects follow the convention that higher return values are better than lower return values. Thus metrics which measure the distance between the model and the data, like metrics. mean_squared_error , are available as neg_mean_squared_error which return the negated value of the metric.\n",
    "\n",
    "In simple words, higher positive values shows that the score is good and thus can be deceiving therefore a negated value is returned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_mae = cross_val_score(model, X, y, scoring = 'neg_mean_absolute_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"neg_mean_absolute_error\" as scoring parameter is: -303.13%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"neg_mean_absolute_error\" as scoring parameter is: {np.mean(cv_mae)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**scoring = 'neg_mean_squared_error (RMSE)'**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_mse = cross_val_score(model, X, y, scoring = 'neg_mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy using \"neg_mean_squared_error\" as scoring parameter is: -2186.65%\n"
     ]
    }
   ],
   "source": [
    "print(f'The cross-validated accuracy using \"neg_mean_squared_error\" as scoring parameter is: {np.mean(cv_mse)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.3 Using different evaluation metrics as Scikit-Learn functions\n",
    "\n",
    "**Classification evaluation functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 78.69%\n",
      "Precision: 86.21%\n",
      "Recall: 73.53%\n",
      "F1: 79.37%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(27)\n",
    "\n",
    "X = heart_disease.drop('target', axis = 1)\n",
    "y = heart_disease['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\n",
    "\n",
    "clf = RandomForestClassifier().fit(X_train, y_train)\n",
    "\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "# Evaluating the classifier \n",
    "print(f'Accuracy: {accuracy_score(y_test, y_preds)*100:.2f}%')\n",
    "print(f'Precision: {precision_score(y_test, y_preds)*100:.2f}%')\n",
    "print(f'Recall: {recall_score(y_test, y_preds)*100:.2f}%')\n",
    "print(f'F1: {f1_score(y_test, y_preds)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Regression evaluation metrics**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2: 0.9033142910210461\n",
      "MAE: 2.0334705882352933\n",
      "MSE: 8.759936745098026\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(27)\n",
    "\n",
    "X = boston_df.drop('target', axis = 1)\n",
    "y = boston_df['target']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\n",
    "\n",
    "model = RandomForestRegressor().fit(X_train, y_train)\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "\n",
    "# Evaluating the regressor\n",
    "print(f'R^2: {r2_score(y_test, y_preds)}')\n",
    "print(f'MAE: {mean_absolute_error(y_test, y_preds)}')\n",
    "print(f'MSE: {mean_squared_error(y_test, y_preds)}')\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}