RandomForestClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load libraries\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from common_functions import preprocess_data, cluster_categorical\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, RandomizedSearchCV, GridSearchCV\n",
    "from IPython.display import display\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_columns = [\n",
    "    \"Age\",\n",
    "    \"Workclass\",\n",
    "    \"final weight\",\n",
    "    \"Education\",\n",
    "    \"Education-Num\",\n",
    "    \"Marital Status\",\n",
    "    \"Occupation\",\n",
    "    \"Relationship\",\n",
    "    \"Ethnic group\",\n",
    "    \"Sex\",\n",
    "    \"Capital Gain\",\n",
    "    \"Capital Loss\",\n",
    "    \"Hours per week\",\n",
    "    \"Country\",\n",
    "    \"Income\",\n",
    "]\n",
    "\n",
    "df = pd.read_csv(\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\", header=None, names=adult_columns)\n",
    "df = df.replace(to_replace= ' ?', value = np.nan)\n",
    "\n",
    "TARGET = 'Income'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32561, 15)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Workclass</th>\n",
       "      <th>final weight</th>\n",
       "      <th>Education</th>\n",
       "      <th>Education-Num</th>\n",
       "      <th>Marital Status</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Relationship</th>\n",
       "      <th>Ethnic group</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Capital Gain</th>\n",
       "      <th>Capital Loss</th>\n",
       "      <th>Hours per week</th>\n",
       "      <th>Country</th>\n",
       "      <th>Income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age          Workclass  final weight   Education  Education-Num   \n",
       "0   39          State-gov         77516   Bachelors             13  \\\n",
       "1   50   Self-emp-not-inc         83311   Bachelors             13   \n",
       "2   38            Private        215646     HS-grad              9   \n",
       "3   53            Private        234721        11th              7   \n",
       "4   28            Private        338409   Bachelors             13   \n",
       "\n",
       "        Marital Status          Occupation    Relationship Ethnic group   \n",
       "0        Never-married        Adm-clerical   Not-in-family        White  \\\n",
       "1   Married-civ-spouse     Exec-managerial         Husband        White   \n",
       "2             Divorced   Handlers-cleaners   Not-in-family        White   \n",
       "3   Married-civ-spouse   Handlers-cleaners         Husband        Black   \n",
       "4   Married-civ-spouse      Prof-specialty            Wife        Black   \n",
       "\n",
       "       Sex  Capital Gain  Capital Loss  Hours per week         Country  Income  \n",
       "0     Male          2174             0              40   United-States   <=50K  \n",
       "1     Male             0             0              13   United-States   <=50K  \n",
       "2     Male             0             0              40   United-States   <=50K  \n",
       "3     Male             0             0              40   United-States   <=50K  \n",
       "4   Female             0             0              40            Cuba   <=50K  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Income</th>\n",
       "      <th>&lt;=50K</th>\n",
       "      <th>&gt;50K</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Education</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HS-grad</th>\n",
       "      <td>8826</td>\n",
       "      <td>1675</td>\n",
       "      <td>10501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Some-college</th>\n",
       "      <td>5904</td>\n",
       "      <td>1387</td>\n",
       "      <td>7291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>above-grad</th>\n",
       "      <td>5981</td>\n",
       "      <td>4535</td>\n",
       "      <td>10516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>under-grad</th>\n",
       "      <td>4009</td>\n",
       "      <td>244</td>\n",
       "      <td>4253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>24720</td>\n",
       "      <td>7841</td>\n",
       "      <td>32561</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Income         <=50K   >50K    All\n",
       "Education                         \n",
       "HS-grad         8826   1675  10501\n",
       "Some-college    5904   1387   7291\n",
       "above-grad      5981   4535  10516\n",
       "under-grad      4009    244   4253\n",
       "All            24720   7841  32561"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Cluster Education to 4 categories: 1)under graduates, 2)high school graduates, 3)some college  and 4)above \n",
    "\n",
    "def cluster_education(df):\n",
    "    df.loc[\n",
    "        lambda x: x[\"Education-Num\"].between(0, 8, \"both\"), \"Education\"\n",
    "    ] = \"under-grad\"\n",
    "\n",
    "    df.loc[\n",
    "        lambda x: x[\"Education-Num\"] == 9, \"Education\"\n",
    "    ] = \"HS-grad\"\n",
    "\n",
    "    df.loc[\n",
    "        lambda x: x[\"Education-Num\"] == 10, \"Education\"\n",
    "    ] = \"Some-college\"\n",
    "\n",
    "    df.loc[\n",
    "        lambda x: x[\"Education-Num\"].between(11, 16, 'both'), \"Education\"\n",
    "    ] = \"above-grad\"\n",
    "\n",
    "cluster_education(data)\n",
    "display(pd.crosstab(data[\"Education\"], data[TARGET], margins=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Workclass</th>\n",
       "      <th>final weight</th>\n",
       "      <th>Education</th>\n",
       "      <th>Education-Num</th>\n",
       "      <th>Marital Status</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Relationship</th>\n",
       "      <th>Ethnic group</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Capital Gain</th>\n",
       "      <th>Capital Loss</th>\n",
       "      <th>Hours per week</th>\n",
       "      <th>Country</th>\n",
       "      <th>Income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13136</th>\n",
       "      <td>41</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>173051</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age   Workclass  final weight  Education  Education-Num Marital Status   \n",
       "13136   41   Local-gov        173051          1             10       Divorced  \\\n",
       "\n",
       "           Occupation Relationship Ethnic group      Sex  Capital Gain   \n",
       "13136   Other-service    Unmarried        White   Female             0  \\\n",
       "\n",
       "       Capital Loss  Hours per week         Country  Income  \n",
       "13136             0              45   United-States   <=50K  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# apply ordinal encording to Education-cluster\n",
    "\n",
    "scale_mapper = {'under-grad':0, 'Some-college':1, 'HS-grad':2, 'above-grad':3}\n",
    "# data_ordinal = data[\"Education\"].replace(scale_mapper)\n",
    "# data_ordinal.head()\n",
    "data[\"Education\"] = data[\"Education\"].replace(scale_mapper)\n",
    "data.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Workclass</th>\n",
       "      <th>final weight</th>\n",
       "      <th>Education</th>\n",
       "      <th>Marital Status</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Relationship</th>\n",
       "      <th>Ethnic group</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Capital Gain</th>\n",
       "      <th>Capital Loss</th>\n",
       "      <th>Hours per week</th>\n",
       "      <th>Country</th>\n",
       "      <th>Income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>77516</td>\n",
       "      <td>3</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>83311</td>\n",
       "      <td>3</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>215646</td>\n",
       "      <td>2</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Private</td>\n",
       "      <td>234721</td>\n",
       "      <td>0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>Private</td>\n",
       "      <td>338409</td>\n",
       "      <td>3</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>Cuba</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32556</th>\n",
       "      <td>27</td>\n",
       "      <td>Private</td>\n",
       "      <td>257302</td>\n",
       "      <td>3</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32557</th>\n",
       "      <td>40</td>\n",
       "      <td>Private</td>\n",
       "      <td>154374</td>\n",
       "      <td>2</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32558</th>\n",
       "      <td>58</td>\n",
       "      <td>Private</td>\n",
       "      <td>151910</td>\n",
       "      <td>2</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32559</th>\n",
       "      <td>22</td>\n",
       "      <td>Private</td>\n",
       "      <td>201490</td>\n",
       "      <td>2</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32560</th>\n",
       "      <td>52</td>\n",
       "      <td>Self-emp-inc</td>\n",
       "      <td>287927</td>\n",
       "      <td>2</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>15024</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30162 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Age          Workclass  final weight  Education       Marital Status   \n",
       "0       39          State-gov         77516          3        Never-married  \\\n",
       "1       50   Self-emp-not-inc         83311          3   Married-civ-spouse   \n",
       "2       38            Private        215646          2             Divorced   \n",
       "3       53            Private        234721          0   Married-civ-spouse   \n",
       "4       28            Private        338409          3   Married-civ-spouse   \n",
       "...    ...                ...           ...        ...                  ...   \n",
       "32556   27            Private        257302          3   Married-civ-spouse   \n",
       "32557   40            Private        154374          2   Married-civ-spouse   \n",
       "32558   58            Private        151910          2              Widowed   \n",
       "32559   22            Private        201490          2        Never-married   \n",
       "32560   52       Self-emp-inc        287927          2   Married-civ-spouse   \n",
       "\n",
       "               Occupation    Relationship Ethnic group      Sex  Capital Gain   \n",
       "0            Adm-clerical   Not-in-family        White     Male          2174  \\\n",
       "1         Exec-managerial         Husband        White     Male             0   \n",
       "2       Handlers-cleaners   Not-in-family        White     Male             0   \n",
       "3       Handlers-cleaners         Husband        Black     Male             0   \n",
       "4          Prof-specialty            Wife        Black   Female             0   \n",
       "...                   ...             ...          ...      ...           ...   \n",
       "32556        Tech-support            Wife        White   Female             0   \n",
       "32557   Machine-op-inspct         Husband        White     Male             0   \n",
       "32558        Adm-clerical       Unmarried        White   Female             0   \n",
       "32559        Adm-clerical       Own-child        White     Male             0   \n",
       "32560     Exec-managerial            Wife        White   Female         15024   \n",
       "\n",
       "       Capital Loss  Hours per week         Country  Income  \n",
       "0                 0              40   United-States   <=50K  \n",
       "1                 0              13   United-States   <=50K  \n",
       "2                 0              40   United-States   <=50K  \n",
       "3                 0              40   United-States   <=50K  \n",
       "4                 0              40            Cuba   <=50K  \n",
       "...             ...             ...             ...     ...  \n",
       "32556             0              38   United-States   <=50K  \n",
       "32557             0              40   United-States    >50K  \n",
       "32558             0              40   United-States   <=50K  \n",
       "32559             0              20   United-States   <=50K  \n",
       "32560             0              40   United-States    >50K  \n",
       "\n",
       "[30162 rows x 14 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(['Education-Num'], axis = 1)\n",
    "data = data.dropna(how='any')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Income</th>\n",
       "      <th>&lt;=50K</th>\n",
       "      <th>&gt;50K</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Workclass</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Federal-gov</th>\n",
       "      <td>578</td>\n",
       "      <td>365</td>\n",
       "      <td>943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Local-gov</th>\n",
       "      <td>1458</td>\n",
       "      <td>609</td>\n",
       "      <td>2067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Private</th>\n",
       "      <td>17410</td>\n",
       "      <td>4876</td>\n",
       "      <td>22286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-emp-inc</th>\n",
       "      <td>474</td>\n",
       "      <td>600</td>\n",
       "      <td>1074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-emp-not-inc</th>\n",
       "      <td>1785</td>\n",
       "      <td>714</td>\n",
       "      <td>2499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>State-gov</th>\n",
       "      <td>935</td>\n",
       "      <td>344</td>\n",
       "      <td>1279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>22640</td>\n",
       "      <td>7508</td>\n",
       "      <td>30148</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Income              <=50K   >50K    All\n",
       "Workclass                              \n",
       " Federal-gov          578    365    943\n",
       " Local-gov           1458    609   2067\n",
       " Private            17410   4876  22286\n",
       " Self-emp-inc         474    600   1074\n",
       " Self-emp-not-inc    1785    714   2499\n",
       " State-gov            935    344   1279\n",
       "All                 22640   7508  30148"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = cluster_categorical(data)\n",
    "\n",
    "# drop instances with 'Without-pay' and 'Never-worked' values in Workclass\n",
    "\n",
    "data = data[~data['Workclass'].isin([' Never-worked', ' Without-pay'])]\n",
    "\n",
    "display(pd.crosstab(data[\"Workclass\"], data[TARGET], margins=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_features_list = ['Age', 'final weight', 'Capital Gain', 'Capital Loss', 'Hours per week']\n",
    "categorical_features_list = ['Workclass', 'Marital Status', 'Occupation', 'Relationship', 'Ethnic group', 'Sex', 'Country']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income</th>\n",
       "      <th>stand scaler__Age</th>\n",
       "      <th>stand scaler__final weight</th>\n",
       "      <th>stand scaler__Capital Gain</th>\n",
       "      <th>stand scaler__Capital Loss</th>\n",
       "      <th>stand scaler__Hours per week</th>\n",
       "      <th>onehot__Workclass_ Local-gov</th>\n",
       "      <th>onehot__Workclass_ Private</th>\n",
       "      <th>onehot__Workclass_ Self-emp-inc</th>\n",
       "      <th>onehot__Workclass_ Self-emp-not-inc</th>\n",
       "      <th>...</th>\n",
       "      <th>onehot__Occupation_ Tech-support</th>\n",
       "      <th>onehot__Occupation_ Transport-moving</th>\n",
       "      <th>onehot__Relationship_Not-in-Family</th>\n",
       "      <th>onehot__Ethnic group_ Asian-Pac-Islander</th>\n",
       "      <th>onehot__Ethnic group_ Black</th>\n",
       "      <th>onehot__Ethnic group_ Other</th>\n",
       "      <th>onehot__Ethnic group_ White</th>\n",
       "      <th>onehot__Sex_ Male</th>\n",
       "      <th>onehot__Country_Developing</th>\n",
       "      <th>Education</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>0</td>\n",
       "      <td>0.57633</td>\n",
       "      <td>-1.01002</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.662589</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Income  stand scaler__Age  stand scaler__final weight   \n",
       "285       0            0.57633                    -1.01002  \\\n",
       "\n",
       "     stand scaler__Capital Gain  stand scaler__Capital Loss   \n",
       "285                   -0.147449                   -0.218639  \\\n",
       "\n",
       "     stand scaler__Hours per week  onehot__Workclass_ Local-gov   \n",
       "285                     -0.662589                           0.0  \\\n",
       "\n",
       "     onehot__Workclass_ Private  onehot__Workclass_ Self-emp-inc   \n",
       "285                         1.0                              0.0  \\\n",
       "\n",
       "     onehot__Workclass_ Self-emp-not-inc  ...   \n",
       "285                                  0.0  ...  \\\n",
       "\n",
       "     onehot__Occupation_ Tech-support  onehot__Occupation_ Transport-moving   \n",
       "285                               0.0                                   0.0  \\\n",
       "\n",
       "     onehot__Relationship_Not-in-Family   \n",
       "285                                 1.0  \\\n",
       "\n",
       "     onehot__Ethnic group_ Asian-Pac-Islander  onehot__Ethnic group_ Black   \n",
       "285                                       0.0                          0.0  \\\n",
       "\n",
       "     onehot__Ethnic group_ Other  onehot__Ethnic group_ White   \n",
       "285                          0.0                          1.0  \\\n",
       "\n",
       "     onehot__Sex_ Male  onehot__Country_Developing  Education  \n",
       "285                0.0                         0.0          3  \n",
       "\n",
       "[1 rows x 33 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_preprocessed = preprocess_data(data, numerical_features_list,  categorical_features_list, education=False)\n",
    "data_preprocessed.sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_cv, y_train, y_cv = train_test_split(\n",
    "    data_preprocessed.drop(columns = [TARGET]), \n",
    "    data_preprocessed[TARGET], \n",
    "    test_size=0.2, \n",
    "    stratify=data_preprocessed.iloc[:, 0]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stand scaler__Age</th>\n",
       "      <th>stand scaler__final weight</th>\n",
       "      <th>stand scaler__Capital Gain</th>\n",
       "      <th>stand scaler__Capital Loss</th>\n",
       "      <th>stand scaler__Hours per week</th>\n",
       "      <th>onehot__Workclass_ Local-gov</th>\n",
       "      <th>onehot__Workclass_ Private</th>\n",
       "      <th>onehot__Workclass_ Self-emp-inc</th>\n",
       "      <th>onehot__Workclass_ Self-emp-not-inc</th>\n",
       "      <th>onehot__Workclass_ State-gov</th>\n",
       "      <th>...</th>\n",
       "      <th>onehot__Occupation_ Tech-support</th>\n",
       "      <th>onehot__Occupation_ Transport-moving</th>\n",
       "      <th>onehot__Relationship_Not-in-Family</th>\n",
       "      <th>onehot__Ethnic group_ Asian-Pac-Islander</th>\n",
       "      <th>onehot__Ethnic group_ Black</th>\n",
       "      <th>onehot__Ethnic group_ Other</th>\n",
       "      <th>onehot__Ethnic group_ White</th>\n",
       "      <th>onehot__Sex_ Male</th>\n",
       "      <th>onehot__Country_Developing</th>\n",
       "      <th>Education</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4949</th>\n",
       "      <td>-0.490039</td>\n",
       "      <td>1.430423</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3343</th>\n",
       "      <td>-1.023224</td>\n",
       "      <td>1.361684</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>0.339429</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4897</th>\n",
       "      <td>-0.566209</td>\n",
       "      <td>0.123234</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29864</th>\n",
       "      <td>0.804838</td>\n",
       "      <td>-0.045050</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29681</th>\n",
       "      <td>0.957176</td>\n",
       "      <td>0.673261</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>0.756936</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19988</th>\n",
       "      <td>-0.642378</td>\n",
       "      <td>0.073414</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4625</th>\n",
       "      <td>-0.947055</td>\n",
       "      <td>1.218215</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20958</th>\n",
       "      <td>-0.718547</td>\n",
       "      <td>-0.252072</td>\n",
       "      <td>-0.147449</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10005</th>\n",
       "      <td>0.881007</td>\n",
       "      <td>-0.298192</td>\n",
       "      <td>0.271429</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3313</th>\n",
       "      <td>1.033345</td>\n",
       "      <td>4.043335</td>\n",
       "      <td>0.890364</td>\n",
       "      <td>-0.218639</td>\n",
       "      <td>-0.078078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22344 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       stand scaler__Age  stand scaler__final weight   \n",
       "4949           -0.490039                    1.430423  \\\n",
       "3343           -1.023224                    1.361684   \n",
       "4897           -0.566209                    0.123234   \n",
       "29864           0.804838                   -0.045050   \n",
       "29681           0.957176                    0.673261   \n",
       "...                  ...                         ...   \n",
       "19988          -0.642378                    0.073414   \n",
       "4625           -0.947055                    1.218215   \n",
       "20958          -0.718547                   -0.252072   \n",
       "10005           0.881007                   -0.298192   \n",
       "3313            1.033345                    4.043335   \n",
       "\n",
       "       stand scaler__Capital Gain  stand scaler__Capital Loss   \n",
       "4949                    -0.147449                   -0.218639  \\\n",
       "3343                    -0.147449                   -0.218639   \n",
       "4897                    -0.147449                   -0.218639   \n",
       "29864                   -0.147449                   -0.218639   \n",
       "29681                   -0.147449                   -0.218639   \n",
       "...                           ...                         ...   \n",
       "19988                   -0.147449                   -0.218639   \n",
       "4625                    -0.147449                   -0.218639   \n",
       "20958                   -0.147449                   -0.218639   \n",
       "10005                    0.271429                   -0.218639   \n",
       "3313                     0.890364                   -0.218639   \n",
       "\n",
       "       stand scaler__Hours per week  onehot__Workclass_ Local-gov   \n",
       "4949                      -0.078078                           0.0  \\\n",
       "3343                       0.339429                           0.0   \n",
       "4897                      -0.078078                           0.0   \n",
       "29864                     -0.078078                           0.0   \n",
       "29681                      0.756936                           0.0   \n",
       "...                             ...                           ...   \n",
       "19988                     -0.078078                           0.0   \n",
       "4625                      -0.078078                           0.0   \n",
       "20958                     -0.078078                           0.0   \n",
       "10005                     -0.078078                           0.0   \n",
       "3313                      -0.078078                           0.0   \n",
       "\n",
       "       onehot__Workclass_ Private  onehot__Workclass_ Self-emp-inc   \n",
       "4949                          1.0                              0.0  \\\n",
       "3343                          1.0                              0.0   \n",
       "4897                          1.0                              0.0   \n",
       "29864                         1.0                              0.0   \n",
       "29681                         0.0                              1.0   \n",
       "...                           ...                              ...   \n",
       "19988                         1.0                              0.0   \n",
       "4625                          1.0                              0.0   \n",
       "20958                         1.0                              0.0   \n",
       "10005                         1.0                              0.0   \n",
       "3313                          0.0                              0.0   \n",
       "\n",
       "       onehot__Workclass_ Self-emp-not-inc  onehot__Workclass_ State-gov  ...   \n",
       "4949                                   0.0                           0.0  ...  \\\n",
       "3343                                   0.0                           0.0  ...   \n",
       "4897                                   0.0                           0.0  ...   \n",
       "29864                                  0.0                           0.0  ...   \n",
       "29681                                  0.0                           0.0  ...   \n",
       "...                                    ...                           ...  ...   \n",
       "19988                                  0.0                           0.0  ...   \n",
       "4625                                   0.0                           0.0  ...   \n",
       "20958                                  0.0                           0.0  ...   \n",
       "10005                                  0.0                           0.0  ...   \n",
       "3313                                   0.0                           0.0  ...   \n",
       "\n",
       "       onehot__Occupation_ Tech-support  onehot__Occupation_ Transport-moving   \n",
       "4949                                0.0                                   0.0  \\\n",
       "3343                                0.0                                   0.0   \n",
       "4897                                0.0                                   0.0   \n",
       "29864                               0.0                                   0.0   \n",
       "29681                               0.0                                   0.0   \n",
       "...                                 ...                                   ...   \n",
       "19988                               0.0                                   0.0   \n",
       "4625                                0.0                                   1.0   \n",
       "20958                               0.0                                   0.0   \n",
       "10005                               1.0                                   0.0   \n",
       "3313                                1.0                                   0.0   \n",
       "\n",
       "       onehot__Relationship_Not-in-Family   \n",
       "4949                                  0.0  \\\n",
       "3343                                  0.0   \n",
       "4897                                  1.0   \n",
       "29864                                 0.0   \n",
       "29681                                 0.0   \n",
       "...                                   ...   \n",
       "19988                                 0.0   \n",
       "4625                                  1.0   \n",
       "20958                                 0.0   \n",
       "10005                                 0.0   \n",
       "3313                                  0.0   \n",
       "\n",
       "       onehot__Ethnic group_ Asian-Pac-Islander  onehot__Ethnic group_ Black   \n",
       "4949                                        0.0                          0.0  \\\n",
       "3343                                        0.0                          0.0   \n",
       "4897                                        0.0                          1.0   \n",
       "29864                                       0.0                          0.0   \n",
       "29681                                       0.0                          0.0   \n",
       "...                                         ...                          ...   \n",
       "19988                                       0.0                          0.0   \n",
       "4625                                        0.0                          0.0   \n",
       "20958                                       0.0                          0.0   \n",
       "10005                                       0.0                          0.0   \n",
       "3313                                        0.0                          1.0   \n",
       "\n",
       "       onehot__Ethnic group_ Other  onehot__Ethnic group_ White   \n",
       "4949                           0.0                          1.0  \\\n",
       "3343                           0.0                          1.0   \n",
       "4897                           0.0                          0.0   \n",
       "29864                          0.0                          1.0   \n",
       "29681                          0.0                          1.0   \n",
       "...                            ...                          ...   \n",
       "19988                          0.0                          1.0   \n",
       "4625                           0.0                          1.0   \n",
       "20958                          0.0                          1.0   \n",
       "10005                          0.0                          1.0   \n",
       "3313                           0.0                          0.0   \n",
       "\n",
       "       onehot__Sex_ Male  onehot__Country_Developing  Education  \n",
       "4949                 1.0                         0.0          3  \n",
       "3343                 1.0                         0.0          3  \n",
       "4897                 0.0                         0.0          3  \n",
       "29864                1.0                         0.0          2  \n",
       "29681                1.0                         0.0          3  \n",
       "...                  ...                         ...        ...  \n",
       "19988                1.0                         0.0          3  \n",
       "4625                 1.0                         0.0          3  \n",
       "20958                1.0                         0.0          2  \n",
       "10005                1.0                         0.0          3  \n",
       "3313                 1.0                         0.0          3  \n",
       "\n",
       "[22344 rows x 32 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "o F1-score (cross-val) on train set: 0.7789488932755584\n",
      "o Classification report on train set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     16781\n",
      "           1       1.00      1.00      1.00      5563\n",
      "\n",
      "    accuracy                           1.00     22344\n",
      "   macro avg       1.00      1.00      1.00     22344\n",
      "weighted avg       1.00      1.00      1.00     22344\n",
      "\n",
      "o Classification report on test set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.92      0.90      4196\n",
      "           1       0.72      0.58      0.64      1391\n",
      "\n",
      "    accuracy                           0.84      5587\n",
      "   macro avg       0.79      0.75      0.77      5587\n",
      "weighted avg       0.83      0.84      0.83      5587\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# track f1-scores of models:\n",
    "scores = pd.DataFrame()\n",
    "\n",
    "# apply random forest classifier\n",
    "\n",
    "# initialize model\n",
    "rf_plain = RandomForestClassifier()\n",
    "\n",
    "def rand_forest_evaluate(model):\n",
    "    \n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # count f1-score on training\n",
    "    score = cross_val_score(model, X_train, y_train, cv=5, scoring='f1_macro')\n",
    "\n",
    "    # predict on train set\n",
    "    y_train_pred = model.predict(X_train)\n",
    "\n",
    "    # predict on test set\n",
    "    y_cv_pred = model.predict(X_cv)\n",
    "\n",
    "    # f1-score\n",
    "    print(f\"o F1-score (cross-val) on train set: {np.mean(score)}\")\n",
    "\n",
    "    # classification report\n",
    "    print(\"o Classification report on train set:\")\n",
    "    print(classification_report(y_train, y_train_pred))\n",
    "    print(\"o Classification report on test set:\")\n",
    "\n",
    "    print(classification_report(y_cv, y_cv_pred))\n",
    "    \n",
    "    return score\n",
    "\n",
    "scores['Plain RF'] = rand_forest_evaluate(rf_plain)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model highly overfits the data. Let's try to tweak RandomForest's hyperparameters to avoid overfitting:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': [True, False],\n",
      " 'class_weight': [{0: 1, 1: 1},\n",
      "                  {0: 1, 1: 1.5},\n",
      "                  {0: 1, 1: 2},\n",
      "                  {0: 1, 1: 3},\n",
      "                  'balanced'],\n",
      " 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]}\n"
     ]
    }
   ],
   "source": [
    "# Number of trees in the Random Forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 100, stop = 1000, num = 10)]\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(1, 10, num = 10)]\n",
    "\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "\n",
    "# Different class weights\n",
    "class_weight = [{0:1, 1:1}, {0:1, 1:1.5}, {0:1, 1:2}, {0:1, 1:3}, 'balanced']\n",
    "\n",
    "# Create a random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap, \n",
    "               'class_weight': class_weight}\n",
    "\n",
    "from pprint import pprint\n",
    "pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(estimator=RandomForestClassifier(), n_iter=100, n_jobs=-1,\n",
       "                   param_distributions={&#x27;bootstrap&#x27;: [True, False],\n",
       "                                        &#x27;class_weight&#x27;: [{0: 1, 1: 1},\n",
       "                                                         {0: 1, 1: 1.5},\n",
       "                                                         {0: 1, 1: 2},\n",
       "                                                         {0: 1, 1: 3},\n",
       "                                                         &#x27;balanced&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [1, 2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                      10],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300, 400,\n",
       "                                                         500, 600, 700, 800,\n",
       "                                                         900, 1000]},\n",
       "                   return_train_score=True, scoring=&#x27;f1&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(estimator=RandomForestClassifier(), n_iter=100, n_jobs=-1,\n",
       "                   param_distributions={&#x27;bootstrap&#x27;: [True, False],\n",
       "                                        &#x27;class_weight&#x27;: [{0: 1, 1: 1},\n",
       "                                                         {0: 1, 1: 1.5},\n",
       "                                                         {0: 1, 1: 2},\n",
       "                                                         {0: 1, 1: 3},\n",
       "                                                         &#x27;balanced&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [1, 2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                      10],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300, 400,\n",
       "                                                         500, 600, 700, 800,\n",
       "                                                         900, 1000]},\n",
       "                   return_train_score=True, scoring=&#x27;f1&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(estimator=RandomForestClassifier(), n_iter=100, n_jobs=-1,\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'class_weight': [{0: 1, 1: 1},\n",
       "                                                         {0: 1, 1: 1.5},\n",
       "                                                         {0: 1, 1: 2},\n",
       "                                                         {0: 1, 1: 3},\n",
       "                                                         'balanced'],\n",
       "                                        'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                      10],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'n_estimators': [100, 200, 300, 400,\n",
       "                                                         500, 600, 700, 800,\n",
       "                                                         900, 1000]},\n",
       "                   return_train_score=True, scoring='f1')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random = RandomizedSearchCV(estimator=RandomForestClassifier(), param_distributions=random_grid, scoring='f1', n_iter=100, n_jobs = -1, return_train_score=True)\n",
    "\n",
    "rf_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 300,\n",
       " 'min_samples_split': 5,\n",
       " 'min_samples_leaf': 4,\n",
       " 'max_depth': 10,\n",
       " 'class_weight': {0: 1, 1: 2},\n",
       " 'bootstrap': False}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets train the model again with best hyperparameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "o F1-score (cross-val) on train set: 0.7879564267802246\n",
      "o Classification report on train set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.85      0.89     16781\n",
      "           1       0.65      0.80      0.72      5563\n",
      "\n",
      "    accuracy                           0.84     22344\n",
      "   macro avg       0.79      0.83      0.80     22344\n",
      "weighted avg       0.86      0.84      0.85     22344\n",
      "\n",
      "o Classification report on test set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.85      0.88      4196\n",
      "           1       0.63      0.78      0.70      1391\n",
      "\n",
      "    accuracy                           0.83      5587\n",
      "   macro avg       0.77      0.81      0.79      5587\n",
      "weighted avg       0.85      0.83      0.84      5587\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_random_search = RandomForestClassifier(n_estimators = 600,\n",
    "                                        min_samples_split = 5,\n",
    "                                        min_samples_leaf = 2,\n",
    "                                        max_depth = 10,\n",
    "                                        class_weight = {0: 1, 1: 2},\n",
    "                                        bootstrap = False)\n",
    "\n",
    "scores['Random Search RF'] = rand_forest_evaluate(rf_random_search)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model still overfits a little bit. As the first hyperparameres search was quite rough, let's now try to make a more specific one around smaller amount of possible values. This time using the GridSearchCV:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': [True, False],\n",
       " 'class_weight': [{0: 1, 1: 1.4},\n",
       "  {0: 1, 1: 1.5},\n",
       "  {0: 1, 1: 1.6},\n",
       "  {0: 1, 1: 1.7},\n",
       "  {0: 1, 1: 1.8}],\n",
       " 'max_depth': [9, 10, 11],\n",
       " 'min_samples_leaf': [2, 3],\n",
       " 'min_samples_split': [4, 5, 6],\n",
       " 'n_estimators': [550, 600, 650]}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of trees in the Random Forest\n",
    "n_estimators = [550, 600, 650]\n",
    "\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [4, 5, 6]\n",
    "\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [2, 3]\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [9,10,11]\n",
    "\n",
    "# Different class weights\n",
    "class_weight = [{0:1, 1:1.4}, {0:1, 1:1.5}, {0:1, 1:1.6}, {0:1, 1:1.7}, {0:1, 1:1.8}]\n",
    "\n",
    "# Method of selecting samples for training each tree\n",
    "boorstrap = [True, False]\n",
    "\n",
    "# Create a random grid\n",
    "grid_params = {'bootstrap': boorstrap,\n",
    "               'class_weight': class_weight,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'n_estimators': n_estimators\n",
    "               }\n",
    "\n",
    "grid_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=4, estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "             param_grid={&#x27;bootstrap&#x27;: [True, False],\n",
       "                         &#x27;class_weight&#x27;: [{0: 1, 1: 1.4}, {0: 1, 1: 1.5},\n",
       "                                          {0: 1, 1: 1.6}, {0: 1, 1: 1.7},\n",
       "                                          {0: 1, 1: 1.8}],\n",
       "                         &#x27;max_depth&#x27;: [9, 10, 11], &#x27;min_samples_leaf&#x27;: [2, 3],\n",
       "                         &#x27;min_samples_split&#x27;: [4, 5, 6],\n",
       "                         &#x27;n_estimators&#x27;: [550, 600, 650]},\n",
       "             scoring=&#x27;f1&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=4, estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "             param_grid={&#x27;bootstrap&#x27;: [True, False],\n",
       "                         &#x27;class_weight&#x27;: [{0: 1, 1: 1.4}, {0: 1, 1: 1.5},\n",
       "                                          {0: 1, 1: 1.6}, {0: 1, 1: 1.7},\n",
       "                                          {0: 1, 1: 1.8}],\n",
       "                         &#x27;max_depth&#x27;: [9, 10, 11], &#x27;min_samples_leaf&#x27;: [2, 3],\n",
       "                         &#x27;min_samples_split&#x27;: [4, 5, 6],\n",
       "                         &#x27;n_estimators&#x27;: [550, 600, 650]},\n",
       "             scoring=&#x27;f1&#x27;)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=4, estimator=RandomForestClassifier(), n_jobs=-1,\n",
       "             param_grid={'bootstrap': [True, False],\n",
       "                         'class_weight': [{0: 1, 1: 1.4}, {0: 1, 1: 1.5},\n",
       "                                          {0: 1, 1: 1.6}, {0: 1, 1: 1.7},\n",
       "                                          {0: 1, 1: 1.8}],\n",
       "                         'max_depth': [9, 10, 11], 'min_samples_leaf': [2, 3],\n",
       "                         'min_samples_split': [4, 5, 6],\n",
       "                         'n_estimators': [550, 600, 650]},\n",
       "             scoring='f1')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid_search = GridSearchCV(estimator=RandomForestClassifier(), param_grid=grid_params, scoring='f1', n_jobs = -1, cv=4)\n",
    "rf_grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': False,\n",
       " 'class_weight': {0: 1, 1: 1.8},\n",
       " 'max_depth': 11,\n",
       " 'min_samples_leaf': 2,\n",
       " 'min_samples_split': 4,\n",
       " 'n_estimators': 600}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "o F1-score (cross-val) on train set: 0.7900788263755175\n",
      "o Classification report on train set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.88      0.89     16781\n",
      "           1       0.67      0.73      0.70      5563\n",
      "\n",
      "    accuracy                           0.84     22344\n",
      "   macro avg       0.79      0.81      0.80     22344\n",
      "weighted avg       0.85      0.84      0.84     22344\n",
      "\n",
      "o Classification report on test set:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.88      0.89      4196\n",
      "           1       0.66      0.72      0.69      1391\n",
      "\n",
      "    accuracy                           0.84      5587\n",
      "   macro avg       0.78      0.80      0.79      5587\n",
      "weighted avg       0.84      0.84      0.84      5587\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rf_grid_search = RandomForestClassifier(bootstrap = False,\n",
    "                            class_weight = {0: 1, 1: 1.8},\n",
    "                            max_depth = 8,\n",
    "                            min_samples_leaf = 2,\n",
    "                            min_samples_split = 6,\n",
    "                            n_estimators = 550)\n",
    "                            \n",
    "\n",
    "scores['Grid Search RF'] = rand_forest_evaluate(rf_grid_search)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Plain RF</th>\n",
       "      <th>Random Search RF</th>\n",
       "      <th>Grid Search RF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.766560</td>\n",
       "      <td>0.779645</td>\n",
       "      <td>0.782295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.772214</td>\n",
       "      <td>0.784785</td>\n",
       "      <td>0.782688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.797069</td>\n",
       "      <td>0.795619</td>\n",
       "      <td>0.796993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.783175</td>\n",
       "      <td>0.789068</td>\n",
       "      <td>0.793925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.775727</td>\n",
       "      <td>0.790665</td>\n",
       "      <td>0.794492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Plain RF  Random Search RF  Grid Search RF\n",
       "0  0.766560          0.779645        0.782295\n",
       "1  0.772214          0.784785        0.782688\n",
       "2  0.797069          0.795619        0.796993\n",
       "3  0.783175          0.789068        0.793925\n",
       "4  0.775727          0.790665        0.794492"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This model already looks better and does not overfit the data. Futhermore, we can see  around 3% of total f1-score improvement and 4% for the minority class"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}