Binary_Classification_E2E.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install the libraries you need\n",
    "\n",
    "! pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install shap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import all the libraries you'll be using\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import seaborn as sns\n",
    "import matplotlib as plt\n",
    "from datetime import date\n",
    "import datetime\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import boto3\n",
    "import datetime\n",
    "from sklearn.model_selection import train_test_split\n",
    "import shap\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset provided from Kaggle\n",
    "# We merge the the training and test dataset for now, as we'll run QA and transformations on all the data\n",
    "\n",
    "df_train = pd.read_csv('job change of data scientists/aug_train.csv')\n",
    "df_test = pd.read_csv('job change of data scientists/aug_test.csv')\n",
    "df = pd.concat([df_train, df_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of the data\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkout the features\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unique on enrollee id\n",
    "\n",
    "df['enrollee_id'].nunique() == df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Any person duplicates?\n",
    "\n",
    "df['duplicated'] = df.drop(columns = ['enrollee_id']).duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['duplicated'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df[df['duplicated'] == True].index, inplace = True)\n",
    "df.head(2)\n",
    "print(f'New df size is {df.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Descriptive stats\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check unique and null counts, does everything look as expected?\n",
    "\n",
    "for col in df.columns:\n",
    "    print(f' \\n {col.upper()} contains {df[col].nunique()} unique values and {df[col].isnull().sum()} nulls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop unlabeled target rows\n",
    "\n",
    "df.dropna(subset=['target'], how='all', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.shape[0])\n",
    "      \n",
    "df['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['company_size'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tidy up 10/49 to 10-49\n",
    "\n",
    "df['company_size'] = df['company_size'].apply(lambda x: '10-49' if x == '10/49' else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['company_size'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training hours distribution\n",
    "\n",
    "df['training_hours'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seaborn example\n",
    "\n",
    "sns.violinplot(data=df, x=\"education_level\", y=\"training_hours\", hue=\"target\", gridsize = 200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Are classes balanced?\n",
    "\n",
    "df['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Total number of nulls in dataset\n",
    "\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop columns we don't want to pass to model\n",
    "\n",
    "df_model = df.drop(['duplicated', 'enrollee_id'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Any categorical columns require numeric indicator variables (i.e. pandas dummies)\n",
    "\n",
    "\n",
    "cat_cols = []\n",
    "\n",
    "for col in df_model.columns:\n",
    "    print('\\n')\n",
    "    print(f'Column {col} is of data type {df_model[col].dtype}')   \n",
    "    if df_model[col].dtype == 'object':\n",
    "        print('Creating dummies for it')\n",
    "        cat_cols.append(col)\n",
    "df_model = pd.get_dummies(df_model, columns=cat_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Target col should be int 0 or 1\n",
    "\n",
    "df_model['target'] = pd.to_numeric(df_model['target'])\n",
    "df_model['target'] = df_model['target'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_model['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure dummies look good\n",
    "\n",
    "df_model.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See what features are most correlated to the dependent variable\n",
    "# How should we interpret this?\n",
    "\n",
    "def gen_corr_grid(df, depvar):\n",
    "    corr_dict = {\n",
    "            'variable': [],\n",
    "            'corr': []\n",
    "        }\n",
    "    for col in list(df.columns):\n",
    "        if df[col].dtype not in ['object', '<M8[ns]'] and col != depvar :\n",
    "            r = df[depvar].corr(df[col])   \n",
    "            corr_dict['variable'].append(col)\n",
    "            corr_dict['corr'].append(r)\n",
    "    return pd.DataFrame(corr_dict).sort_values(by=['corr'], ascending=False)\n",
    "\n",
    "\n",
    "df_corrs = gen_corr_grid(df_model, 'target')\n",
    "df_corrs.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create training and test data to pass to the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 80/20 split\n",
    "train, test = train_test_split(df_model, test_size=0.2)\n",
    "\n",
    "# Breakout identifies the feature cols so we break out our x (features) and y (target col) values in next cell\n",
    "breakout = list(set(train.columns) - set(['target']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_x_y(test_in, train_in, breakout, depvar):\n",
    "     \n",
    "\n",
    "    train = train_in.copy()\n",
    "    test = test_in.copy()\n",
    "\n",
    "    X_train = train[breakout].copy()\n",
    "    X_test = test[breakout].copy()\n",
    "    \n",
    "    y_train = train[depvar]\n",
    "    y_test = test[depvar]\n",
    "        \n",
    "    X_columns = X_train.columns\n",
    "    \n",
    "        \n",
    "    return X_train, X_test, y_train, y_test, X_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test, X_columns = split_x_y(test, train, breakout, 'target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confirm 80/20 split\n",
    "\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate 5 splits\n",
    "\n",
    "cv_sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=77)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model\n",
    "\n",
    "def fit_lgb(param_grid, X, y, cv):\n",
    "    \n",
    "    gbm = lgb.LGBMClassifier(objective='binary',\n",
    "                            feature_fraction= 0.9)\n",
    "    \n",
    "    grid_search = GridSearchCV(gbm, \n",
    "                               param_grid=param_grid,\n",
    "                               scoring='roc_auc',\n",
    "                               cv=cv,\n",
    "                               verbose=1,\n",
    "                              n_jobs=-1)   \n",
    "    \n",
    "    grid_search.fit(X=X, y=y.values.ravel())\n",
    "    \n",
    "    print(grid_search.best_score_)\n",
    "    print(grid_search.best_estimator_)\n",
    "    print(grid_search.best_params_)\n",
    "    return grid_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can test out other hyperparameter values to see if it improves performance\n",
    "\n",
    "param_grid = { 'learning_rate': [0.05],\n",
    "                'n_estimators': [400],\n",
    "                'max_depth': [5, 7],\n",
    "                'min_child_samples': [50, 200, 600],\n",
    "                'num_leaves': [31, 63] }\n",
    "\n",
    "grid_search = fit_lgb(param_grid, X_train, y_train, cv_sss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pass test data to our trained model. \n",
    "\n",
    "y_pred = grid_search.best_estimator_.predict_proba(X_test)[:,1]\n",
    "auc = metrics.roc_auc_score(y_true=y_test, y_score=y_pred)\n",
    "print(auc)\n",
    "\n",
    "fi = pd.Series(grid_search.best_estimator_.feature_importances_, index=X_columns).sort_values(ascending=False)\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explainability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using SHAP values\n",
    "# Fit the explainer\n",
    "explainer = shap.Explainer(grid_search.best_estimator_.predict, X_test)\n",
    "# Calculates the SHAP values - this takes some time\n",
    "shap_values = explainer(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features ordered from highest to lowest impact on prediction\n",
    "\n",
    "shap.plots.bar(shap_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How do higher/lower values of each feature affect the prediction?\n",
    "\n",
    "shap.summary_plot(shap_values, plot_size = (12,12), max_display = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}