slides_imbalance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playground for Class Imbalance Slides\n",
    "\n",
    "- Stephen W. Thomas\n",
    "- Used for MMA 869, MMAI 869, and GMMA 869"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-07-09 08:55:25.429194\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "print(datetime.datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "INSTALLED VERSIONS\n",
      "------------------\n",
      "commit           : None\n",
      "python           : 3.6.10.final.0\n",
      "python-bits      : 64\n",
      "OS               : Windows\n",
      "OS-release       : 10\n",
      "machine          : AMD64\n",
      "processor        : Intel64 Family 6 Model 142 Stepping 10, GenuineIntel\n",
      "byteorder        : little\n",
      "LC_ALL           : None\n",
      "LANG             : None\n",
      "LOCALE           : None.None\n",
      "\n",
      "pandas           : 1.0.3\n",
      "numpy            : 1.18.1\n",
      "pytz             : 2019.3\n",
      "dateutil         : 2.8.1\n",
      "pip              : 20.0.2\n",
      "setuptools       : 46.1.3.post20200325\n",
      "Cython           : None\n",
      "pytest           : 5.4.1\n",
      "hypothesis       : None\n",
      "sphinx           : None\n",
      "blosc            : None\n",
      "feather          : None\n",
      "xlsxwriter       : None\n",
      "lxml.etree       : 4.5.0\n",
      "html5lib         : None\n",
      "pymysql          : None\n",
      "psycopg2         : None\n",
      "jinja2           : 2.11.2\n",
      "IPython          : 7.13.0\n",
      "pandas_datareader: None\n",
      "bs4              : 4.9.0\n",
      "bottleneck       : None\n",
      "fastparquet      : None\n",
      "gcsfs            : None\n",
      "lxml.etree       : 4.5.0\n",
      "matplotlib       : 3.0.0\n",
      "numexpr          : None\n",
      "odfpy            : None\n",
      "openpyxl         : None\n",
      "pandas_gbq       : None\n",
      "pyarrow          : None\n",
      "pytables         : None\n",
      "pytest           : 5.4.1\n",
      "pyxlsb           : None\n",
      "s3fs             : None\n",
      "scipy            : 1.3.1\n",
      "sqlalchemy       : None\n",
      "tables           : None\n",
      "tabulate         : 0.8.7\n",
      "xarray           : None\n",
      "xlrd             : None\n",
      "xlwt             : None\n",
      "xlsxwriter       : None\n",
      "numba            : 0.48.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'0.21.3'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.show_versions(as_json=False)\n",
    "\n",
    "import sklearn\n",
    "sklearn.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "import scipy\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset\n",
    "\n",
    "The following dataset is a well-known imbalanced dataset from Kaggle:\n",
    "    https://www.kaggle.com/mlg-ulb/creditcardfraud/home?select=creditcard.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 142403 entries, 0 to 142402\n",
      "Data columns (total 31 columns):\n",
      " #   Column  Non-Null Count   Dtype  \n",
      "---  ------  --------------   -----  \n",
      " 0   Time    142403 non-null  int64  \n",
      " 1   V1      142403 non-null  float64\n",
      " 2   V2      142403 non-null  float64\n",
      " 3   V3      142403 non-null  float64\n",
      " 4   V4      142403 non-null  float64\n",
      " 5   V5      142403 non-null  float64\n",
      " 6   V6      142403 non-null  float64\n",
      " 7   V7      142403 non-null  float64\n",
      " 8   V8      142403 non-null  float64\n",
      " 9   V9      142403 non-null  float64\n",
      " 10  V10     142403 non-null  float64\n",
      " 11  V11     142403 non-null  float64\n",
      " 12  V12     142403 non-null  float64\n",
      " 13  V13     142403 non-null  float64\n",
      " 14  V14     142403 non-null  float64\n",
      " 15  V15     142403 non-null  float64\n",
      " 16  V16     142403 non-null  float64\n",
      " 17  V17     142403 non-null  float64\n",
      " 18  V18     142403 non-null  float64\n",
      " 19  V19     142403 non-null  float64\n",
      " 20  V20     142403 non-null  float64\n",
      " 21  V21     142403 non-null  float64\n",
      " 22  V22     142403 non-null  float64\n",
      " 23  V23     142403 non-null  float64\n",
      " 24  V24     142403 non-null  float64\n",
      " 25  V25     142403 non-null  float64\n",
      " 26  V26     142403 non-null  float64\n",
      " 27  V27     142403 non-null  float64\n",
      " 28  V28     142403 non-null  float64\n",
      " 29  Amount  142403 non-null  float64\n",
      " 30  Class   142403 non-null  int64  \n",
      "dtypes: float64(29), int64(2)\n",
      "memory usage: 33.7 MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "      <th>V9</th>\n",
       "      <th>...</th>\n",
       "      <th>V21</th>\n",
       "      <th>V22</th>\n",
       "      <th>V23</th>\n",
       "      <th>V24</th>\n",
       "      <th>V25</th>\n",
       "      <th>V26</th>\n",
       "      <th>V27</th>\n",
       "      <th>V28</th>\n",
       "      <th>Amount</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-1.359807</td>\n",
       "      <td>-0.072781</td>\n",
       "      <td>2.536347</td>\n",
       "      <td>1.378155</td>\n",
       "      <td>-0.338321</td>\n",
       "      <td>0.462388</td>\n",
       "      <td>0.239599</td>\n",
       "      <td>0.098698</td>\n",
       "      <td>0.363787</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018307</td>\n",
       "      <td>0.277838</td>\n",
       "      <td>-0.110474</td>\n",
       "      <td>0.066928</td>\n",
       "      <td>0.128539</td>\n",
       "      <td>-0.189115</td>\n",
       "      <td>0.133558</td>\n",
       "      <td>-0.021053</td>\n",
       "      <td>149.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1.191857</td>\n",
       "      <td>0.266151</td>\n",
       "      <td>0.166480</td>\n",
       "      <td>0.448154</td>\n",
       "      <td>0.060018</td>\n",
       "      <td>-0.082361</td>\n",
       "      <td>-0.078803</td>\n",
       "      <td>0.085102</td>\n",
       "      <td>-0.255425</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.225775</td>\n",
       "      <td>-0.638672</td>\n",
       "      <td>0.101288</td>\n",
       "      <td>-0.339846</td>\n",
       "      <td>0.167170</td>\n",
       "      <td>0.125895</td>\n",
       "      <td>-0.008983</td>\n",
       "      <td>0.014724</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>-1.358354</td>\n",
       "      <td>-1.340163</td>\n",
       "      <td>1.773209</td>\n",
       "      <td>0.379780</td>\n",
       "      <td>-0.503198</td>\n",
       "      <td>1.800499</td>\n",
       "      <td>0.791461</td>\n",
       "      <td>0.247676</td>\n",
       "      <td>-1.514654</td>\n",
       "      <td>...</td>\n",
       "      <td>0.247998</td>\n",
       "      <td>0.771679</td>\n",
       "      <td>0.909412</td>\n",
       "      <td>-0.689281</td>\n",
       "      <td>-0.327642</td>\n",
       "      <td>-0.139097</td>\n",
       "      <td>-0.055353</td>\n",
       "      <td>-0.059752</td>\n",
       "      <td>378.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.966272</td>\n",
       "      <td>-0.185226</td>\n",
       "      <td>1.792993</td>\n",
       "      <td>-0.863291</td>\n",
       "      <td>-0.010309</td>\n",
       "      <td>1.247203</td>\n",
       "      <td>0.237609</td>\n",
       "      <td>0.377436</td>\n",
       "      <td>-1.387024</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.108300</td>\n",
       "      <td>0.005274</td>\n",
       "      <td>-0.190321</td>\n",
       "      <td>-1.175575</td>\n",
       "      <td>0.647376</td>\n",
       "      <td>-0.221929</td>\n",
       "      <td>0.062723</td>\n",
       "      <td>0.061458</td>\n",
       "      <td>123.50</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>-1.158233</td>\n",
       "      <td>0.877737</td>\n",
       "      <td>1.548718</td>\n",
       "      <td>0.403034</td>\n",
       "      <td>-0.407193</td>\n",
       "      <td>0.095921</td>\n",
       "      <td>0.592941</td>\n",
       "      <td>-0.270533</td>\n",
       "      <td>0.817739</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009431</td>\n",
       "      <td>0.798278</td>\n",
       "      <td>-0.137458</td>\n",
       "      <td>0.141267</td>\n",
       "      <td>-0.206010</td>\n",
       "      <td>0.502292</td>\n",
       "      <td>0.219422</td>\n",
       "      <td>0.215153</td>\n",
       "      <td>69.99</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Time        V1        V2        V3        V4        V5        V6        V7  \\\n",
       "0     0 -1.359807 -0.072781  2.536347  1.378155 -0.338321  0.462388  0.239599   \n",
       "1     0  1.191857  0.266151  0.166480  0.448154  0.060018 -0.082361 -0.078803   \n",
       "2     1 -1.358354 -1.340163  1.773209  0.379780 -0.503198  1.800499  0.791461   \n",
       "3     1 -0.966272 -0.185226  1.792993 -0.863291 -0.010309  1.247203  0.237609   \n",
       "4     2 -1.158233  0.877737  1.548718  0.403034 -0.407193  0.095921  0.592941   \n",
       "\n",
       "         V8        V9  ...       V21       V22       V23       V24       V25  \\\n",
       "0  0.098698  0.363787  ... -0.018307  0.277838 -0.110474  0.066928  0.128539   \n",
       "1  0.085102 -0.255425  ... -0.225775 -0.638672  0.101288 -0.339846  0.167170   \n",
       "2  0.247676 -1.514654  ...  0.247998  0.771679  0.909412 -0.689281 -0.327642   \n",
       "3  0.377436 -1.387024  ... -0.108300  0.005274 -0.190321 -1.175575  0.647376   \n",
       "4 -0.270533  0.817739  ... -0.009431  0.798278 -0.137458  0.141267 -0.206010   \n",
       "\n",
       "        V26       V27       V28  Amount  Class  \n",
       "0 -0.189115  0.133558 -0.021053  149.62      0  \n",
       "1  0.125895 -0.008983  0.014724    2.69      0  \n",
       "2 -0.139097 -0.055353 -0.059752  378.66      0  \n",
       "3 -0.221929  0.062723  0.061458  123.50      0  \n",
       "4  0.502292  0.219422  0.215153   69.99      0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/creditcard_sample.csv')\n",
    "df.info()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a random sample, and then create X, y variables, and then split into training and testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# This dataset is huge, so let's take a sample to speed things up\n",
    "# df = df.sample(frac=0.5, replace=False, random_state=1, axis=0)\n",
    "X = df.drop(['Class'], axis=1)\n",
    "y = df['Class']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=44)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    113707\n",
       "1       215\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0    28427\n",
       "1       54\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()\n",
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, recall_score, precision_score, roc_auc_score\n",
    "\n",
    "# Helper function\n",
    "def quick_evaluate_with_dt(X_train, X_test, y_train, y_test, name, balance_weights=False):\n",
    "    \n",
    "    cw = None\n",
    "    if balance_weights == True:\n",
    "        cw = 'balanced'\n",
    "        \n",
    "    clf = DecisionTreeClassifier(random_state=0, class_weight=cw)\n",
    "    clf = RandomForestClassifier(random_state=0, n_estimators=100, class_weight=cw)\n",
    "    \n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    \n",
    "    tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()\n",
    "    accuracy       = accuracy_score(y_test, y_pred)\n",
    "    f1             = f1_score(y_test, y_pred)\n",
    "    recall         = recall_score(y_test, y_pred)\n",
    "    precision      = precision_score(y_test, y_pred)\n",
    "    roc_auc        = roc_auc_score(y_test, y_pred)\n",
    "    \n",
    "    df = pd.DataFrame({\"Method\"    : [name],\n",
    "                       \"Neg\"       : [tn + fn],\n",
    "                       \"True Neg\"  : [tn],\n",
    "                       \"False Neg\" : [fn],\n",
    "                       \"Pos\"       : [tp + fp],\n",
    "                       \"TP\"        : [tp],\n",
    "                       \"FP\"        : [fp],\n",
    "                       \"Accuracy\"  : [accuracy],\n",
    "                       \"Recall\"    : [recall],\n",
    "                       \"Precision\" : [precision],\n",
    "                       \"F1\"        : [f1],\n",
    "                       \"AUC\"       : [roc_auc],\n",
    "                      })\n",
    "    \n",
    "    print(df)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "evals = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113922, 30)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(113922,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([113707,    215], dtype=int64)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Method    Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0   None  28435     28425         10   46  44   2  0.999579  0.814815   \n",
      "\n",
      "   Precision    F1       AUC  \n",
      "0   0.956522  0.88  0.907372  \n"
     ]
    }
   ],
   "source": [
    "X_train.shape\n",
    "y_train.shape\n",
    "np.bincount(y_train)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_train, X_test, y_train, y_test, 'None'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113922, 30)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(113922,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([113707,    215], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Method    Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Class Weights  28435     28425         10   46  44   2  0.999579  0.814815   \n",
      "\n",
      "   Precision    F1       AUC  \n",
      "0   0.956522  0.88  0.907372  \n"
     ]
    }
   ],
   "source": [
    "X_train.shape\n",
    "y_train.shape\n",
    "np.bincount(y_train)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_train, X_test, y_train, y_test, 'Class Weights', balance_weights=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(227414, 30)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(227414,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([113707, 113707], dtype=int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Method    Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Over Random  28434     28425          9   47  45   2  0.999614  0.833333   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.957447  0.891089  0.916631  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "ros = RandomOverSampler(random_state=0)\n",
    "\n",
    "X_resampled, y_resampled = ros.fit_resample(X_train, y_train)\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test, 'Over Random'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(227414, 30)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(227414,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([113707, 113707], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Method    Neg  True Neg  False Neg  Pos  TP  FP  Accuracy   Recall  \\\n",
      "0  Over SMOTE  28428     28421          7   53  47   6  0.999544  0.87037   \n",
      "\n",
      "   Precision        F1      AUC  \n",
      "0   0.886792  0.878505  0.93508  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "X_resampled, y_resampled = SMOTE(random_state=0).fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test, 'Over SMOTE'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(227450, 30)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(227450,)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([113707, 113743], dtype=int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Method    Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Over ADASYN  28429     28421          8   52  46   6  0.999508  0.851852   \n",
      "\n",
      "   Precision        F1      AUC  \n",
      "0   0.884615  0.867925  0.92582  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import ADASYN\n",
    "\n",
    "X_resampled, y_resampled = ADASYN(random_state=0).fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test,  'Over ADASYN'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(430, 30)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(430,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([215, 215], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         Method    Neg  True Neg  False Neg  Pos  TP   FP  Accuracy    Recall  \\\n",
      "0  Under Sample  27874     27868          6  607  48  559  0.980162  0.888889   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.079077  0.145234  0.934612  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "rus = RandomUnderSampler(random_state=0)\n",
    "X_resampled, y_resampled = rus.fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test,  'Under Sample'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    28427\n",
       "1       54\n",
       "Name: Class, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Neg</th>\n",
       "      <th>True Neg</th>\n",
       "      <th>False Neg</th>\n",
       "      <th>Pos</th>\n",
       "      <th>TP</th>\n",
       "      <th>FP</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "      <th>AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Over Random</td>\n",
       "      <td>28434</td>\n",
       "      <td>28425</td>\n",
       "      <td>9</td>\n",
       "      <td>47</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0.999614</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.957447</td>\n",
       "      <td>0.891089</td>\n",
       "      <td>0.916631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>28435</td>\n",
       "      <td>28425</td>\n",
       "      <td>10</td>\n",
       "      <td>46</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>0.999579</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>0.956522</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.907372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Class Weights</td>\n",
       "      <td>28435</td>\n",
       "      <td>28425</td>\n",
       "      <td>10</td>\n",
       "      <td>46</td>\n",
       "      <td>44</td>\n",
       "      <td>2</td>\n",
       "      <td>0.999579</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>0.956522</td>\n",
       "      <td>0.880000</td>\n",
       "      <td>0.907372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Over SMOTE</td>\n",
       "      <td>28428</td>\n",
       "      <td>28421</td>\n",
       "      <td>7</td>\n",
       "      <td>53</td>\n",
       "      <td>47</td>\n",
       "      <td>6</td>\n",
       "      <td>0.999544</td>\n",
       "      <td>0.870370</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.878505</td>\n",
       "      <td>0.935080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Over ADASYN</td>\n",
       "      <td>28429</td>\n",
       "      <td>28421</td>\n",
       "      <td>8</td>\n",
       "      <td>52</td>\n",
       "      <td>46</td>\n",
       "      <td>6</td>\n",
       "      <td>0.999508</td>\n",
       "      <td>0.851852</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.867925</td>\n",
       "      <td>0.925820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Under Sample</td>\n",
       "      <td>27874</td>\n",
       "      <td>27868</td>\n",
       "      <td>6</td>\n",
       "      <td>607</td>\n",
       "      <td>48</td>\n",
       "      <td>559</td>\n",
       "      <td>0.980162</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.079077</td>\n",
       "      <td>0.145234</td>\n",
       "      <td>0.934612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Method    Neg  True Neg  False Neg  Pos  TP   FP  Accuracy  \\\n",
       "2    Over Random  28434     28425          9   47  45    2  0.999614   \n",
       "0           None  28435     28425         10   46  44    2  0.999579   \n",
       "1  Class Weights  28435     28425         10   46  44    2  0.999579   \n",
       "3     Over SMOTE  28428     28421          7   53  47    6  0.999544   \n",
       "4    Over ADASYN  28429     28421          8   52  46    6  0.999508   \n",
       "5   Under Sample  27874     27868          6  607  48  559  0.980162   \n",
       "\n",
       "     Recall  Precision        F1       AUC  \n",
       "2  0.833333   0.957447  0.891089  0.916631  \n",
       "0  0.814815   0.956522  0.880000  0.907372  \n",
       "1  0.814815   0.956522  0.880000  0.907372  \n",
       "3  0.870370   0.886792  0.878505  0.935080  \n",
       "4  0.851852   0.884615  0.867925  0.925820  \n",
       "5  0.888889   0.079077  0.145234  0.934612  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recall the actual distrition of the truth labels of the testing set.\n",
    "y_test.value_counts()\n",
    "\n",
    "evals_all = pd.concat([m for m in evals], axis = 0).reset_index()\n",
    "\n",
    "evals_all = evals_all.drop(columns = \"index\",axis =1)\n",
    "evals_all.sort_values(by=['F1'], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Another Dataset\n",
    "\n",
    "The dataset is also imbalanced, although less so. It comes from:\n",
    "\n",
    "https://www.kaggle.com/uciml/pima-indians-diabetes-database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 10 columns):\n",
      " #   Column              Non-Null Count  Dtype  \n",
      "---  ------              --------------  -----  \n",
      " 0   Id                  768 non-null    int64  \n",
      " 1   num_times_pregnant  768 non-null    int64  \n",
      " 2   plasma_glucose      768 non-null    int64  \n",
      " 3   DBP                 768 non-null    int64  \n",
      " 4   triceps_skin        768 non-null    int64  \n",
      " 5   serum_insulin       768 non-null    int64  \n",
      " 6   BMI                 768 non-null    float64\n",
      " 7   pedigree            768 non-null    float64\n",
      " 8   age                 768 non-null    int64  \n",
      " 9   diabetes            768 non-null    int64  \n",
      "dtypes: float64(2), int64(8)\n",
      "memory usage: 60.1 KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>num_times_pregnant</th>\n",
       "      <th>plasma_glucose</th>\n",
       "      <th>DBP</th>\n",
       "      <th>triceps_skin</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>pedigree</th>\n",
       "      <th>age</th>\n",
       "      <th>diabetes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  num_times_pregnant  plasma_glucose  DBP  triceps_skin  serum_insulin  \\\n",
       "0   1                   6             148   72            35              0   \n",
       "1   2                   1              85   66            29              0   \n",
       "2   3                   8             183   64             0              0   \n",
       "3   4                   1              89   66            23             94   \n",
       "4   5                   0             137   40            35            168   \n",
       "\n",
       "    BMI  pedigree  age  diabetes  \n",
       "0  33.6     0.627   50         1  \n",
       "1  26.6     0.351   31         0  \n",
       "2  23.3     0.672   32         1  \n",
       "3  28.1     0.167   21         0  \n",
       "4  43.1     2.288   33         1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_d = pd.read_csv('data/diabetes_orig.csv')\n",
    "df_d.info()\n",
    "df_d.head()\n",
    "\n",
    "X = df_d.drop(['Id', 'diabetes'], axis=1)\n",
    "y = df_d['diabetes']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=44)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    400\n",
       "1    214\n",
       "Name: diabetes, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0    100\n",
       "1     54\n",
       "Name: diabetes, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()\n",
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "evals_d = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 8)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(614,)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([400, 214], dtype=int64)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0   None  116        88         28   38  26  12   0.74026  0.481481   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.684211  0.565217  0.680741  \n"
     ]
    }
   ],
   "source": [
    "X_train.shape\n",
    "y_train.shape\n",
    "np.bincount(y_train)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_train, X_test, y_train, y_test, 'None'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 8)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(614,)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([400, 214], dtype=int64)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Class Weights  117        92         25   37  29   8  0.785714  0.537037   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.783784  0.637363  0.728519  \n"
     ]
    }
   ],
   "source": [
    "X_train.shape\n",
    "y_train.shape\n",
    "np.bincount(y_train)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_train, X_test, y_train, y_test, 'Class Weights', balance_weights=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 8)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(800,)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([400, 400], dtype=int64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Over Random  110        86         24   44  30  14  0.753247  0.555556   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.681818  0.612245  0.707778  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "\n",
    "ros = RandomOverSampler(random_state=0)\n",
    "\n",
    "X_resampled, y_resampled = ros.fit_resample(X_train, y_train)\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test, 'Over Random'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 8)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(800,)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([400, 400], dtype=int64)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Over SMOTE  105        84         21   49  33  16   0.75974  0.611111   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.673469  0.640777  0.725556  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "X_resampled, y_resampled = SMOTE(random_state=0).fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test, 'Over SMOTE'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(775, 8)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(775,)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([400, 375], dtype=int64)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy   Recall  \\\n",
      "0  Over ADASYN   99        79         20   55  34  21  0.733766  0.62963   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.618182  0.623853  0.709815  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import ADASYN\n",
    "\n",
    "X_resampled, y_resampled = ADASYN(random_state=0).fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test,  'Over ADASYN'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(428, 8)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(428,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([214, 214], dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
      "0  Under Sample   92        73         19   62  35  27  0.701299  0.648148   \n",
      "\n",
      "   Precision        F1       AUC  \n",
      "0   0.564516  0.603448  0.689074  \n"
     ]
    }
   ],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "rus = RandomUnderSampler(random_state=0)\n",
    "X_resampled, y_resampled = rus.fit_resample(X_train, y_train)\n",
    "\n",
    "X_resampled.shape\n",
    "y_resampled.shape\n",
    "np.bincount(y_resampled)\n",
    "\n",
    "evals_d.append(quick_evaluate_with_dt(X_resampled, X_test, y_resampled, y_test,  'Under Sample'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    100\n",
       "1     54\n",
       "Name: diabetes, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Neg</th>\n",
       "      <th>True Neg</th>\n",
       "      <th>False Neg</th>\n",
       "      <th>Pos</th>\n",
       "      <th>TP</th>\n",
       "      <th>FP</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "      <th>AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Over SMOTE</td>\n",
       "      <td>105</td>\n",
       "      <td>84</td>\n",
       "      <td>21</td>\n",
       "      <td>49</td>\n",
       "      <td>33</td>\n",
       "      <td>16</td>\n",
       "      <td>0.759740</td>\n",
       "      <td>0.611111</td>\n",
       "      <td>0.673469</td>\n",
       "      <td>0.640777</td>\n",
       "      <td>0.725556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Class Weights</td>\n",
       "      <td>117</td>\n",
       "      <td>92</td>\n",
       "      <td>25</td>\n",
       "      <td>37</td>\n",
       "      <td>29</td>\n",
       "      <td>8</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.537037</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.637363</td>\n",
       "      <td>0.728519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Over ADASYN</td>\n",
       "      <td>99</td>\n",
       "      <td>79</td>\n",
       "      <td>20</td>\n",
       "      <td>55</td>\n",
       "      <td>34</td>\n",
       "      <td>21</td>\n",
       "      <td>0.733766</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>0.618182</td>\n",
       "      <td>0.623853</td>\n",
       "      <td>0.709815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Over Random</td>\n",
       "      <td>110</td>\n",
       "      <td>86</td>\n",
       "      <td>24</td>\n",
       "      <td>44</td>\n",
       "      <td>30</td>\n",
       "      <td>14</td>\n",
       "      <td>0.753247</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.681818</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>0.707778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Under Sample</td>\n",
       "      <td>92</td>\n",
       "      <td>73</td>\n",
       "      <td>19</td>\n",
       "      <td>62</td>\n",
       "      <td>35</td>\n",
       "      <td>27</td>\n",
       "      <td>0.701299</td>\n",
       "      <td>0.648148</td>\n",
       "      <td>0.564516</td>\n",
       "      <td>0.603448</td>\n",
       "      <td>0.689074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>116</td>\n",
       "      <td>88</td>\n",
       "      <td>28</td>\n",
       "      <td>38</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "      <td>0.740260</td>\n",
       "      <td>0.481481</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.680741</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Method  Neg  True Neg  False Neg  Pos  TP  FP  Accuracy    Recall  \\\n",
       "3     Over SMOTE  105        84         21   49  33  16  0.759740  0.611111   \n",
       "1  Class Weights  117        92         25   37  29   8  0.785714  0.537037   \n",
       "4    Over ADASYN   99        79         20   55  34  21  0.733766  0.629630   \n",
       "2    Over Random  110        86         24   44  30  14  0.753247  0.555556   \n",
       "5   Under Sample   92        73         19   62  35  27  0.701299  0.648148   \n",
       "0           None  116        88         28   38  26  12  0.740260  0.481481   \n",
       "\n",
       "   Precision        F1       AUC  \n",
       "3   0.673469  0.640777  0.725556  \n",
       "1   0.783784  0.637363  0.728519  \n",
       "4   0.618182  0.623853  0.709815  \n",
       "2   0.681818  0.612245  0.707778  \n",
       "5   0.564516  0.603448  0.689074  \n",
       "0   0.684211  0.565217  0.680741  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recall the actual distrition of the truth labels of the testing set.\n",
    "y_test.value_counts()\n",
    "\n",
    "evals_d_all = pd.concat([m for m in evals_d], axis = 0).reset_index()\n",
    "\n",
    "evals_d_all = evals_d_all.drop(columns = \"index\",axis =1)\n",
    "evals_d_all.sort_values(by=['F1'], ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "small_sklearn_kernel",
   "language": "python",
   "name": "small_sklearn_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}