flu_shot_learning-TUNING.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline\n",
    "sns.set()\n",
    "\n",
    "plt.rcParams['figure.figsize'] = [8.0, 8.0]\n",
    "plt.rcParams['figure.dpi'] = 140"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_full_seas = pd.read_csv('interim_data\\\\preprocessed_train_features_seas.csv', index_col = 'respondent_id')\n",
    "X_train_full_h1n1 = pd.read_csv('interim_data\\\\preprocessed_train_features_h1n1.csv', index_col = 'respondent_id')\n",
    "\n",
    "y_train_full = pd.read_csv('input_data\\\\training_set_labels.csv', index_col = 'respondent_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_seas = X_train_full_seas.join(y_train_full)\n",
    "train_df_h1n1 = X_train_full_h1n1.join(y_train_full)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna import Trial, visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making validation folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "N_folds = 5\n",
    "kf = KFold(n_splits=N_folds, random_state=42, shuffle=True)\n",
    "\n",
    "for i, (trn, val) in enumerate(kf.split(train_df_seas)):\n",
    "    train_df_seas.loc[val, 'kfold'] = i\n",
    "    train_df_h1n1.loc[val, 'kfold'] = i\n",
    "    \n",
    "train_df_seas['kfold'] = train_df_seas['kfold'].astype(int)\n",
    "train_df_h1n1['kfold'] = train_df_h1n1['kfold'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost - H1N1 Vaccine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = [col for col in X_train_full_h1n1.columns.tolist() if col not in ['h1n1_vaccine', 'seasonal_vaccine', 'kfold']]\n",
    "target_cols = ['h1n1_vaccine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_xgb_h1n1(trial):\n",
    "    roc = 0\n",
    "    for fold in range(5):\n",
    "        trn_idx = train_df_h1n1['kfold'] != fold\n",
    "        val_idx = train_df_h1n1['kfold'] == fold\n",
    "        trn = train_df_h1n1.loc[trn_idx, :]\n",
    "        val = train_df_h1n1.loc[val_idx, :]\n",
    "\n",
    "        x_tr, y_tr = trn[feature_cols].values, trn[target_cols].values\n",
    "        x_val, y_val = val[feature_cols].values, val[target_cols].values\n",
    "        \n",
    "        model, log = fit_xgb(trial, x_tr, y_tr, x_val, y_val)\n",
    "        roc += log['valid roc']/5\n",
    "        \n",
    "    return roc\n",
    "\n",
    "def objective_xgb_seas(trial):\n",
    "    roc = 0\n",
    "    for fold in range(5):\n",
    "        trn_idx = train_df_seas['kfold'] != fold\n",
    "        val_idx = train_df_seas['kfold'] == fold\n",
    "        trn = train_df_seas.loc[trn_idx, :]\n",
    "        val = train_df_seas.loc[val_idx, :]\n",
    "\n",
    "        x_tr, y_tr = trn[feature_cols].values, trn[target_cols].values\n",
    "        x_val, y_val = val[feature_cols].values, val[target_cols].values\n",
    "        \n",
    "        model, log = fit_xgb(trial, x_tr, y_tr, x_val, y_val)\n",
    "        roc += log['valid roc']/5\n",
    "        \n",
    "    return roc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "def fit_xgb(trial, x_train, y_train, x_val, y_val):\n",
    "    params = {\n",
    "        \"n_estimators\": trial.suggest_categorical(\"n_estimators\", [80, 85, 90, 95, 100, 105, 110, 115, 120]),\n",
    "        \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [0.025, 0.03, 0.04, 0.05, 0.75, 0.1]),\n",
    "        \"subsample\": trial.suggest_discrete_uniform(\"subsample\", 0.4,1,0.1),\n",
    "        \"colsample_bytree\": trial.suggest_discrete_uniform(\"colsample_bytree\", 0.6,1,0.1),\n",
    "        \"max_depth\": trial.suggest_categorical(\"max_depth\",[4,5,6,7,8,9,10]),\n",
    "        \"min_child_weight\": trial.suggest_int(\"min_child_weight\",1,5),\n",
    "        \"gamma\": 0,\n",
    "        \"base_score\": 0.5,\n",
    "        \"random_state\": 42,\n",
    "        \"use_label_encoder\": False,\n",
    "        \"objective\": 'reg:logistic',\n",
    "        \"tree_method\": 'exact'\n",
    "    }\n",
    "    \n",
    "    model = XGBClassifier(**params)\n",
    "    model.fit(x_train, y_train.reshape(-1,))\n",
    "\n",
    "    y_pred_tr = model.predict_proba(x_train)[:, 1]\n",
    "    y_pred_val = model.predict_proba(x_val)[:, 1]\n",
    "\n",
    "    log = {\n",
    "        \"train roc\": roc_auc_score(y_train, y_pred_tr),\n",
    "        \"valid roc\": roc_auc_score(y_val, y_pred_val)\n",
    "    }\n",
    "    \n",
    "    return model, log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-02-10 14:49:08,333]\u001b[0m A new study created in memory with name: XGBoost H1N1 Vaccine optimization\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:49:17,987]\u001b[0m Trial 0 finished with value: 0.7962669056052987 and parameters: {'n_estimators': 90, 'learning_rate': 0.75, 'subsample': 1.0, 'colsample_bytree': 1.0, 'max_depth': 6, 'min_child_weight': 2}. Best is trial 0 with value: 0.7962669056052987.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:49:28,357]\u001b[0m Trial 1 finished with value: 0.8366221277386817 and parameters: {'n_estimators': 110, 'learning_rate': 0.04, 'subsample': 0.9, 'colsample_bytree': 1.0, 'max_depth': 4, 'min_child_weight': 2}. Best is trial 1 with value: 0.8366221277386817.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:49:36,431]\u001b[0m Trial 2 finished with value: 0.8385778920085278 and parameters: {'n_estimators': 85, 'learning_rate': 0.05, 'subsample': 0.9, 'colsample_bytree': 0.7, 'max_depth': 5, 'min_child_weight': 1}. Best is trial 2 with value: 0.8385778920085278.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:49:45,545]\u001b[0m Trial 3 finished with value: 0.8400315662689282 and parameters: {'n_estimators': 80, 'learning_rate': 0.1, 'subsample': 0.5, 'colsample_bytree': 0.6, 'max_depth': 6, 'min_child_weight': 3}. Best is trial 3 with value: 0.8400315662689282.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:49:58,648]\u001b[0m Trial 4 finished with value: 0.8373381415018091 and parameters: {'n_estimators': 105, 'learning_rate': 0.03, 'subsample': 1.0, 'colsample_bytree': 0.7, 'max_depth': 6, 'min_child_weight': 2}. Best is trial 3 with value: 0.8400315662689282.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:50:06,558]\u001b[0m Trial 5 finished with value: 0.8303265214706056 and parameters: {'n_estimators': 80, 'learning_rate': 0.03, 'subsample': 0.7000000000000001, 'colsample_bytree': 0.6, 'max_depth': 4, 'min_child_weight': 3}. Best is trial 3 with value: 0.8400315662689282.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:50:26,314]\u001b[0m Trial 6 finished with value: 0.8355809078267212 and parameters: {'n_estimators': 85, 'learning_rate': 0.03, 'subsample': 0.9, 'colsample_bytree': 0.9, 'max_depth': 10, 'min_child_weight': 4}. Best is trial 3 with value: 0.8400315662689282.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:50:37,509]\u001b[0m Trial 7 finished with value: 0.8367937284299908 and parameters: {'n_estimators': 80, 'learning_rate': 0.1, 'subsample': 0.4, 'colsample_bytree': 1.0, 'max_depth': 6, 'min_child_weight': 2}. Best is trial 3 with value: 0.8400315662689282.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:50:49,285]\u001b[0m Trial 8 finished with value: 0.8406644455968788 and parameters: {'n_estimators': 105, 'learning_rate': 0.05, 'subsample': 0.7000000000000001, 'colsample_bytree': 0.6, 'max_depth': 6, 'min_child_weight': 5}. Best is trial 8 with value: 0.8406644455968788.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:51:00,664]\u001b[0m Trial 9 finished with value: 0.8398232296630215 and parameters: {'n_estimators': 110, 'learning_rate': 0.05, 'subsample': 0.4, 'colsample_bytree': 0.6, 'max_depth': 6, 'min_child_weight': 5}. Best is trial 8 with value: 0.8406644455968788.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "XGB_study_H1N1 = optuna.create_study(direction=\"maximize\", study_name='XGBoost H1N1 Vaccine optimization')\n",
    "XGB_study_H1N1.optimize(objective_xgb_h1n1, n_trials=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 105,\n",
       " 'learning_rate': 0.05,\n",
       " 'subsample': 0.7000000000000001,\n",
       " 'colsample_bytree': 0.6,\n",
       " 'max_depth': 6,\n",
       " 'min_child_weight': 5}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XGB_study_H1N1.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "from pathlib import Path\n",
    "\n",
    "root = Path(\".\")\n",
    "\n",
    "with open(root / \"models\\\\XGB_H1N1_best_params.pkl\",\"wb\") as f:\n",
    "    pickle.dump(XGB_study_H1N1.best_params, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost - Seasonal Vaccine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = [col for col in X_train_full_seas.columns.tolist() if col not in ['h1n1_vaccine', 'seasonal_vaccine', 'kfold']]\n",
    "target_cols = ['seasonal_vaccine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-02-10 14:51:00,737]\u001b[0m A new study created in memory with name: XGBoost Seasonal Vaccine optimization\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:51:18,655]\u001b[0m Trial 0 finished with value: 0.8607585662258218 and parameters: {'n_estimators': 120, 'learning_rate': 0.04, 'subsample': 0.4, 'colsample_bytree': 0.7, 'max_depth': 7, 'min_child_weight': 5}. Best is trial 0 with value: 0.8607585662258218.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:51:36,040]\u001b[0m Trial 1 finished with value: 0.8561484052213391 and parameters: {'n_estimators': 85, 'learning_rate': 0.1, 'subsample': 0.9, 'colsample_bytree': 1.0, 'max_depth': 9, 'min_child_weight': 4}. Best is trial 0 with value: 0.8607585662258218.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:51:45,072]\u001b[0m Trial 2 finished with value: 0.8609992997351913 and parameters: {'n_estimators': 85, 'learning_rate': 0.1, 'subsample': 0.7000000000000001, 'colsample_bytree': 0.7, 'max_depth': 4, 'min_child_weight': 2}. Best is trial 2 with value: 0.8609992997351913.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:52:01,311]\u001b[0m Trial 3 finished with value: 0.8008670820275882 and parameters: {'n_estimators': 80, 'learning_rate': 0.75, 'subsample': 0.7000000000000001, 'colsample_bytree': 0.7, 'max_depth': 10, 'min_child_weight': 5}. Best is trial 2 with value: 0.8609992997351913.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:52:13,228]\u001b[0m Trial 4 finished with value: 0.8154115493946319 and parameters: {'n_estimators': 115, 'learning_rate': 0.75, 'subsample': 0.4, 'colsample_bytree': 0.9, 'max_depth': 4, 'min_child_weight': 3}. Best is trial 2 with value: 0.8609992997351913.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:52:29,885]\u001b[0m Trial 5 finished with value: 0.8597438394678563 and parameters: {'n_estimators': 120, 'learning_rate': 0.03, 'subsample': 0.7000000000000001, 'colsample_bytree': 0.7, 'max_depth': 7, 'min_child_weight': 5}. Best is trial 2 with value: 0.8609992997351913.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:52:46,561]\u001b[0m Trial 6 finished with value: 0.8590846107119956 and parameters: {'n_estimators': 115, 'learning_rate': 0.025, 'subsample': 0.4, 'colsample_bytree': 0.7, 'max_depth': 8, 'min_child_weight': 4}. Best is trial 2 with value: 0.8609992997351913.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:52:55,735]\u001b[0m Trial 7 finished with value: 0.8614429869121276 and parameters: {'n_estimators': 115, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bytree': 0.6, 'max_depth': 5, 'min_child_weight': 5}. Best is trial 7 with value: 0.8614429869121276.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:15,135]\u001b[0m Trial 8 finished with value: 0.8592364603698759 and parameters: {'n_estimators': 90, 'learning_rate': 0.04, 'subsample': 0.5, 'colsample_bytree': 0.7, 'max_depth': 10, 'min_child_weight': 4}. Best is trial 7 with value: 0.8614429869121276.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:26,564]\u001b[0m Trial 9 finished with value: 0.8545503164636972 and parameters: {'n_estimators': 95, 'learning_rate': 0.025, 'subsample': 0.9, 'colsample_bytree': 0.8, 'max_depth': 5, 'min_child_weight': 1}. Best is trial 7 with value: 0.8614429869121276.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "XGB_study_SEAS = optuna.create_study(direction=\"maximize\", study_name='XGBoost Seasonal Vaccine optimization')\n",
    "XGB_study_SEAS.optimize(objective_xgb_seas, n_trials=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 115,\n",
       " 'learning_rate': 0.1,\n",
       " 'subsample': 1.0,\n",
       " 'colsample_bytree': 0.6,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 5}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XGB_study_SEAS.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(root / \"models\\\\XGB_SEAS_best_params.pkl\",\"wb\") as f:\n",
    "    pickle.dump(XGB_study_SEAS.best_params, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM - H1N1 Vaccine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = [col for col in X_train_full_h1n1.columns.tolist() if col not in ['h1n1_vaccine', 'seasonal_vaccine', 'kfold']]\n",
    "target_cols = ['h1n1_vaccine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_lgbm_h1n1(trial):\n",
    "    roc = 0\n",
    "    for fold in range(5):\n",
    "        trn_idx = train_df_h1n1['kfold'] != fold\n",
    "        val_idx = train_df_h1n1['kfold'] == fold\n",
    "        trn = train_df_h1n1.loc[trn_idx, :]\n",
    "        val = train_df_h1n1.loc[val_idx, :]\n",
    "\n",
    "        x_tr, y_tr = trn[feature_cols].values, trn[target_cols].values\n",
    "        x_val, y_val = val[feature_cols].values, val[target_cols].values\n",
    "        \n",
    "        model, log = fit_lgbm(trial, x_tr, y_tr, x_val, y_val)\n",
    "        roc += log['valid roc']/5\n",
    "        \n",
    "    return roc\n",
    "\n",
    "def objective_lgbm_seas(trial):\n",
    "    roc = 0\n",
    "    for fold in range(5):\n",
    "        trn_idx = train_df_seas['kfold'] != fold\n",
    "        val_idx = train_df_seas['kfold'] == fold\n",
    "        trn = train_df_seas.loc[trn_idx, :]\n",
    "        val = train_df_seas.loc[val_idx, :]\n",
    "\n",
    "        x_tr, y_tr = trn[feature_cols].values, trn[target_cols].values\n",
    "        x_val, y_val = val[feature_cols].values, val[target_cols].values\n",
    "        \n",
    "        model, log = fit_lgbm(trial, x_tr, y_tr, x_val, y_val)\n",
    "        roc += log['valid roc']/5\n",
    "        \n",
    "    return roc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_lgbm(trial, x_train, y_train, x_val, y_val):\n",
    "    params = {\n",
    "        \"n_estimators\": trial.suggest_int(\"n_estimators\", 60, 150, 10),\n",
    "        \"learning_rate\": trial.suggest_categorical(\"learning_rate\", [0.025, 0.05, 0.75, 0.1]),\n",
    "        \"subsample\": trial.suggest_discrete_uniform(\"subsample\", 0.8,1,0.05),\n",
    "        \"colsample_bytree\": trial.suggest_discrete_uniform(\"colsample_bytree\", 0.6,0.9,0.05),\n",
    "        \"min_child_samples\": trial.suggest_int(\"min_child_samples\",18,30),\n",
    "        \"min_child_weight\": trial.suggest_discrete_uniform(\"min_child_weight\",0.0005,0.0015,0.0005),\n",
    "        \"max_depth\": -1,\n",
    "        \"random_state\": 42,\n",
    "        \"silent\":True\n",
    "    }\n",
    "    \n",
    "    model = LGBMClassifier(**params)\n",
    "    model.fit(x_train, y_train.reshape(-1,))\n",
    "\n",
    "    y_pred_tr = model.predict_proba(x_train)[:, 1]\n",
    "    y_pred_val = model.predict_proba(x_val)[:, 1]\n",
    "\n",
    "    log = {\n",
    "        \"train roc\": roc_auc_score(y_train, y_pred_tr),\n",
    "        \"valid roc\": roc_auc_score(y_val, y_pred_val)\n",
    "    }\n",
    "    \n",
    "    return model, log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-02-10 14:53:26,703]\u001b[0m A new study created in memory with name: LightGBM H1N1 Vaccine optimization\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:29,255]\u001b[0m Trial 0 finished with value: 0.7957361498285433 and parameters: {'n_estimators': 90, 'learning_rate': 0.75, 'subsample': 0.8500000000000001, 'colsample_bytree': 0.6, 'min_child_samples': 21, 'min_child_weight': 0.0005}. Best is trial 0 with value: 0.7957361498285433.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:31,930]\u001b[0m Trial 1 finished with value: 0.8410632039728194 and parameters: {'n_estimators': 80, 'learning_rate': 0.1, 'subsample': 0.9500000000000001, 'colsample_bytree': 0.65, 'min_child_samples': 19, 'min_child_weight': 0.001}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:34,591]\u001b[0m Trial 2 finished with value: 0.8395079145063364 and parameters: {'n_estimators': 90, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bytree': 0.85, 'min_child_samples': 30, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:37,602]\u001b[0m Trial 3 finished with value: 0.840697216483513 and parameters: {'n_estimators': 140, 'learning_rate': 0.05, 'subsample': 0.8, 'colsample_bytree': 0.9, 'min_child_samples': 20, 'min_child_weight': 0.0015}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:39,888]\u001b[0m Trial 4 finished with value: 0.8404192469017471 and parameters: {'n_estimators': 70, 'learning_rate': 0.1, 'subsample': 0.8, 'colsample_bytree': 0.9, 'min_child_samples': 27, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:43,121]\u001b[0m Trial 5 finished with value: 0.8394541123474175 and parameters: {'n_estimators': 140, 'learning_rate': 0.1, 'subsample': 0.8, 'colsample_bytree': 0.6, 'min_child_samples': 21, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:45,398]\u001b[0m Trial 6 finished with value: 0.7957848121313903 and parameters: {'n_estimators': 80, 'learning_rate': 0.75, 'subsample': 0.9500000000000001, 'colsample_bytree': 0.85, 'min_child_samples': 28, 'min_child_weight': 0.001}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:48,193]\u001b[0m Trial 7 finished with value: 0.8397135730769368 and parameters: {'n_estimators': 110, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bytree': 0.7, 'min_child_samples': 21, 'min_child_weight': 0.001}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:52,007]\u001b[0m Trial 8 finished with value: 0.789792498529468 and parameters: {'n_estimators': 100, 'learning_rate': 0.75, 'subsample': 0.8500000000000001, 'colsample_bytree': 0.9, 'min_child_samples': 28, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:54,929]\u001b[0m Trial 9 finished with value: 0.8355911275840949 and parameters: {'n_estimators': 80, 'learning_rate': 0.025, 'subsample': 1.0, 'colsample_bytree': 0.75, 'min_child_samples': 21, 'min_child_weight': 0.0015}. Best is trial 1 with value: 0.8410632039728194.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "LGBM_study_H1N1 = optuna.create_study(direction=\"maximize\", study_name='LightGBM H1N1 Vaccine optimization')\n",
    "LGBM_study_H1N1.optimize(objective_lgbm_h1n1, n_trials=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 80,\n",
       " 'learning_rate': 0.1,\n",
       " 'subsample': 0.9500000000000001,\n",
       " 'colsample_bytree': 0.65,\n",
       " 'min_child_samples': 19,\n",
       " 'min_child_weight': 0.001}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LGBM_study_H1N1.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(root / \"models\\\\LGBM_H1N1_best_params.pkl\",\"wb\") as f:\n",
    "    pickle.dump(LGBM_study_H1N1.best_params, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM - Seasonal Vaccine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_cols = [col for col in X_train_full_seas.columns.tolist() if col not in ['h1n1_vaccine', 'seasonal_vaccine', 'kfold']]\n",
    "target_cols = ['seasonal_vaccine']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2021-02-10 14:53:55,012]\u001b[0m A new study created in memory with name: LightGBM Seasonal Vaccine optimization\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:53:57,967]\u001b[0m Trial 0 finished with value: 0.819270770727602 and parameters: {'n_estimators': 130, 'learning_rate': 0.75, 'subsample': 0.8500000000000001, 'colsample_bytree': 0.8, 'min_child_samples': 23, 'min_child_weight': 0.001}. Best is trial 0 with value: 0.819270770727602.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:00,982]\u001b[0m Trial 1 finished with value: 0.8609902587065685 and parameters: {'n_estimators': 110, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bytree': 0.75, 'min_child_samples': 28, 'min_child_weight': 0.001}. Best is trial 1 with value: 0.8609902587065685.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:04,906]\u001b[0m Trial 2 finished with value: 0.8605655330230816 and parameters: {'n_estimators': 100, 'learning_rate': 0.1, 'subsample': 0.8, 'colsample_bytree': 0.7, 'min_child_samples': 23, 'min_child_weight': 0.001}. Best is trial 1 with value: 0.8609902587065685.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:09,319]\u001b[0m Trial 3 finished with value: 0.8524134460564359 and parameters: {'n_estimators': 70, 'learning_rate': 0.025, 'subsample': 1.0, 'colsample_bytree': 0.8, 'min_child_samples': 25, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8609902587065685.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:13,371]\u001b[0m Trial 4 finished with value: 0.8600400139966164 and parameters: {'n_estimators': 140, 'learning_rate': 0.1, 'subsample': 0.9, 'colsample_bytree': 0.85, 'min_child_samples': 21, 'min_child_weight': 0.0005}. Best is trial 1 with value: 0.8609902587065685.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:15,475]\u001b[0m Trial 5 finished with value: 0.8301206610683718 and parameters: {'n_estimators': 60, 'learning_rate': 0.75, 'subsample': 0.9500000000000001, 'colsample_bytree': 0.65, 'min_child_samples': 29, 'min_child_weight': 0.0015}. Best is trial 1 with value: 0.8609902587065685.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:17,661]\u001b[0m Trial 6 finished with value: 0.8610665213967357 and parameters: {'n_estimators': 70, 'learning_rate': 0.1, 'subsample': 0.9, 'colsample_bytree': 0.8, 'min_child_samples': 25, 'min_child_weight': 0.0015}. Best is trial 6 with value: 0.8610665213967357.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:20,379]\u001b[0m Trial 7 finished with value: 0.8150666354179523 and parameters: {'n_estimators': 130, 'learning_rate': 0.75, 'subsample': 0.9, 'colsample_bytree': 0.65, 'min_child_samples': 26, 'min_child_weight': 0.0015}. Best is trial 6 with value: 0.8610665213967357.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:22,800]\u001b[0m Trial 8 finished with value: 0.8211180376715208 and parameters: {'n_estimators': 110, 'learning_rate': 0.75, 'subsample': 0.9500000000000001, 'colsample_bytree': 0.8, 'min_child_samples': 23, 'min_child_weight': 0.0015}. Best is trial 6 with value: 0.8610665213967357.\u001b[0m\n",
      "\u001b[32m[I 2021-02-10 14:54:25,330]\u001b[0m Trial 9 finished with value: 0.8195036118864525 and parameters: {'n_estimators': 110, 'learning_rate': 0.75, 'subsample': 0.9, 'colsample_bytree': 0.7, 'min_child_samples': 21, 'min_child_weight': 0.0005}. Best is trial 6 with value: 0.8610665213967357.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "LGBM_study_SEAS = optuna.create_study(direction=\"maximize\", study_name='LightGBM Seasonal Vaccine optimization')\n",
    "LGBM_study_SEAS.optimize(objective_lgbm_seas, n_trials=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 70,\n",
       " 'learning_rate': 0.1,\n",
       " 'subsample': 0.9,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'min_child_samples': 25,\n",
       " 'min_child_weight': 0.0015}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LGBM_study_SEAS.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(root / \"models\\\\LGBM_SEAS_best_params.pkl\",\"wb\") as f:\n",
    "    pickle.dump(LGBM_study_SEAS.best_params, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}