notebooks/webinar/how_to_optimize_hyperparameters_with_hyperopt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:18:16.424371",
     "start_time": "2017-01-31T16:18:15.315651"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.metrics import mean_squared_error, make_scorer\n",
    "from sklearn.datasets import make_regression \n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "from hyperopt import fmin, tpe, Trials, hp, STATUS_OK\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:19:44.572255",
     "start_time": "2017-01-31T16:19:44.567557"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_EVALS = 100 \n",
    "SEED = 314 \n",
    "NUMBER_SAMPLES = 200\n",
    "TEST_SIZE = 0.2 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A hp grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:22:34.859901",
     "start_time": "2017-01-31T16:22:34.850250"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hyperopt_hp_grid = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 10, 1000, 1),\n",
    "    'learning_rate': hp.loguniform('learning_rate', 0.001, 0.1),\n",
    "    'max_depth': hp.quniform('max_depth', 3, 15, 1),\n",
    "    'gamma': hp.loguniform('gamma', 0.01, 1)\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:23:23.596656",
     "start_time": "2017-01-31T16:23:23.435300"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features, targets = make_regression(NUMBER_SAMPLES, random_state=SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:24:44.904383",
     "start_time": "2017-01-31T16:24:44.895873"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_features, test_features, train_targets, test_targets = train_test_split(features, targets, test_size=TEST_SIZE, random_state=SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define the score function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T16:25:18.701802",
     "start_time": "2017-01-31T16:25:18.697175"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mse_scorer = make_scorer(mean_squared_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperopt methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T17:26:26.989513",
     "start_time": "2017-01-31T17:26:26.942460"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def transform_params(params):\n",
    "    params['gamma'] = np.log(params['gamma'])\n",
    "    params['learning_rate'] = np.log(params['learning_rate'])\n",
    "    params['n_estimators'] = int(params['n_estimators'])\n",
    "    params['max_depth'] = int(params['max_depth'])\n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T17:26:27.986817",
     "start_time": "2017-01-31T17:26:27.980594"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def loss(params):\n",
    "    params = transform_params(params)\n",
    "    model = xgb.XGBRegressor(silent=False, **params)\n",
    "    cv_loss = cross_val_score(model, train_features, train_targets, cv=5, n_jobs=4, scoring=mse_scorer)\n",
    "    return {'loss': cv_loss.mean(), 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T17:46:49.598313",
     "start_time": "2017-01-31T17:46:49.590907"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimize(trials, space):\n",
    "    best = fmin(loss, space, algo=tpe.suggest, max_evals=MAX_EVALS, trials=trials)\n",
    "    return best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T17:26:49.139041",
     "start_time": "2017-01-31T17:26:49.134008"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trials = Trials()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T17:52:50.936609",
     "start_time": "2017-01-31T17:46:53.120437"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hyperopt_opt_hp = optimize(trials, hyperopt_hp_grid)\n",
    "hyperopt_opt_hp = transform_params(hyperopt_opt_hp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-01-31T18:03:12.296297",
     "start_time": "2017-01-31T18:03:12.278535"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'gamma': 0.55199695557430306,\n",
       " 'learning_rate': 0.046608179354446229,\n",
       " 'max_depth': 3,\n",
       " 'n_estimators': 420}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hyperopt_opt_hp"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:hyperparameters-optimization]",
   "language": "python",
   "name": "conda-env-hyperparameters-optimization-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "toc": {
   "toc_cell": false,
   "toc_number_sections": true,
   "toc_threshold": 6,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}