Parameter Tuning with Bayesian Optimization (final).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "from skopt import gp_minimize, forest_minimize\n",
    "from skopt.utils import use_named_args\n",
    "from skopt.space import Real, Integer, uniform, Log10, Categorical \n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from catboost import CatBoostRegressor\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, KFold\n",
    "from sklearn.preprocessing import RobustScaler, MinMaxScaler\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import make_scorer, mean_squared_error\n",
    "\n",
    "from sklearn.linear_model import Ridge, Lasso, ElasticNet, BayesianRidge\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "\n",
    "from skopt.plots import plot_convergence\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not sys.warnoptions:\n",
    "    import warnings\n",
    "    warnings.simplefilter(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train   = pd.read_csv('train_cleaning.csv')\n",
    "test    = pd.read_csv('test_cleaning.csv')\n",
    "y_train = pd.read_csv('y_train.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.values.ravel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split Data into training and validation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, Y_train, Y_val = train_test_split(train, y_train, test_size = 0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tune hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kernel Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.12017173630105528\n",
      "Best parameters using forest: [0.8823747834898533, 'polynomial', 2, 3.271883237199301]\n",
      "Best score using gp: 0.1201002868065733\n",
      "Best parameters using gp: [0.9815970581715737, 'polynomial', 2, 3.5]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "KR = KernelRidge()\n",
    "\n",
    "### Suggest parameters\n",
    "params_KR = [Real(0, 1, name='alpha'),\n",
    "             Categorical(['linear', 'polynomial'], name='kernel'),\n",
    "            Integer(0, 5, name='degree'),\n",
    "            Real(0.8, 3.5, name='coef0')] \n",
    "\n",
    "\n",
    "@use_named_args(params_KR) \n",
    "def KR_cross_val(**params):\n",
    "    KR.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(KR, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "KR_forest = forest_minimize(KR_cross_val, params_KR, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "KR_gp = gp_minimize(KR_cross_val, params_KR, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(KR_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(KR_forest.x))\n",
    "print(\"Best score using gp: {}\".format(KR_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(KR_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elastic Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.11791052772354833\n",
      "Best parameters using forest: [0.0003319688370362717, 0.9491280902552712, 3]\n",
      "Best score using gp: 0.11780500178610491\n",
      "Best parameters using gp: [0.00028349467970292434, 0.999990545429329, 0]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "ENet = ElasticNet()\n",
    "\n",
    "### Suggest parameters\n",
    "params_ENet = [Real(5e-8, 5e-4, name='alpha'),\n",
    "              Real(0.2, 1.0, name='l1_ratio'),\n",
    "              Integer(0, 5, name='random_state')]\n",
    "\n",
    "@use_named_args(params_ENet) \n",
    "def ENet_cross_val(**params):\n",
    "    ENet.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(ENet, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "ENet_forest = forest_minimize(ENet_cross_val, params_ENet, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "ENet_gp = gp_minimize(ENet_cross_val, params_ENet, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(ENet_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(ENet_forest.x))\n",
    "print(\"Best score using gp: {}\".format(ENet_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(ENet_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bayesian Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.12091791624380736\n",
      "Best parameters using forest: [636, 0.006679796535760416, 0.009297122618697523, 0.004265266270456232, 0.00019827930480987256, 0.0003983852956390158]\n",
      "Best score using gp: 0.12091745390933518\n",
      "Best parameters using gp: [100, 6e-05, 0.01, 0.01, 6e-05, 1e-05]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "BRidge = BayesianRidge()\n",
    "\n",
    "### Suggest parameters\n",
    "params_BRidge = [Integer(100, 800, name='n_iter'),\n",
    "                Real(6.e-5, 1.e-2, name='alpha_1'),\n",
    "                Real(6.e-5, 1.e-2, name='alpha_2'),\n",
    "                Real(6.e-5, 1.e-2, name='lambda_1'),\n",
    "                Real(6.e-5, 1.e-2, name='lambda_2'),\n",
    "                Real(1.e-5, 1.e-3, name='tol')]\n",
    "\n",
    "\n",
    "@use_named_args(params_BRidge) \n",
    "def BRidge_cross_val(**params):\n",
    "    BRidge.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(BRidge, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "BRidge_forest = forest_minimize(BRidge_cross_val, params_BRidge, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "BRidge_gp = gp_minimize(BRidge_cross_val, params_BRidge, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(BRidge_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(BRidge_forest.x))\n",
    "print(\"Best score using gp: {}\".format(BRidge_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(BRidge_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.11781159209470064\n",
      "Best parameters using forest: [0.0002649493027946075, 1]\n",
      "Best score using gp: 0.11780330323019933\n",
      "Best parameters using gp: [0.0002809545919333257, 5]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "Lass = Lasso()\n",
    "\n",
    "### Suggest parameters\n",
    "params_Lass = [Real(5e-5, 5e-4, name='alpha'),\n",
    "               Integer(1, 5, name='random_state')]\n",
    "\n",
    "@use_named_args(params_Lass) \n",
    "def Lass_cross_val(**params):\n",
    "    Lass.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(Lass, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "Lass_forest = forest_minimize(Lass_cross_val, params_Lass, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "Lass_gp = gp_minimize(Lass_cross_val, params_Lass, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(Lass_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(Lass_forest.x))\n",
    "print(\"Best score using gp: {}\".format(Lass_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(Lass_gp.x)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.11708911585189467\n",
      "Best parameters using forest: [4, 0.008462445689673028, 0.96507271384635, 1, 0.5384179495416508, 2048]\n",
      "Best score using gp: 0.11213851621648407\n",
      "Best parameters using gp: [3, 0.022496892681737726, 0.6982452734370306, 1, 0.1, 2500]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "XGBoost = xgb.XGBRegressor()\n",
    "\n",
    "### Suggest parameters\n",
    "params_XGBoost = [Integer(1, 5, name='max_depth'),\n",
    "          Real(10**-5, 10**0, \"log-uniform\", name='learning_rate'),\n",
    "          Real(0, 1, name='subsample'),\n",
    "#           Real(0, 1, name='reg_alpha'),\n",
    "#           Real(0, 1, name='reg_lambda'),\n",
    "          Integer(1, 100, name='min_child_weight'),\n",
    "          Real(0.1, 1, name='colsample_bytree'),\n",
    "#           Integer(0, 10, name='random_state'),\n",
    "          Integer(500, 2500, name='n_estimators'),\n",
    "#           Real(0, 1, name='gamma')\n",
    "         ]\n",
    "\n",
    "@use_named_args(params_XGBoost) \n",
    "def XGBoost_cross_val(**params):\n",
    "    XGBoost.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(XGBoost, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "XGBoost_forest = forest_minimize(XGBoost_cross_val, params_XGBoost, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "XGBoost_gp = gp_minimize(XGBoost_cross_val, params_XGBoost, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(XGBoost_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(XGBoost_forest.x))\n",
    "print(\"Best score using gp: {}\".format(XGBoost_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(XGBoost_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.11484191083214167\n",
      "Best parameters using forest: [4, 0.009731934054429674, 'log2', 59, 'ls', 5, 2500, 5]\n",
      "Best score using gp: 0.11158165300008473\n",
      "Best parameters using gp: [4, 0.022295633577807207, 'sqrt', 100, 'huber', 2, 3000, 0]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "Graboost = GradientBoostingRegressor()\n",
    "\n",
    "### Suggest parameters\n",
    "params_Graboost = [Integer(1, 5, name='max_depth'),\n",
    "          Real(5e-7, 5e-2, name='learning_rate'),\n",
    "          #Integer(1, x_train.shape[1], name='max_features'),\n",
    "          Categorical(['auto', 'sqrt', 'log2'], name='max_features'),\n",
    "          Integer(2, 100, name='min_samples_split'),\n",
    "          Categorical(['huber', 'ls', 'lad', 'quantile'], name='loss'),\n",
    "          Integer(2, 100, name='min_samples_leaf'),\n",
    "          Integer(50, 3000, name='n_estimators'),\n",
    "          Integer(0, 5, name='random_state'),\n",
    "         ]\n",
    "\n",
    "@use_named_args(params_Graboost) \n",
    "def Graboost_cross_val(**params):\n",
    "    Graboost.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(Graboost, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "Graboost_forest = forest_minimize(Graboost_cross_val, params_Graboost, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "Graboost_gp = gp_minimize(Graboost_cross_val, params_Graboost, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(Graboost_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(Graboost_forest.x))\n",
    "print(\"Best score using gp: {}\".format(Graboost_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(Graboost_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.12088383125384887\n",
      "Best parameters using forest: [9.57933749557496, 'svd']\n",
      "Best score using gp: 0.12084343540191145\n",
      "Best parameters using gp: [8.843480739403299, 'cholesky']\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "RidgeR = Ridge()\n",
    "\n",
    "### Suggest parameters\n",
    "params_RidgeR = [Real(0.05, 35, name='alpha'),\n",
    "                Categorical(['auto', 'svd', 'cholesky'], name='solver')]\n",
    "\n",
    "@use_named_args(params_RidgeR) \n",
    "def RidgeR_cross_val(**params):\n",
    "    RidgeR.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(RidgeR, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "RidgeR_forest = forest_minimize(RidgeR_cross_val, params_RidgeR, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "RidgeR_gp = gp_minimize(RidgeR_cross_val, params_RidgeR, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(RidgeR_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(RidgeR_forest.x))\n",
    "print(\"Best score using gp: {}\".format(RidgeR_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(RidgeR_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.12631631587322953\n",
      "Best parameters using forest: [0.012955227267013002, 49, 'regression', 0.3525455167623106, 476, 0.8492987114701519, 16, 90]\n",
      "Best score using gp: 0.11860755354959966\n",
      "Best parameters using gp: [0.016612303637686433, 30, 'regression', 1.0, 500, 0.21753146276979324, 7, 68]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "LGBM = lgb.LGBMRegressor()\n",
    "\n",
    "### Suggest parameters\n",
    "params_LGBM = [#Categorical(['gbdt','rf','dart', 'goss'], name='boosting_type'),\n",
    "#              Integer(50, 10000, name='n_estimators'),\n",
    "             Real(1e-7, 2e-2, name='learning_rate'),\n",
    "             Integer(2, 100, name='num_leaves'),\n",
    "             Categorical(['regression'], name='objective'),\n",
    "#              Integer(1, 100, name='bagging_freq'),\n",
    "             Real(0.1, 1.0, name='bagging_fraction'),\n",
    "             Integer(50, 500, name='num_iterations'),\n",
    "             Real(0.1, 1, name='feature_fraction'),\n",
    "#              Real(0, 10, name='reg_alpha'),\n",
    "             Integer(1, 100, name='min_data_in_leaf'),\n",
    "             Integer(1, 100, name='max_depth')]\n",
    "\n",
    "@use_named_args(params_LGBM) \n",
    "def LGBM_cross_val(**params):\n",
    "    LGBM.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(LGBM, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "LGBM_forest = forest_minimize(LGBM_cross_val, params_LGBM, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "LGBM_gp = gp_minimize(LGBM_cross_val, params_LGBM, n_calls=100, acq_func=\"gp_hedge\", random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(LGBM_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(LGBM_forest.x))\n",
    "print(\"Best score using gp: {}\".format(LGBM_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(LGBM_gp.x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CatBoostRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score using forest: 0.14860429908957817\n",
      "Best parameters using forest: [420, 0.03826404453243567, 3, 'RMSE', 1, 0.36722528261107323, 20]\n",
      "Best score using gp: 0.12706474879033458\n",
      "Best parameters using gp: [929, 0.022859933246184225, 2, 'RMSE', 0, 1.0, 194]\n"
     ]
    }
   ],
   "source": [
    "### Define model\n",
    "CatB = CatBoostRegressor()\n",
    "\n",
    "### Suggest parameters\n",
    "params_CatB = [Integer(300, 1000, name='iterations'),\n",
    "              Real(10**-5, 10**0, \"log-uniform\", name='learning_rate'),\n",
    "              Integer(0, 10, name='depth'),\n",
    "              Categorical(['RMSE'], name='loss_function'),\n",
    "#               Integer(0, 100, name='random_seed'),\n",
    "#               Categorical(['Silent'],name='logging_level'),\n",
    "#               Integer(0, 100, name='bagging_temperature'),\n",
    "              Integer(0, 100, name='l2_leaf_reg'),\n",
    "              Real(0.001, 1, name='rsm'),\n",
    "              Integer(1, 255, name='one_hot_max_size')\n",
    "              ]\n",
    "\n",
    "@use_named_args(params_CatB) \n",
    "def CatB_cross_val(**params):\n",
    "    CatB.set_params(**params)\n",
    "    kf = KFold(5, shuffle=True, random_state=42).get_n_splits(train)\n",
    "    rmse= np.sqrt(-np.mean(cross_val_score(CatB, X=train, y=y_train, scoring=\"neg_mean_squared_error\", cv = kf, n_jobs=-1)))\n",
    "    return(rmse)\n",
    "\n",
    "CatB_forest = forest_minimize(CatB_cross_val, params_CatB, acq_func=\"EI\", n_calls=100, random_state=0)\n",
    "CatB_gp = gp_minimize(CatB_cross_val, params_CatB, acq_func=\"gp_hedge\", n_calls=100, random_state=0)\n",
    "\n",
    "print(\"Best score using forest: {}\".format(CatB_forest.fun))\n",
    "print(\"Best parameters using forest: {}\".format(CatB_forest.x))\n",
    "print(\"Best score using gp: {}\".format(CatB_gp.fun))\n",
    "print(\"Best parameters using gp: {}\".format(CatB_gp.x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}