generate_results-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# batch script for reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "jupyter nbconvert --to script generate_results.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "matplotlib.use('Agg')\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "from matplotlib.colors import LinearSegmentedColormap as lsc\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "import sklearn \n",
    "import xgboost\n",
    "\n",
    "import math\n",
    "import random \n",
    "import re\n",
    "import itertools\n",
    "\n",
    "from collections import Counter \n",
    "\n",
    "from scipy.stats import binom_test\n",
    "from scipy.stats import norm\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.preprocessing import PolynomialFeatures as plf\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_val_predict \n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.metrics import mean_squared_error as mse\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.linear_model import Lasso \n",
    "from sklearn.linear_model import Ridge \n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "from ksuzuki_pylib import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sklearn: 0.21.0\n",
      "xgboost: 0.82\n",
      "pandas: 0.24.2\n",
      "numpy: 1.16.3\n",
      "scipy: 1.2.1\n",
      "matplotlib: 3.0.3\n"
     ]
    }
   ],
   "source": [
    "print('sklearn:', sklearn.__version__)\n",
    "print('xgboost:', xgboost.__version__)\n",
    "print('pandas:', pd.__version__)\n",
    "print('numpy:', np.__version__)\n",
    "print('scipy:', scipy.__version__)\n",
    "print('matplotlib:', matplotlib.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xgb ranges\n",
    "range_depth = [6, 7, 8]\n",
    "range_subsample = [0.8, 0.9, 1]\n",
    "range_colsample = [0.8, 0.9, 1]\n",
    "range_lr = [0.1, 0.05]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ranges\n",
    "logparams1 = [1e-2, 1e-1, 1e0, 1e1, 1e2]\n",
    "logparams2 = [1.0, 1e1, 1e2, 1e3, 1e4, 1e5]\n",
    "logparams3 = [1.0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5]\n",
    "logparams4 = [1.0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10]\n",
    "\n",
    "range_lasso = logparams1\n",
    "range_ridge = logparams1\n",
    "range_krr_alpha = logparams3\n",
    "range_krr_gamma = logparams3\n",
    "range_svr_C = logparams2\n",
    "range_svr_gamma = logparams4\n",
    "range_svr_epsilon = logparams1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for debugging\n",
    "debug_run = False\n",
    "debug_run = True\n",
    "if debug_run:\n",
    "    # xgb ranges\n",
    "    range_depth = [3,4]\n",
    "    range_subsample = [0.9, 1]\n",
    "    range_colsample = [0.9, 1]\n",
    "    range_lr = [0.1, 0.05]\n",
    "    \n",
    "    # ranges\n",
    "    logparams1 = [1e-2, 1e-1]\n",
    "    logparams2 = [1.0, 1e1]\n",
    "    logparams3 = [1.0, 1e-1]\n",
    "    logparams4 = [1.0, 1e-1]\n",
    "    \n",
    "    range_lasso = logparams1\n",
    "    range_ridge = logparams1\n",
    "    range_krr_alpha = logparams3\n",
    "    range_krr_gamma = logparams3\n",
    "    range_svr_C = logparams2\n",
    "    range_svr_gamma = logparams4\n",
    "    range_svr_epsilon = logparams1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OCM\n",
    "ntree = 500\n",
    "if debug_run:\n",
    "    ntree = 10\n",
    "    \n",
    "# Lasso\n",
    "cv_lasso = GridSearchCV(\n",
    "    Lasso(random_state=929), \n",
    "    param_grid={\"alpha\": range_lasso})\n",
    "\n",
    "# Ridge\n",
    "cv_ridge = GridSearchCV(\n",
    "    Ridge(random_state=929), \n",
    "    param_grid={\"alpha\": range_ridge})\n",
    "\n",
    "# KRR\n",
    "cv_krr = GridSearchCV(\n",
    "    KernelRidge(kernel='rbf'), \n",
    "    param_grid={\"alpha\": range_krr_alpha, \n",
    "                \"gamma\": range_krr_gamma})\n",
    "\n",
    "# SVR\n",
    "cv_svr = GridSearchCV(\n",
    "    SVR(kernel='rbf'), \n",
    "    param_grid={\"C\": range_svr_C, \n",
    "                \"gamma\": range_svr_gamma, \n",
    "                \"epsilon\": range_svr_epsilon})\n",
    "\n",
    "# XGB\n",
    "cv_xgb = GridSearchCV(\n",
    "    XGBRegressor(random_state=929), \n",
    "    param_grid={\"max_depth\": range_depth, \n",
    "                \"subsample\": range_subsample, \n",
    "                \"colsample_bytree\": range_colsample,\n",
    "                \"learning_rate\": range_lr})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OCM data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocm = pd.read_csv(\"input/OCM_matrix.csv\").drop(['Unnamed: 0'], axis=1)\n",
    "\n",
    "ocm_desc = pd.read_csv(\"input/OCM_matrix_desc.csv\")\n",
    "ind = ocm_desc['Unnamed: 0']\n",
    "ocm_desc.index = ind\n",
    "ocm_desc = ocm_desc.drop(['Unnamed: 0'], axis=1)\n",
    "\n",
    "#exclude cat which has 'Th' in its compser\n",
    "ind = ocm['Th'] == 0\n",
    "ocm = ocm[ind]\n",
    "\n",
    "#exclude data which has over 5 metals in his component\n",
    "ind = (ocm.loc[:,:'Zr']>0).sum(axis=1)\n",
    "ind = ind < 5\n",
    "ocm = ocm.loc[ind]\n",
    "\n",
    "ind = (ocm_desc.loc[:,:'Zr']>0).sum(axis=1)\n",
    "ind = ind < 5\n",
    "ocm_desc = ocm_desc.loc[ind]\n",
    "\n",
    "ocm = ocm.drop(['Support_Co'],axis=1)\n",
    "ocm_desc = ocm_desc.drop(['Support_Co'],axis=1)\n",
    "\n",
    "# rename columns\n",
    "sup = ocm.loc[:,'Support_Zr':'Support_Si'].columns\n",
    "prom = ocm.loc[:,'Promotor_B':'Promotor_S'].columns\n",
    "comp = ocm.loc[:,:'Zr'].columns\n",
    "cond = ocm.loc[:,'Temperature, K':'P total, bar'].columns\n",
    "prep = ocm.loc[:,'Impregnation':'Therm.decomp.'].columns\n",
    "\n",
    "#exclude experimental condition feature\n",
    "nocond = list(comp) + list(sup)\n",
    "ocm_nocond = ocm[nocond]\n",
    "ocm_nocond = pd.concat([ocm_nocond,ocm.iloc[:,-1]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pd.read_csv('input/Descriptors.csv',skiprows = [0],index_col='symbol').drop(['Unnamed: 0',\n",
    "                                                                               'name',\n",
    "                                                                               'ionic radius',\n",
    "                                                                               'covalent radius',\n",
    "                                                                               'VdW radius',\n",
    "                                                                               'crystal radius',\n",
    "                                                                               'a x 106 ',\n",
    "                                                                               'Heat capacity ',\n",
    "                                                                               'l',\n",
    "                                                                               'electron affinity ',\n",
    "                                                                               'VE',\n",
    "                                                                               'Surface energy '],axis=1)\n",
    "desc = desc.loc[ocm.loc[:,:'Zr'].columns,]\n",
    "desc = desc.fillna(desc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({2: 779, 3: 578, 1: 345, 4: 131})\n"
     ]
    }
   ],
   "source": [
    "# num of elements in each multicomponent catalyst\n",
    "print(Counter((ocm.loc[:,:'Zr']>0).sum(axis=1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scatter(pred,exp,xlabel='Yield_ex(%)',ylabel='Yield_pr(%)',Title='',col='blue',wt_value=True,save=False):\n",
    "    range = [ocm.iloc[:,-1].min() ,ocm.iloc[:,-1].max() ]\n",
    "    val0 = (pred - exp)**2\n",
    "    str0 = 'RMSE: %1.3f' % math.sqrt(val0.mean())\n",
    "    plt.figure(figsize=(3, 3), dpi=100)\n",
    "    plt.plot(range, range, color='0.5')\n",
    "    plt.scatter(exp,pred, s=10,\n",
    "                facecolors=col, edgecolors=col, alpha=0.4)\n",
    "    plt.xlim(range[0], range[1])\n",
    "    plt.ylim(range[0], range[1])\n",
    "    if wt_value:\n",
    "        plt.text(range[1] - 0.97 * (range[1] - range[0]),\n",
    "                 range[1] - 0.05 * (range[1] - range[0]), str0, fontsize=8.5)\n",
    "\n",
    "    plt.xlabel(xlabel, fontsize=14)\n",
    "    plt.ylabel(ylabel, fontsize=14)\n",
    "    plt.title(Title, fontsize=15)\n",
    "    #plt.legend(loc='lower right')\n",
    "\n",
    "    if save :\n",
    "        plt.savefig(Title.replace(' ','_') + 'one_shot' + '.png', format='png',\n",
    "                    bbox_inches='tight', dpi=300)\n",
    "        plt.savefig(Title.replace(' ','_') + 'one_shot' + '.pdf', format='pdf',\n",
    "                    bbox_inches='tight', dpi=300)\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 1 0.9 0.05\n"
     ]
    }
   ],
   "source": [
    "cv_xgb.fit(ocm_nocond.iloc[:,:-1], ocm_nocond.iloc[:,-1])\n",
    "\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "\n",
    "print(cvdepth, cvsubsample, cvcolsample, cvlr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bytree=0.9, gamma=0, importance_type='gain',\n",
      "             learning_rate=0.05, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=None, n_estimators=500, n_jobs=-1,\n",
      "             nthread=None, objective='reg:linear', random_state=929,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=929,\n",
      "             silent=True, subsample=1)\n"
     ]
    }
   ],
   "source": [
    "estimator = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)\n",
    "print(estimator)\n",
    "\n",
    "best_xgb_ocmnocond = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(ocm_nocond, random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE_test: 5.373 STD_test: 0.368\n",
      "RMSE_train: 4.090 STD_train: 0.024\n",
      "Data size:(1833, 69)\n"
     ]
    }
   ],
   "source": [
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10)\n",
    "cv.cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1650, 69) (1650,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE(Train): 4.050\n",
      "RMSE(Test): 5.349\n"
     ]
    }
   ],
   "source": [
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      titlesize = 11, \\\n",
    "                      filename='output/fig5_onlycomposition_',\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 1 0.9 0.1\n"
     ]
    }
   ],
   "source": [
    "cv_xgb.fit(ocm.iloc[:,:-1], ocm.iloc[:,-1])\n",
    "\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "\n",
    "print(cvdepth, cvsubsample, cvcolsample, cvlr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bytree=0.9, gamma=0, importance_type='gain',\n",
      "             learning_rate=0.1, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=None, n_estimators=500, n_jobs=-1,\n",
      "             nthread=None, objective='reg:linear', random_state=929,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=929,\n",
      "             silent=True, subsample=1)\n"
     ]
    }
   ],
   "source": [
    "estimator = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)\n",
    "print(estimator)\n",
    "\n",
    "best_xgb_ocm = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(ocm, random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE_test: 4.443 STD_test: 0.324\n",
      "RMSE_train: 2.224 STD_train: 0.025\n",
      "Data size:(1833, 104)\n"
     ]
    }
   ],
   "source": [
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10)\n",
    "cv.cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1650, 104) (1650,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE(Train): 2.224\n",
      "RMSE(Test): 4.519\n"
     ]
    }
   ],
   "source": [
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      titlesize = 11, \\\n",
    "                      filename='output/fig5_composition&condition_',\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 0.9 1 0.05\n"
     ]
    }
   ],
   "source": [
    "cv_xgb.fit(ocm_desc.iloc[:,:-1],ocm_desc.iloc[:,-1])\n",
    "\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "\n",
    "print(cvdepth, cvsubsample, cvcolsample, cvlr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "             colsample_bytree=1, gamma=0, importance_type='gain',\n",
      "             learning_rate=0.05, max_delta_step=0, max_depth=4,\n",
      "             min_child_weight=1, missing=None, n_estimators=500, n_jobs=-1,\n",
      "             nthread=None, objective='reg:linear', random_state=929,\n",
      "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=929,\n",
      "             silent=True, subsample=0.9)\n"
     ]
    }
   ],
   "source": [
    "estimator = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)\n",
    "print(estimator)\n",
    "\n",
    "best_xgb_ocmdesc = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(ocm_desc, random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE_test: 4.354 STD_test: 0.242\n",
      "RMSE_train: 2.280 STD_train: 0.035\n",
      "Data size:(1833, 148)\n"
     ]
    }
   ],
   "source": [
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_cvbest = cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1649, 148) (1649,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE(Train): 2.273\n",
      "RMSE(Test): 4.288\n"
     ]
    }
   ],
   "source": [
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4-fig5_gb_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(ocm_desc, random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 17197.705109852704, tolerance: 5.5818553610883805\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 17097.472180321434, tolerance: 5.508298215425533\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 15835.167281500924, tolerance: 5.381936177577743\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 19531.383364111738, tolerance: 5.5818553610883805\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 18672.49757780483, tolerance: 5.508298215425533\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 18175.867534146866, tolerance: 5.381936177577743\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 27829.82333835948, tolerance: 8.251709865248227\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso(alpha=0.01, copy_X=True, fit_intercept=True, max_iter=1000,\n",
      "      normalize=False, positive=False, precompute=False, random_state=929,\n",
      "      selection='cyclic', tol=0.0001, warm_start=False)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 25163.081208441778, tolerance: 7.544120143723469\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 25022.687105336274, tolerance: 7.431372534869618\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 25715.04039060453, tolerance: 7.516430010915706\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24479.535559067954, tolerance: 7.313736882575759\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 25121.169611549343, tolerance: 7.505410631666666\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24591.76850899667, tolerance: 7.405482246212122\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24689.096255673663, tolerance: 7.377854583181818\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24621.80324195742, tolerance: 7.440634806212122\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 25071.94796890718, tolerance: 7.442958406212123\n",
      "  positive)\n",
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24113.74008292837, tolerance: 7.283308322575759\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE_test: 5.987 STD_test: 0.512\n",
      "RMSE_train: 5.402 STD_train: 0.049\n",
      "Data size:(1833, 148)\n",
      "(1650, 148) (1650,)\n",
      "RMSE(Train): 5.381\n",
      "RMSE(Test): 6.059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/linear_model/coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 24689.096255673692, tolerance: 7.377854583181819\n",
      "  positive)\n"
     ]
    }
   ],
   "source": [
    "# LASSO\n",
    "cv_lasso.fit(ocm_desc.iloc[:,:-1],ocm_desc.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "estimator = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "best_lasso_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_lasso_', \\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge(alpha=0.1, copy_X=True, fit_intercept=True, max_iter=None,\n",
      "      normalize=False, random_state=929, solver='auto', tol=0.001)\n",
      "RMSE_test: 6.338 STD_test: 0.546\n",
      "RMSE_train: 5.356 STD_train: 0.049\n",
      "Data size:(1833, 148)\n",
      "(1649, 148) (1649,)\n",
      "RMSE(Train): 5.453\n",
      "RMSE(Test): 6.345\n"
     ]
    }
   ],
   "source": [
    "# Ridge\n",
    "cv_ridge.fit(ocm_desc.iloc[:,:-1],ocm_desc.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "estimator = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "best_ridge_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_ridge_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KernelRidge(alpha=0.1, coef0=1, degree=3, gamma=0.1, kernel='rbf',\n",
      "            kernel_params=None)\n",
      "RMSE_test: 10.857 STD_test: 0.571\n",
      "RMSE_train: 1.063 STD_train: 0.007\n",
      "Data size:(1833, 148)\n",
      "(1650, 148) (1650,)\n",
      "RMSE(Train): 1.054\n",
      "RMSE(Test): 10.910\n"
     ]
    }
   ],
   "source": [
    "# KRR\n",
    "cv_krr.fit(ocm_desc.iloc[:,:-1],ocm_desc.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "estimator = KernelRidge(kernel='rbf',alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "best_krr_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='', \\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_krr_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/takigawa/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVR(C=10.0, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma=0.1,\n",
      "    kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)\n",
      "RMSE_test: 6.415 STD_test: 0.308\n",
      "RMSE_train: 1.510 STD_train: 0.052\n",
      "Data size:(1833, 148)\n",
      "(1649, 148) (1649,)\n",
      "RMSE(Train): 1.533\n",
      "RMSE(Test): 6.468\n"
     ]
    }
   ],
   "source": [
    "# SVR\n",
    "cv_svr.fit(ocm_desc.iloc[:,:-1], ocm_desc.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "estimator = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "best_svr_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_svr_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='auto', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=2,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1,\n",
      "                      oob_score=False, random_state=929, verbose=0,\n",
      "                      warm_start=False)\n",
      "RMSE_test: 4.376 STD_test: 0.278\n",
      "RMSE_train: 1.623 STD_train: 0.015\n",
      "Data size:(1833, 148)\n",
      "(1649, 148) (1649,)\n",
      "RMSE(Train): 1.633\n",
      "RMSE(Test): 4.297\n"
     ]
    }
   ],
   "source": [
    "# RFR\n",
    "estimator = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "best_rfr_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_rfr_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='auto', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1,\n",
      "                    oob_score=False, random_state=929, verbose=0,\n",
      "                    warm_start=False)\n",
      "RMSE_test: 4.332 STD_test: 0.295\n",
      "RMSE_train: 0.163 STD_train: 0.018\n",
      "Data size:(1833, 148)\n",
      "(1650, 148) (1650,)\n",
      "RMSE(Train): 0.173\n",
      "RMSE(Test): 4.347\n"
     ]
    }
   ],
   "source": [
    "# ETR\n",
    "estimator = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "best_etr_ocmdesc = estimator\n",
    "print(estimator)\n",
    "\n",
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()\n",
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/fig4_etr_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.6 (A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 4), dpi=300)\n",
    "xgb_cvbest.importance.mean(axis=1).sort_values(ascending=False).iloc[:20][::-1].plot(kind='barh',color='b')\n",
    "plt.savefig('output/fig6_ocm.png', format='png',bbox_inches='tight', dpi=300)\n",
    "plt.savefig('output/fig6_ocm.pdf', format='pdf',bbox_inches='tight', dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.2 (A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "common = []\n",
    "tmp = ocm.loc[:,:'Zr']\n",
    "count = 0\n",
    "for key, row in tmp.iterrows():\n",
    "    for i, v in tmp.iloc[count + 1:].iterrows():\n",
    "        temp = (np.array(row) > 1e-5)\n",
    "        temp2 = (np.array(v) > 1e-5)\n",
    "        val = (temp * temp2).sum()\n",
    "        common.append(val)  \n",
    "    count = count + 1\n",
    "\n",
    "freq = dict(Counter(common))\n",
    "plt.figure(figsize=(2.5, 3))\n",
    "plt.bar(freq.keys(), freq.values())\n",
    "plt.xlabel('# Common Elements')\n",
    "plt.ylabel('Frequency (log)')\n",
    "plt.yscale('log')\n",
    "plt.savefig('output/fig2_ocm.png',format='png', bbox_inches='tight',dpi=300)\n",
    "plt.savefig('output/fig2_ocm.pdf',format='pdf', bbox_inches='tight',dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ocm_orig = pd.read_csv(\"input/OCM_matrix.csv\").drop(['Unnamed: 0'], axis=1)\n",
    "val = (ocm_orig.loc[:,:'Zr'] > 0).sum(axis=1).mean()\n",
    "print('%1.2f' % val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat = (ocm_orig.loc[:,:'Zr'] > 0).sum(axis=0).sort_values()\n",
    "ind = dat < 10\n",
    "print(dat[ind])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.3 & Table 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crossvalid(xx,yy,model,cv,txt):\n",
    "    cv = Cv_Pred_Expected(estimator=model,X=xx,y=yy,cv=10,redc = True)\n",
    "    cv.cross_validation()\n",
    "    tes_rmse = cv.rmse\n",
    "    tes_std = cv.std\n",
    "    trn_rmse = cv.rmse_train\n",
    "    trn_std = cv.std_train\n",
    "    print()\n",
    "    print(\"[%s] RMSE %1.3f (STD: %1.3f) ... test\" % (txt,tes_rmse,tes_std))\n",
    "    print(\"[%s] RMSE %1.3f (STD: %1.3f) ... train\" % (txt,trn_rmse,trn_std))\n",
    "    ret_obj = {}\n",
    "    ret_obj['tes_mean'] = tes_rmse\n",
    "    ret_obj['tes_sd']   = tes_std \n",
    "    ret_obj['trn_mean'] = trn_rmse\n",
    "    ret_obj['trn_sd']   = trn_std \n",
    "    return ret_obj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_ocm_nocond = {}\n",
    "data = shuffle(ocm_nocond, random_state=10)\n",
    "\n",
    "target_data = ocm_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_ocm_nocond['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_ocm_nocond['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10,  'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_ocm_nocond['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_ocm_nocond['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_ocm_nocond['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_ocm_nocond['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb_ocmnocond\n",
    "print(model)\n",
    "rmse_ocm_nocond['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_ocm = {}\n",
    "data = shuffle(ocm, random_state=10)\n",
    "target_data = ocm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_ocm['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_ocm['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_ocm['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_ocm['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_ocm['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_ocm['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb_ocm\n",
    "print(model)\n",
    "rmse_ocm['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ocm_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_ocm_desc = {}\n",
    "data = shuffle(ocm_desc, random_state=929)\n",
    "target_data = ocm_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb_ocmdesc\n",
    "print(model)\n",
    "rmse_ocm_desc['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fig.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "methods = ['Lasso','Ridge','KRR','SVR','RFR','ETR','XGB']\n",
    "\n",
    "tes_mean = [rmse_ocm_desc[m]['tes_mean'] for m in methods]\n",
    "tes_sd   = [rmse_ocm_desc[m]['tes_sd']   for m in methods]\n",
    "trn_mean = [rmse_ocm_desc[m]['trn_mean'] for m in methods]\n",
    "trn_sd   = [rmse_ocm_desc[m]['trn_sd']   for m in methods]\n",
    "\n",
    "# 2*sigma = 95% CI\n",
    "tes_ci = 2 * np.array(tes_sd)\n",
    "trn_ci = 2 * np.array(trn_sd)\n",
    "\n",
    "plt.figure(figsize=(6,3))\n",
    "\n",
    "ind = np.arange(len(methods))\n",
    "width = 0.4\n",
    "plt.bar(ind, trn_mean, width, color='0.9', align='center', yerr=trn_ci)\n",
    "plt.bar(ind+width, tes_mean, width, color='0.6', align='center', yerr=tes_ci)\n",
    "\n",
    "plt.ylabel('RMSE (%)')\n",
    "plt.xticks(ind+width/2,methods,rotation=45, fontsize=12)\n",
    "plt.legend(('Training Error',\n",
    "            'Test Error'), loc =\"upper right\", prop={'size':12})\n",
    "plt.ylim(0,np.max(tes_mean)+5)\n",
    "\n",
    "for x,y in zip(ind, trn_mean):\n",
    "    plt.text(x, y+1.5, '%.2f' % y, ha='center', va= 'bottom', rotation='90', fontsize=10)\n",
    "\n",
    "for x,y in zip(ind+width, tes_mean):\n",
    "    plt.text(x, y+1.5, '%.2f' % y, ha='center', va= 'bottom', rotation='90', fontsize=10)\n",
    "\n",
    "plt.savefig('output/fig3.png',format='png', bbox_inches='tight',dpi=300)\n",
    "plt.savefig('output/fig3.pdf',format='pdf', bbox_inches='tight',dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.precision = 2\n",
    "\n",
    "print(pd.DataFrame(rmse_ocm_nocond))\n",
    "\n",
    "print(pd.DataFrame(rmse_ocm))\n",
    "\n",
    "print(pd.DataFrame(rmse_ocm_desc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WGS data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pd.read_csv('input/Descriptors.csv',skiprows = [0],index_col='symbol').drop(['Unnamed: 0',\n",
    "                                                                               'name',\n",
    "                                                                               'ionic radius',\n",
    "                                                                               'covalent radius',\n",
    "                                                                               'VdW radius',\n",
    "                                                                               'crystal radius',\n",
    "                                                                               'a x 106 ',\n",
    "                                                                               'Heat capacity ',\n",
    "                                                                               'l',\n",
    "                                                                               'electron affinity ',\n",
    "                                                                               'VE',\n",
    "                                                                               'Surface energy '],axis=1)\n",
    "desc = desc.iloc[:83, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wgs = pd.read_csv(\"input/wgs.csv\")\n",
    "wgs.index = list(wgs.iloc[:,0])\n",
    "wgs = wgs.drop([wgs.columns[0]], axis=1)\n",
    "\n",
    "wgs_desc = pd.read_csv(\"input/wgs_desc.csv\")\n",
    "wgs_desc.index = list(wgs_desc.iloc[:,0])\n",
    "wgs_desc = wgs_desc.drop([wgs_desc.columns[0]], axis=1)\n",
    "\n",
    "atom = list(wgs.loc[:,:'Pd'].columns) + list(wgs.loc[:,'Li':'Sr'].columns)\n",
    "desc = desc.loc[atom]\n",
    "\n",
    "desc=desc.fillna(desc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WGS\n",
    "ntree = 1500\n",
    "if debug_run:\n",
    "    ntree = 10\n",
    "\n",
    "# Lasso\n",
    "cv_lasso = GridSearchCV(\n",
    "    Lasso(random_state=929), \n",
    "    param_grid={\"alpha\": range_lasso})\n",
    "\n",
    "# Ridge\n",
    "cv_ridge = GridSearchCV(\n",
    "    Ridge(random_state=929), \n",
    "    param_grid={\"alpha\": range_ridge})\n",
    "\n",
    "# KRR\n",
    "cv_krr = GridSearchCV(\n",
    "    KernelRidge(kernel='rbf'), \n",
    "    param_grid={\"alpha\": range_krr_alpha, \n",
    "                \"gamma\": range_krr_gamma})\n",
    "\n",
    "# SVR\n",
    "cv_svr = GridSearchCV(\n",
    "    SVR(kernel='rbf'), \n",
    "    param_grid={\"C\": range_svr_C, \n",
    "                \"gamma\": range_svr_gamma, \n",
    "                \"epsilon\": range_svr_epsilon})\n",
    "\n",
    "# XGB\n",
    "cv_xgb = GridSearchCV(\n",
    "    XGBRegressor(random_state=929), \n",
    "    param_grid={\"max_depth\": range_depth, \n",
    "                \"subsample\": range_subsample, \n",
    "                \"colsample_bytree\": range_colsample,\n",
    "                \"learning_rate\": range_lr})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.6 (B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_xgb.fit(wgs_desc.iloc[:,:-1], wgs_desc.iloc[:,-1])\n",
    "\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "\n",
    "print(cvdepth, cvsubsample, cvcolsample, cvlr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)\n",
    "print(estimator)\n",
    "\n",
    "best_xgb_wgsdesc = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(wgs_desc, random_state=929)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_cvbest = cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/wgs_gb_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 4), dpi=300)\n",
    "xgb_cvbest.importance.mean(axis=1).sort_values(ascending=False).iloc[:20][::-1].plot(kind='barh',color='b')\n",
    "plt.savefig('output/fig6_wgs.png', format='png',bbox_inches='tight', dpi=300)\n",
    "plt.savefig('output/fig6_wgs.pdf', format='pdf',bbox_inches='tight', dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.2 (B)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "common = []\n",
    "hoge = wgs.ix[:,'Pt':'Pd']\n",
    "count = 0\n",
    "for key, row in hoge.iterrows():\n",
    "    for i, v in hoge.iloc[count + 1:].iterrows():\n",
    "        temp = (np.array(row) > 1e-5)\n",
    "        temp2 = (np.array(v) > 1e-5)\n",
    "        val = (temp * temp2).sum()\n",
    "        common.append(val)     \n",
    "    count = count + 1\n",
    "\n",
    "freq = dict(Counter(common))\n",
    "plt.figure(figsize=(1.5, 3))\n",
    "plt.bar(freq.keys(), freq.values())\n",
    "plt.xlabel('# Common Elements')\n",
    "plt.ylabel('Frequency (log)')\n",
    "plt.yscale('log')\n",
    "plt.savefig('output/fig2_wgs.png',format='png', bbox_inches='tight',dpi=300)\n",
    "plt.savefig('output/fig2_wgs.pdf',format='pdf', bbox_inches='tight',dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 4 (wgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### wgs_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comppro = list(wgs.loc[:, :'Pd'].columns)\n",
    "wgs_nocond = wgs.loc[:, comppro + ['CO Conversion']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_wgs_nocond = {}\n",
    "data = shuffle(wgs_nocond, random_state=10)\n",
    "target_data = wgs_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# XGB\n",
    "cv_xgb.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "best_xgb = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_wgs_nocond['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_wgs_nocond['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10,  'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_wgs_nocond['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_wgs_nocond['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_wgs_nocond['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_wgs_nocond['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb\n",
    "print(model)\n",
    "rmse_wgs_nocond['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### wgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_wgs = {}\n",
    "data = shuffle(wgs, random_state=10)\n",
    "target_data = wgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# XGB\n",
    "cv_xgb.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "best_xgb = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_wgs['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_wgs['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_wgs['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_wgs['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_wgs['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_wgs['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb\n",
    "print(model)\n",
    "rmse_wgs['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### wgs_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_wgs_desc = {}\n",
    "data = shuffle(wgs_desc, random_state=929)\n",
    "target_data = wgs_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_wgs_desc['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_wgs_desc['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_wgs_desc['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_wgs_desc['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_wgs_desc['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_wgs_desc['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb_wgsdesc\n",
    "print(model)\n",
    "rmse_wgs_desc['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.precision = 2\n",
    "\n",
    "print(pd.DataFrame(rmse_wgs_nocond))\n",
    "\n",
    "print(pd.DataFrame(rmse_wgs))\n",
    "\n",
    "print(pd.DataFrame(rmse_wgs_desc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CO oxidation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "co = pd.read_csv('input/co.csv')\n",
    "ind = co['Data No']\n",
    "co.index = ind\n",
    "co= co.drop(['Data No'], axis=1)\n",
    "\n",
    "co_desc = pd.read_csv('input/co_desc.csv')\n",
    "ind = co_desc['Data No']\n",
    "co_desc.index = ind\n",
    "co_desc= co_desc.drop(['Data No'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# devine to each groups\n",
    "baseatom = co.loc[:,:'Pd'].columns\n",
    "supatom = co.loc[:,'Al_s':'Co_s'].columns\n",
    "pro = co.loc[:,'Ce_p':'Cu_p']\n",
    "proatom = [re.findall(r'[A-Z][a-z]?',x) for x in co.loc[:,'Ce_p':'Cu_p']]\n",
    "proatom = [y for x in proatom for y in x ]\n",
    "env = co.drop(list(baseatom)+list(pro),axis=1).iloc[:,:-1].columns\n",
    "\n",
    "conv = co.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc = pd.read_csv('input/Descriptors.csv',skiprows = [0],index_col='symbol').drop(['Unnamed: 0',\n",
    "                                                                               'name',\n",
    "                                                                               'ionic radius',\n",
    "                                                                               'covalent radius',\n",
    "                                                                               'VdW radius',\n",
    "                                                                               'crystal radius',\n",
    "                                                                               'a x 106 ',\n",
    "                                                                               'Heat capacity ',\n",
    "                                                                               'l',\n",
    "                                                                               'electron affinity ',\n",
    "                                                                               'VE',\n",
    "                                                                               'Surface energy '],axis=1)\n",
    "desc=desc.loc[list(baseatom)+list(proatom),:]\n",
    "desc = desc.fillna(desc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CO oxdation\n",
    "ntree = 1500\n",
    "if debug_run:\n",
    "    ntree = 10\n",
    "\n",
    "# Lasso\n",
    "cv_lasso = GridSearchCV(\n",
    "    Lasso(random_state=929), \n",
    "    param_grid={\"alpha\": range_lasso})\n",
    "\n",
    "# Ridge\n",
    "cv_ridge = GridSearchCV(\n",
    "    Ridge(random_state=929), \n",
    "    param_grid={\"alpha\": range_ridge})\n",
    "\n",
    "# KRR\n",
    "cv_krr = GridSearchCV(\n",
    "    KernelRidge(kernel='rbf'), \n",
    "    param_grid={\"alpha\": range_krr_alpha, \n",
    "                \"gamma\": range_krr_gamma})\n",
    "\n",
    "# SVR\n",
    "cv_svr = GridSearchCV(\n",
    "    SVR(kernel='rbf'), \n",
    "    param_grid={\"C\": range_svr_C, \n",
    "                \"gamma\": range_svr_gamma, \n",
    "                \"epsilon\": range_svr_epsilon})\n",
    "\n",
    "# XGB\n",
    "cv_xgb = GridSearchCV(\n",
    "    XGBRegressor(random_state=929), \n",
    "    param_grid={\"max_depth\": range_depth, \n",
    "                \"subsample\": range_subsample, \n",
    "                \"colsample_bytree\": range_colsample,\n",
    "                \"learning_rate\": range_lr})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.6 (C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_xgb.fit(co_desc.iloc[:,:-1], co_desc.iloc[:,-1])\n",
    "\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "\n",
    "print(cvdepth, cvsubsample, cvcolsample, cvlr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)\n",
    "print(estimator)\n",
    "\n",
    "best_xgb_codesc = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_test = shuffle(co_desc, random_state=929)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = Cv_Pred_Expected(estimator=estimator,\\\n",
    "                      X=train_test.iloc[:,:-1],\\\n",
    "                      y=train_test.iloc[:,-1],\\\n",
    "                      cv=10, redc=True)\n",
    "cv.cross_validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_cvbest = cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv.plot_pred_expected(save=True,\\\n",
    "                      title='',\\\n",
    "                      os_title='',\\\n",
    "                      filename='output/co_gb_',\\\n",
    "                      titlesize=11,\n",
    "                      close=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(6, 4), dpi=300)\n",
    "xgb_cvbest.importance.mean(axis=1).sort_values(ascending=False).iloc[:20][::-1].plot(kind='barh',color='b')\n",
    "plt.savefig('output/fig6_co.png', format='png',bbox_inches='tight', dpi=300)\n",
    "plt.savefig('output/fig6_co.pdf', format='pdf',bbox_inches='tight', dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fig.2 (C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "common = []\n",
    "hoge = co.loc[:,'Pt':'Pd']\n",
    "count = 0\n",
    "for key, row in hoge.iterrows():\n",
    "    for i, v in hoge.iloc[count + 1:].iterrows():\n",
    "        temp = (np.array(row) > 1e-5)\n",
    "        temp2 = (np.array(v) > 1e-5)\n",
    "        val = (temp * temp2).sum()\n",
    "        common.append(val)    \n",
    "    count = count + 1\n",
    "\n",
    "freq = dict(Counter(common))\n",
    "plt.figure(figsize=(1.5, 3))\n",
    "plt.bar(freq.keys(), freq.values())\n",
    "plt.xlabel('# Common Elements')\n",
    "plt.ylabel('Frequency (log)')\n",
    "plt.yscale('log')\n",
    "plt.savefig('output/fig2_co.png',format='png', bbox_inches='tight',dpi=300)\n",
    "plt.savefig('output/fig2_co.pdf',format='pdf', bbox_inches='tight',dpi=300)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 4 (co)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### co_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_nocond = pd.concat([co.loc[:,:'Pd'], pro, co.iloc[:,-1]],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_co_nocond = {}\n",
    "data = shuffle(co_nocond, random_state=10)\n",
    "target_data = co_nocond"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# XGB\n",
    "cv_xgb.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "best_xgb = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_co_nocond['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_co_nocond['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10,  'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_co_nocond['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_co_nocond['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_co_nocond['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_co_nocond['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb\n",
    "print(model)\n",
    "rmse_co_nocond['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### co"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_co = {}\n",
    "data = shuffle(co, random_state=10)\n",
    "target_data = co"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# XGB\n",
    "cv_xgb.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvdepth = cv_xgb.best_params_['max_depth']\n",
    "cvsubsample = cv_xgb.best_params_['subsample']\n",
    "cvcolsample = cv_xgb.best_params_['colsample_bytree']\n",
    "cvlr = cv_xgb.best_params_['learning_rate']\n",
    "best_xgb = XGBRegressor(n_estimators=ntree,\\\n",
    "                       max_depth=cvdepth,\\\n",
    "                       subsample=cvsubsample,\\\n",
    "                       colsample_bytree=cvcolsample,\\\n",
    "                       learning_rate=cvlr,\\\n",
    "                       random_state=929, seed=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_co['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_co['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_co['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_co['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_co['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_co['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb\n",
    "print(model)\n",
    "rmse_co['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### co_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse_co_desc = {}\n",
    "data = shuffle(co_desc, random_state=929)\n",
    "target_data = co_desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso\n",
    "cv_lasso.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_lasso.best_params_['alpha']\n",
    "best_lasso = Lasso(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# Ridge\n",
    "cv_ridge.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_ridge.best_params_['alpha']\n",
    "best_ridge = Ridge(random_state=929, alpha=cvalpha)\n",
    "\n",
    "# KRR\n",
    "cv_krr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvalpha = cv_krr.best_params_['alpha']\n",
    "cvgamma = cv_krr.best_params_['gamma']\n",
    "best_krr = KernelRidge(kernel='rbf', alpha=cvalpha, gamma=cvgamma)\n",
    "\n",
    "# SVR\n",
    "cv_svr.fit(target_data.iloc[:,:-1], target_data.iloc[:,-1])\n",
    "cvC = cv_svr.best_params_['C']\n",
    "cvgamma = cv_svr.best_params_['gamma']\n",
    "cvepsilon = cv_svr.best_params_['epsilon']\n",
    "best_svr = SVR(kernel='rbf', epsilon=cvepsilon, C=cvC, gamma=cvgamma)\n",
    "\n",
    "# RFR\n",
    "best_rfr = RandomForestRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)\n",
    "\n",
    "# ETR\n",
    "best_etr = ExtraTreesRegressor(n_estimators=ntree, random_state=929, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best_lasso\n",
    "print(model)\n",
    "rmse_co_desc['Lasso'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Lasso')\n",
    "\n",
    "model = best_ridge\n",
    "print(model)\n",
    "rmse_co_desc['Ridge'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'Ridge')\n",
    "\n",
    "model = best_krr\n",
    "print(model)\n",
    "rmse_co_desc['KRR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'KRR')\n",
    "\n",
    "model = best_svr\n",
    "print(model)\n",
    "rmse_co_desc['SVR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'SVR')\n",
    "\n",
    "model = best_rfr\n",
    "print(model)\n",
    "rmse_co_desc['RFR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'RFR')\n",
    "\n",
    "model = best_etr\n",
    "print(model)\n",
    "rmse_co_desc['ETR'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'ETR')\n",
    "\n",
    "model = best_xgb_codesc\n",
    "print(model)\n",
    "rmse_co_desc['XGB'] = crossvalid(data.iloc[:,:-1], data.iloc[:,-1], model, 10, 'XGB')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.precision = 2\n",
    "\n",
    "print(pd.DataFrame(rmse_co_nocond))\n",
    "\n",
    "print(pd.DataFrame(rmse_co))\n",
    "\n",
    "print(pd.DataFrame(rmse_co_desc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}