xgboost_pca.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.io import arff\n",
    "import os\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import scipy\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score, f1_score\n",
    "import xgboost as xgb\n",
    "from hyperopt import tpe, hp, fmin, STATUS_OK,Trials\n",
    "from hyperopt.pyll.base import scope\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for file at wisdm-dataset/arff_files/phone/accel/data_1600_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1601_accel_phone.arffshape is: (407, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1602_accel_phone.arffshape is: (424, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1603_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1604_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1605_accel_phone.arffshape is: (406, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1606_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1607_accel_phone.arffshape is: (383, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1608_accel_phone.arffshape is: (482, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1609_accel_phone.arffshape is: (306, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1610_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1611_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1612_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1613_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1615_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1616_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1617_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1618_accel_phone.arffshape is: (303, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1619_accel_phone.arffshape is: (407, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1620_accel_phone.arffshape is: (555, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1621_accel_phone.arffshape is: (408, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1622_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1623_accel_phone.arffshape is: (455, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1624_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1625_accel_phone.arffshape is: (574, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1626_accel_phone.arffshape is: (375, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1627_accel_phone.arffshape is: (775, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1628_accel_phone.arffshape is: (541, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1629_accel_phone.arffshape is: (348, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1630_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1631_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1632_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1633_accel_phone.arffshape is: (429, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1634_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1635_accel_phone.arffshape is: (644, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1636_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1637_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1638_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1639_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1640_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1641_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1642_accel_phone.arffshape is: (714, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1643_accel_phone.arffshape is: (759, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1644_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1645_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1646_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1647_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1648_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1649_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1650_accel_phone.arffshape is: (803, 93)\n",
      "(23074, 93)\n",
      "(23074, 91)\n",
      "(23074,)\n"
     ]
    }
   ],
   "source": [
    "phone_accel_df = pd.DataFrame()\n",
    "filenames = os.listdir('wisdm-dataset/arff_files/phone/accel')\n",
    "filenames.pop(0)\n",
    "for file in filenames:\n",
    "    path = \"wisdm-dataset/arff_files/phone/accel/\" + file\n",
    "    data = arff.loadarff(path)\n",
    "    df = pd.DataFrame(data[0])\n",
    "    print(\"for file at \" + path + \"shape is: \" + str(df.shape))\n",
    "    frames = [df, phone_accel_df]\n",
    "    phone_accel_df = pd.concat(frames, sort = False)\n",
    "    \n",
    "print(phone_accel_df.shape)\n",
    "    \n",
    "phone_accel_old_column_names = phone_accel_df.columns\n",
    "\n",
    "phone_accel_new_column_names = []\n",
    "for name in phone_accel_old_column_names:\n",
    "    phone_accel_new_column_names.append(name.replace('\"', ''))\n",
    "    \n",
    "replacement_dict = dict(zip(phone_accel_old_column_names, phone_accel_new_column_names))\n",
    "phone_accel_df = phone_accel_df.rename(columns = replacement_dict)\n",
    "#phone_accel_df.head()\n",
    "\n",
    "labelencoder = LabelEncoder()\n",
    "phone_accel_df['ACTIVITY'] = labelencoder.fit_transform(phone_accel_df['ACTIVITY'])\n",
    "#phone_accel_df.head()\n",
    "\n",
    "phone_accel_df.pop('class')\n",
    "\n",
    "phone_accel_df_shuffle = shuffle(phone_accel_df)\n",
    "\n",
    "phone_accel_data = phone_accel_df_shuffle.to_numpy()\n",
    "\n",
    "X_phone_accel = phone_accel_data[:,1:]\n",
    "Y_phone_accel = phone_accel_data[:,0]\n",
    "print(X_phone_accel.shape)\n",
    "print(Y_phone_accel.shape)\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X_phone_accel, Y_phone_accel, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAAFlCAYAAAAOIeUsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqdElEQVR4nO3deXxV1b338e8PkCFOTAGZkqh1rGhVnKotqKUqVNHbwQEVeWHR3tpHvfZxoo4t1WprByeMQ/UpqbZ1qnW81SsOF61inasoRQiIA9U6giLwe/5YOeYk2fvkJJyTc07W5/16nVey195n73V2luHrytprmbsLAAAAiFGPUlcAAAAAKBXCMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFq9SnXhwYMHe11dXdHO//HHH2v99dcv2vlRuWgbSEPbQC60D6ShbZS/p5566l/uXp20r2RhuK6uTvPmzSva+efMmaNx48YV7fyoXLQNpKFtIBfaB9LQNsqfmS1O28cwCQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFrthmEzu87M3jazF1L2m5n9xswWmNlzZrZT4asJAN1fQ4NUVyf16BG+NjQUbx/X6vi+ffYZW7LPFcP9reRrtW4b5fK5yqUeud5TFtw950vSVyXtJOmFlP0TJN0jySTtLulv7Z3T3bXzzjt7MT344INFPT8qF20DaZLaxuzZ7rW17mbh6+zZ67YvV3lVlbvU/KqqCuWF3ve973GtSrpWudSDa1XWtcqlHrmu1ZUkzXNPybppO1ocJNXlCMNXSTo8a3u+pGHtnZMwjFKhbcSr/YC6tmgBtV8/9yOOcO/bt2V5nz7uZ53lPnRoy/LMq7raffDg5H0jR7oPGpS8r7bWfcSI5H09eyaXDxgQXkn7+vcPr47uM0uvX9pnHjUq7E+7Vk1Nx67V2boPHpy+L+0eDh6cu+5pP8tC39+amvT31NaGe5y0b5NN0utf6Puba1+PHsnlI0bkvr8jRxbuHuZ6z5Ahue9vdXXH2k3//u4DB5b+/tbWpt/fIUNy3/shQzp2f2tru+53v7t7scPwnZL2ytp+QNKYlGOnS5onaV5NTU1RPzSBB2loG91bR3tec/VapP3ir65O39ejh/tGGyXvK/Qr7R+ZzL5c+0v9Wpe6l/PnKpf6ddf7Wwl1L4c6VML9NevafxtyheFCPEBnSaMvkg5093p3H+PuY6qrqwtwaQDdVWfHpU2fLi1eHH7dLl4ctq+7Tjr1VGnFipbXWLFCmjUruXzGDKmxMbluy5en71u7Vvrww459VjNpyJDkfcOGhVeSUaOk4cOT99XUhP1JevZMLh85MrzSrpV2vlz70q5VUxNeaefLta+j1+ps3Tt7rc7UvRj1SHtPrraR6+fSXe9vd72Hhf7vsrP/zea6VtlIS8nZLzFMAt0IbaO8JPXkdnQIQp8+7tOmuW+8cWF7LdJ6f2tq0vfl+jNj2p9IM5+7HMbwca3yu1a51INrVda1yqUesYwZnqiWD9A9kc85CcMoFdpG1+vo0IW0cbDrrx+Cb9K+9l5p5+zKgNrePwpp96kY+7hWZ/a1HVPeVZ8rjvtbyddaW5afq1zqkes9XWWdwrCkGyW9IekzSUslTZN0vKTjm/abpMsl/VPS80oZL9z6RRhGqdA2iqOjgTdzfGeCbdLLLP3BqlzBtqsDajn8o4DO4XcH0tA2yt869wwX40UYRqnQNjqvM4E3bUaDESM6/jBGriEI7fXktlf/tN4dIIPfHUhD2yh/ucJwr2KORwZQmRoamh8gq6mRZs4M5dOnNz9slnk4bdUq6cwz0x9Ce/315GssWxbOvXhx232DBkkrV7Y8Z1VVcj2y902eHLZb1z1TPnly8/fZMuVz5jykcePG5bw3AIDuhTAMoIXMjAytQ2/fvsmB94wzpLfeSj5XY2OYnWDp0rb7MkE1Kdj++tfh+7RQm2tfWuAFACBJIaZWA1CB0qYnO+OM5ND77rvJ53n7bSltpsSaGunCC0PAzZbdk1tfL9XWhunFamvDdibQLloUpipbtKhlwM21DwCAjiAMA91YR+bjnTJF2mQTacmSjl2jpkb65S87F3glgi0AoLQIw0CF60jgnTZNOvhg6dhj2/b+rlkjffCBtPHGydcZNIjACwDofhgzDFSwtPG9778vnX9+28D76afSn/+cfr5PPpGuvrpz43gZqwsAqET0DAMVIK33d8aM5PG93/9++kNtmZ7bJDU1nR/HCwBAJSIMA2UuabjDscdKRx4ZemjTDB2aXJ7p0U0b8iARegEA8bAwD3HXG7Phhj5v552Ldv733ntP/fv3L9r5UbnKuW289Zb02mvSJ59KfftIm27avJ1kvfWkzz5rW5557/xXQqDN6NFD2mrLEJSTrpUWoGNRzm0DpUf7QBraRvmzhx56yt3HJO1jzDBQJt56q2V4/eRTaf58aW2O/1/9wubJgTc72KYF3qFDCb8AAJQuDG+1lTRnTtFO/8ycOawkhUSlbhtJq7sdcYS0/SbS22tbHexSz55hpofWamulRS9J9yec76tNwxqGNr2Qn1K3DZQ32gfS0DYqgFnqLnqGgS6UNvuDFBavSLJmTRjPm7Y0MbM4AADQeTxABxRB0uwPK1dKJ5+cPPvDjBnSsGHJ58rM5pA2hy8AAOg8eoaBAkvr/W1okJYvT35PY6P0u98lz++bmcuX8AsAQOHRMwx0Utrcv6eemtz7+8wz0pAhyedqb35fAABQHIRhoBOS5v797ndDKF62LPk9b74pXXIJ8/sCAFBOCMNAO5J6gJNWflu5MkyPljbVJL2/AACUH8YMAzkkjf+dOjV5oQtJ+vRT6Zpr0sf+Soz/BQCgnNAzDOSQ1AOcFoQlen8BAKg0hGFAbYdCXH+99Kc/hZ7gNIz9BQCg8hGGEb2kh+GmTpW+852w+lsS5v4FAKB7IAwjemec0XYohBSmQfvtb9N7gOn9BQCg8hGGEY3MUIh99hmrujrpgguk44+XlixJPn75cumoo+gBBgCgO2M2CUSh5awQpsWLpTPPlHr1ktZfX/r447bvqakJX5n9AQCA7oueYXQraavCnXZa8lCITTaRrroq98NwAACg+yIMo9tIehBu2jRpjz2k119Pfs/rrzMVGgAAMWOYBLqNpDmBP/1U+tvfpA03lD78sO17GAoBAEDc4uoZfvBBaa+9pH79pIEDw9NRb73V8phFi0L3YNLrvfeaj1uxInQ7Dhwobb659Ic/tL3eRRdJO+wgrV5dzE+V7Nxzpf/5n8Kf9+OPw30bMiTck5NOCuWPPSbttlsYgGsmPfNM8vtvv1265JK25XPmhPfdf39e1UgaDtHYmH78lVd2YCjEueeGuhTq55Y5X3sy92DOnMJcFwAAtCuenuFHHpG+/nVpv/2kW26R3nlH+tGPpH33lZ56SurTp+XxZ5whHXRQy7INN2z+/sILpb/+NazO8Nxz0pFHSjvtJG2xRdi/dKn0k59I994bntLqauedF7pK99mnsOe9/HLpxhul666TttxSGjYslE+bFv4n4y9/CSlzyy2T33/77SHw/td/dboKSUskT5kShkYkyawKJ4Vb0tjoqqmxz6dHAwAA8YonDJ93XhgMevvtzeF0662lXXeVrr1W+s//bHn8ZptJu++efr577pFOOCEE5oMOCgnt/vubw/CJJ4ZVG7785aJ8nJJ56SVp+HDp6KOby9aulebPL074TtB6OEQPrZHWuPr06aUePaSVK5v3tV4VbvJkac6chzRu3Lii1xMAAJS/eIZJPP64NH58y17aXXaRBg2Sbrut4+dbtSr0hGZUVUmffBK+v/de6aGHpJ/9rOPnvfrq0MPcr580YIA0dqw0d27Yl/Zn9OuvD+WLFoXtzJ/kZ85sHuJx7rntX3v27DCso29fafDgMBzijTea95uFay1Z0nze668Py7StXSv9+MehrK4u+fzHHCPdcEN4ai3z/tbHrlgR/idj8GCpujr0uGcPT5G0aLHpJ5qh03ShFmpTrVJvjdbzWrVK+vN/PaRH++6rD7ShPrb19erm+2nyDi+0eP+AJ56Q9txT2nhjaYMNpK22ks4/v219X3tNmjgxHFNbG45Zu7blMfPnS4ccIvXvH35mu+8efv7tWb5cOuIIaaONwnuPPrrN5wQAAMUXTxju2VPq3btteZ8+0gsvtC0/44wQnDfeOPT8Pv98y/277RaC3RtvSPfdF8bI7r57eGLrBz8IwygGDepYHX/4w/D3/512kv74xxBOv/rV3INhkzz2WPh6zDHh+8cek449Nvd76utD+N1mG+nWW0P977svhPGPPmo+7377hfnIMufde2/p0UfD/mnTQlna/1ycdZY0YUIIuZn3tz72xBNDSP797/XkhLP1ye9v0Q0DTlRNTciOxx3X9NF0vSbqLv1QP9dE3aVlGq4pg+/S+Av31Z5f30Ab3j5bVbf9XsM3+FD6yleaV9ZYuFCjf/SjEML/8AfpjjvCkI2kiYYPOST0dN9+u3TwwdI554SfecayZWEM+rPPSpddFn5m/fuHAH3PPbnv93/8h3TnndJPfxrq0atXaDcAAKBruXtJXjvvvLMX04MPPtiyYJdd3HfdtWXZokXuZu69ezeXLVvmftxx7rfc4v7ww+719e51de4bbOD+j380H7d0qfvo0e5hqKr7//2/ofzcc9332MN97dqOVfjVV9179HA/+eRcHypcq/Vn++1vQ/lrrzWXSe4zZuR37dWr3YcMcR83rmX5I4+E8/z6181lkye719a2PO6zz8Jx55zT/rWmTHEfMaJteeazHX20u7vPnu1eVeV+qb7vK9XHpbUuuX/xi+GzLbNh3lcrPr/9VVXuHwzZ3H2ffVqe9/333QcNcj/xxLD9pz+FN7z/fnodzzknHHPddS3Lt9vOffz45u1TTnHv2TP87DJWr3bfckv3HXdse76M//7vsH3jjS3Pv//+yT9fdJk2vzeALLQPpKFtlD9J8zwlk8bTM3ziidITT4SH5t5+W3r55dAT2qNHeGUMGybNmhV67r7yFem735Uefjj0VmZPPTBiROgRXLBA+te/wswRCxdKP/95mLpg5cqw1u/QodKmm0qXXpq7fvffH/4EP316cT6/FHLj6tXNrzVrQvn8+eGetH6abK+9wvCAhx4qXp1amzhRUvO44Oc1Wn31qYYqzPqR6aRe8ZX9NbS23+fzAt94/qva8O1/hs+Q/RmrqsJEww8/HN74pS9pba9e0mGHSTffHD53O3X53Hbbteylf/jh8NeAL3yhuaxnT+nww8NfCj74IPm8jz0WjvvmN1uWH3ZYOzcHAAAUWjxhePLkEIR/8YsQULfdNgTaCROaZ0RIM2pUCIZPPtmy3CxMq5YZDvGDH4ThCDvsEILzvHlhCMZtt4W1fx94IP0a77wTvo4c2fnP2J4bbpDWW6/5tfnmofzdd8PXpPuwySbN+4sgM0Xa3nuH7VseHKi//705c36qMMtHX4Xx2JnyzfcapkWLwv8/LFokHbR7U6idNq3lZ1xvvTAcIXN/v/AFPXfRReGNRx0VPt9uuyUH/oEDW2736dM8LlwK9yXtnrlL//538od+440wHny99VqWDx2afDwAACiaeGaTkMIDXqefHnpwhwwJ4WObbULQbY977rlib7st9AbedFPYvvfeMGa3ujq8vv71ULbvvsnvHzw4fH399fBAV5K+fcPXVatalmeCXnsOPLBloM9MJ5cJfW++2fY9b74pjRmT3/k7KHuKtLqmsitnSY0PhOnQFi9u+56aGkmL1fZnkfkfkgsukL72tbZvzBov/t6OO0onnxzGd//v/0pnnx16gRctav455GPgwPR7ZtY2TGcMGxaC8meftQzEree8BgAARRdPz3DG+utLo0eHIHzvvWG4xPHH535PY2MITbvtlrx/xYqw+MQvf9lyLuLsh7I++ih9IlwpBLgePcKDbGlqa8PX1g/83X1322N79245x5gUAuOYMc2v0aND+VZbhfuRCfIZc+eGRDp2bHqdOqpPn8/rlbRinBTKZs7swCIZUvgMdXXSiy+2/IyZ1/bbJ9dln32kU08NP6vXXuvYZxk7NsxSkpnFQwpDT/7wB2nHHVu2hWx77BGOu+WWluWt7z8AACi6eHqGn346POG/005h+9FHpYsvDkEoey7gU04Jf0LfY4/Qozt/fuht7NEjDHVI8uMfhzD2ne80l33ta2GGga23DrMOPPBAOHeazTcPvZWXXBLWDT7ooDCu9IknwjkOPTT0KI4dG+ozeHDo3Z49W/rnP9ueb9ttpbvukvbfP/xJfvjw8ErSs2eYNuy448JUZkceGXqoZ8wI8yZPnZr73rajoSGz2IV0Tv9tdc6/35VfcaWGLB6jDdVXL2h0i+OXLWsevvz0iZLekUaOkGb+rKn8yISLmIUFQSZNCj3n3/lOuEdvvRVCfU1NmDVi1ixt86c/hUVRRo0K470vuCDcm+2269gHO/nkMLXc+PFhHuuNNpKuuEJ65ZVw79OMHx/+GnHcceH6W2wRAnTSrCYAAKC40p6sK/ary2eTeOEF9z33dN94Y/e+fcPT/q1nC3B3v/Za9zFj3Pv3DzMFDB3qfvjh7i+/nHyhl15y33BD91deaVn+4YfuxxzjPmCA+/Dh7hdfnF/Fr7wyzFLRu3d479ix7nPnNu9fssT9G98In2PoUPczznC/+uq2s0k8+qj7Tju59+mT/0wPv/ud+/bbh2sPHOh+5JFhdo1sHZxNIjMrxOezPugjv9EO849693eX/DXVuuQ+Vg+6S76v/try9B2dKWPuXPeJE8PPr0+fUNdDD22+h3Pn+vIvf9l95MjwOTfZxP1b32r5883M/vDZZy3PPWVK28/+8svukya5b7RRuN5uu7nfc0/LY1rPJuHu/vbb7ocdFmYp2Xhj96OOcr/9dmaTKDGeCEcutA+koW2UP+WYTcI815/ui2jMmDE+b968op1/zpw5rDJWBurqksf+DhgQJtjIXlZZCkMh6uuLu0wybQNpaBvIhfaBNLSN8mdmT7l74kNQ8Y0ZRpdKWy/kvfdC4K2vD0OhM1OkFTsIAwAAZCMMo2Ay06T16BGG415zTdPsDwky5ZMnq8UUaQRhAADQlQjDKIjMNGmLF4fRwUuXSt/7XpjGuUOzQgAAAHQhwjAK4swz206Ttnp1mPWNoRAAAKBcxTO1GooqbWxwY2MIvoRfAABQjugZRodkjwseMUL6zW9CedoUxmljhgEAAMoBYRh5az0ueNmysI5FQ4N00UWMDQYAAJWHYRLIW9K44DVrwupymRWJMyvN1dSEIMzwCAAAUM4Iw8hbrnHBEmODAQBA5WGYBNrIHhdcUyP96lehfOTI5OMZFwwAACpVXmHYzPY3s/lmtsDMTk/Yv7GZ/cXMnjWzF81sauGriq7QelzwkiXN44IvvJBxwQAAoHtpd5iEmfWUdLmk8ZKWSnrSzO5w939kHfZ9Sf9w9wPNrFrSfDNrcPdVRak1imbGjLbjgt0ZFwwAALqnfMYM7yppgbsvlCQzu0nSJEnZYdglbWhmJmkDSe9KWl3guqILMC4YAADEJJ9hEiMkLcnaXtpUlu0ySdtIWibpeUknuvvagtQQRZE9LriuTrrhhlA+alTy8YwLBgAA3VE+YdgSyrzV9n6SnpE0XNKXJF1mZhu1OZHZdDObZ2bzli9f3sGqolBajwtevFiaOlW65hrppz9lXDAAAIhHPmF4qaTs/sKRCj3A2aZKutWDBZJek7R16xO5e727j3H3MdXV1Z2tM9ZR2rjg884LQyDq66XaWsksfK2vZ2gEAADonvIZM/ykpC3MbFNJr0s6TNIRrY5plLSvpEfMbKikrSQtLGRFUThp44Jffz18ZVwwAACIRbs9w+6+WtIJku6T9JKkP7r7i2Z2vJkd33TYjyV92cyel/SApNPc/V/FqjTWDeOCAQAAgrzmGXb3u919S3ff3N1nNpXNcvdZTd8vc/evu/tod9/O3WcXs9LIX/aDckOGSJddFsYF9+vX8jjGBQMAgBixAl031vpBueXLpZNPDvuuvppxwQAAAPmMGUaFSnpQbvXq5gU0CL8AACB29Ax3Y+0toAEAABA7wnA3NnJkcjkPygEAAASE4W4g+yG5ESOkceOkTz6RLriABTQAAAByIQxXuNYPyS1bJj30kPSLX7CABgAAQHsIwxUu6SE5KcwWIYXgu2iRtHYtD80BAAC0RhiucDwkBwAA0HmE4QqX9jAcD8kBAAC0jzBcof75T+mss8LDcDwkBwAA0DmE4QqRPWPEsGHSjjtKV14p7bUXD8kBAAB0FivQVYDMjBGZB+XefDME34svDuG3tpbwCwAA0Bn0DFeApBkj3KVLLy1NfQAAALoLwnAFYMYIAACA4iAMV4Bhw5LLmTECAABg3RCGy9zVV4dV5fr0aVnOjBEAAADrjjBcxq69Njw4N2GCdNVVzBgBAABQaMwmUUYaGsLDco2N0sCB0jvvSPvvL91yi9S3rzRlSqlrCAAA0L0QhstE6+nT3nknzCl86KEhCAMAAKDwGCZRJpKmT1u7Vjr33JJUBwAAIAqE4TLB9GkAAABdjzBcJkaNSi5n+jQAAIDiIQyXiV13bVvG9GkAAADFRRguA088Id1+u7T77qEnmOnTAAAAugazSZTYRx+FwDt8uHTPPVL//qWuEQAAQDwIwyW23nrSgQdKBx9MEAYAAOhqhOEScg/LLF9ySalrAgAAECfGDHexhgapri4sqNG3r/STn5S6RgAAAPEiDHehzCpzixeHXuFVq8JsEQ0Npa4ZAABAnAjDXShplblPPgnlAAAA6HqE4S7EKnMAAADlhTDchdJWk2OVOQAAgNIgDHehmTPDqnLZWGUOAACgdAjDXeTll6V99w2rytXWssocAABAOWCe4S7w6afSt78dAvCzzxJ+AQAAygVhuAucfbb0wgvSXXeFQAwAAIDywDCJIsleXOOii6S995YmTCh1rQAAAJCNMFwErRfXkKTHH2dxDQAAgHJDGC6CpMU1Vq5kcQ0AAIByQxguAhbXAAAAqAyE4SIYOTK5nMU1AAAAygthuAh22KFtGYtrAAAAlB/CcIH9/e/SPfdIY8eyuAYAAEC5Y57hAlq9Wjr2WKm6WrrtNmnAgFLXCAAAALkQhguovl56+mnp5psJwgAAAJWAMFxAU6dKG2wgffObpa4JAAAA8sGY4QJwD/MK9+snHX10qWsDAACAfBGG10H2kssbbSRdemmpawQAAICOYJhEJ2WWXM6sNLdmjXT66dLAgcwaAQAAUCnoGe6kpCWXV6xgyWUAAIBKklcYNrP9zWy+mS0ws9NTjhlnZs+Y2Ytm9lBhq1l+WHIZAACg8rUbhs2sp6TLJR0gaVtJh5vZtq2O6S/pCkkHufsXJX278FUtL2lLK7PkMgAAQOXIp2d4V0kL3H2hu6+SdJOkSa2OOULSre7eKEnu/nZhq1l+Zs4MSyxnY8llAACAypJPGB4haUnW9tKmsmxbShpgZnPM7Ckz69YTjH34oTR+fFhkgyWXAQAAKlc+s0lYQpknnGdnSftK6ifpMTN73N1faXEis+mSpktSTQWPJ5g5U5o1S1qwgPALAABQyfLpGV4qaVTW9khJyxKOudfdP3b3f0l6WNIOrU/k7vXuPsbdx1RXV3e2ziW1cKH0y19KkyZJgweXujYAAABYF/mE4SclbWFmm5pZb0mHSbqj1TF/lvQVM+tlZlWSdpP0UmGrWh5OO03q1Uv66U9LXRMAAACsq3aHSbj7ajM7QdJ9knpKus7dXzSz45v2z3L3l8zsXknPSVor6Rp3f6GYFS+Fhx+Wbr5ZOv98aUTrUdMAAACoOHmtQOfud0u6u1XZrFbbF0u6uHBVKz8PPCCNGiWdckqpawIAAIBCYAW6djQ0SHV1Uo8e0g03SGed1XZKNQAAAFSmvHqGY9XQIE2f3rzs8uLF0kknhTDMLBIAAACVj57hHGbMaA7CGStWhHIAAABUPsJwDo2NHSsHAABAZSEM55C2LkgFrxcCAACALIThHGbOlHr3bllWVRXKAQAAUPkIwzlMniyNHi317CmZSbW1Un09D88BAAB0F8wm0Y777gtLMO+yS6lrAgAAgEIjDLdj0KDwAgAAQPfDMIkcrrpKuuKKUtcCAAAAxUIYzuEXv5D+8pdS1wIAAADFQhhO8eqr4TVxYqlrAgAAgGIhDKe4++7wdcKE0tYDAAAAxUMYTnHXXdLWW0ubbVbqmgAAAKBYCMMJ3MNr0qRS1wQAAADFxNRqCcykv/41BGIAAAB0X/QMJ/jss/DVrLT1AAAAQHERhltxD2OFzz671DUBAABAsRGGW3n++bD8cm1tqWsCAACAYiMMt5KZUu2AA0pbDwAAABQfYbiVu+6SdtxRGj681DUBAABAsRGGs/z739LcuSy0AQAAEAumVmvlZz+T9tuv1LUAAABAV6BnWFJDg1RXJw0aJF12mfTcc6WuEQAAALpC9D3DDQ3S9OnSihVhe/HisC1JkyeXrl4AAAAovuh7hmfMaA7CGStWhHIAAAB0b9GH4cbGjpUDAACg+4g+DNfUdKwcAAAA3Uf0YXjmTKmqqmVZVVUoBwAAQPcWfRiePDnMIJFRWyvV1/PwHAAAQAyin01CkqZMkfbaS+rXTxo5stS1AQAAQFchDEvq0UPaYotS1wIAAABdLfphEpI0e7Z0ww2lrgUAAAC6GmFY0m9+QxgGAACIUfRhePVq6fnnpR13LHVNAAAA0NWiD8Mvvyx98glhGAAAIEbRh+Gnnw5fCcMAAADxiT4ML14cFtnYaqtS1wQAAABdLfow/KMfScuXS72YZA4AACA60Ydhqe1yzAAAAIhD1GG4sVE68EBp3rxS1wQAAAClEHUYnjdPuvNOae3aUtcEAAAApRB1GH76aalnT2n06FLXBAAAAKUQfRjeemupX79S1wQAAAClEH0YZn5hAACAeEUbhleulDbbTNpzz1LXBAAAAKUS7ey6/fpJjzxS6loAAACglKLtGQYAAACiDcNTpkiHHFLqWgAAAKCUog3Djz0mmZW6FgAAACilKMPwhx9Kr77KTBIAAACxyysMm9n+ZjbfzBaY2ek5jtvFzNaY2bcKV8XCe/bZ8JUwDAAAELd2w7CZ9ZR0uaQDJG0r6XAz2zbluJ9Juq/QlSy0p58OXwnDAAAAccunZ3hXSQvcfaG7r5J0k6RJCcf9QNItkt4uYP2KYrPNpKlTpeHDS10TAAAAlFI+YXiEpCVZ20ubyj5nZiMkHSJpVq4Tmdl0M5tnZvOWL1/e0boWzMSJ0nXX8QAdAABA7PIJw0mR0Vtt/0rSae6+JteJ3L3e3ce4+5jq6uo8q1hYq1dLb5d93zUAAAC6Qj5heKmkUVnbIyUta3XMGEk3mdkiSd+SdIWZHVyIChba889LQ4dKt91W6poAAACg1PJZjvlJSVuY2aaSXpd0mKQjsg9w900z35vZ9ZLudPfbC1fNwsk8PPfFL5a2HgAAACi9dnuG3X21pBMUZol4SdIf3f1FMzvezI4vdgULqaFBOumk8P348WEbAAAA8cqnZ1jufreku1uVJT4s5+7HrHu1Cq+hQZo+XVqxImw3NoZtSZo8uXT1AgAAQOlEswLdjBnNQThjxYpQDgAAgDhFE4YbGztWDgAAgO4vmjBcU9OxcgAAAHR/0YThmTOlqqqWZVVVoRwAAABxiiYMT54s1ddLtbVh5bna2rDNw3MAAADxyms2ie5i8mTCLwAAAJpF0zMMAAAAtEYYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0corDJvZ/mY238wWmNnpCfsnm9lzTa+5ZrZD4asKAAAAFFa7YdjMekq6XNIBkraVdLiZbdvqsNckjXX37SX9WFJ9oSsKAAAAFFo+PcO7Slrg7gvdfZWkmyRNyj7A3ee6+7+bNh+XNLKw1QQAAAAKL58wPELSkqztpU1laaZJuidph5lNN7N5ZjZv+fLl+dcSAAAAKIJ8wrAllHnigWZ7K4Th05L2u3u9u49x9zHV1dX51xIAAAAogl55HLNU0qis7ZGSlrU+yMy2l3SNpAPc/Z3CVA8AAAAonnx6hp+UtIWZbWpmvSUdJumO7APMrEbSrZKOcvdXCl9NAAAAoPDa7Rl299VmdoKk+yT1lHSdu79oZsc37Z8l6WxJgyRdYWaStNrdxxSv2gAAAMC6y2eYhNz9bkl3tyqblfX9sZKOLWzVAAAAgOJiBToAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaOUVhs1sfzObb2YLzOz0hP1mZr9p2v+cme1U+KoCAAAAhdVuGDaznpIul3SApG0lHW5m27Y67ABJWzS9pku6ssD1BAAAAAoun57hXSUtcPeF7r5K0k2SJrU6ZpKk/+fB45L6m9mwAtcVAAAAKKh8wvAISUuytpc2lXX0GAAAAKCs9MrjGEso804cIzObrjCMQpI+MrP5eVy/swZL+lcRz4/KRdtAGtoGcqF9IA1to/zVpu3IJwwvlTQqa3ukpGWdOEbuXi+pPo9rrjMzm+fuY7riWqgstA2koW0gF9oH0tA2Kls+wySelLSFmW1qZr0lHSbpjlbH3CHp6KZZJXaX9L67v1HgugIAAAAF1W7PsLuvNrMTJN0nqaek69z9RTM7vmn/LEl3S5ogaYGkFZKmFq/KAAAAQGHkM0xC7n63QuDNLpuV9b1L+n5hq7bOumQ4BioSbQNpaBvIhfaBNLSNCmYhxwIAAADxYTlmAAAARKvbheH2lo5GXMxslJk9aGYvmdmLZnZiU/lAM/urmb3a9HVAqeuKrmdmPc3saTO7s2mbdgFJkpn1N7Obzezlpt8fe9A+IElmdnLTvycvmNmNZtaXtlHZulUYznPpaMRltaRT3H0bSbtL+n5Tmzhd0gPuvoWkB5q2EZ8TJb2UtU27QMavJd3r7ltL2kGhndA+ImdmIyT9H0lj3H07hYkFDhNto6J1qzCs/JaORkTc/Q13/3vT9x8q/IM2QqFd3NB02A2SDi5JBVEyZjZS0kRJ12QV0y4gM9tI0lclXStJ7r7K3d8T7QNBL0n9zKyXpCqFdRVoGxWsu4VhloVGKjOrk7SjpL9JGpqZC7vp65ASVg2l8StJp0pam1VGu4AkbSZpuaTfNg2jucbM1hftI3ru/rqkn0tqlPSGwroK/y3aRkXrbmE4r2WhER8z20DSLZJOcvcPSl0flJaZfUPS2+7+VKnrgrLUS9JOkq509x0lfSz+7A1JTWOBJ0naVNJwSeub2ZGlrRXWVXcLw3ktC424mNl6CkG4wd1vbSp+y8yGNe0fJuntUtUPJbGnpIPMbJHCcKp9zGy2aBcIlkpa6u5/a9q+WSEc0z7wNUmvuftyd/9M0q2SvizaRkXrbmE4n6WjEREzM4Vxfy+5+yVZu+6QNKXp+ymS/tzVdUPpuPsZ7j7S3esUfk/8j7sfKdoFJLn7m5KWmNlWTUX7SvqHaB8IwyN2N7Oqpn9f9lV4FoW2UcG63aIbZjZBYSxgZunomaWtEUrJzPaS9Iik59U8NvRMhXHDf5RUo/DL7dvu/m5JKomSMrNxkn7o7t8ws0GiXUCSmX1J4eHK3pIWSpqq0IFE+4icmZ0n6VCF2YqelnSspA1E26hY3S4MAwAAAPnqbsMkAAAAgLwRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABE6/8DHafyh5bBh9IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_pca = StandardScaler().fit_transform(x_train)\n",
    "\n",
    "pca = PCA().fit(x_pca)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "xi = np.arange(1, 92, step=1)\n",
    "yi = np.cumsum(pca.explained_variance_ratio_)\n",
    "\n",
    "plt.ylim(0.0,1.1)\n",
    "plt.plot(xi, yi, marker='o', linestyle='--', color='b')\n",
    "\n",
    "plt.axhline(y=0.95, color='r', linestyle='-')\n",
    "plt.text(0.5, 0.85, '95% cut-off threshold', color = 'red', fontsize = 16)\n",
    "\n",
    "ax.grid(axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 28)\n",
    "pca.fit(x_train)\n",
    "\n",
    "x_train_pca = pca.transform(x_train)\n",
    "x_test_pca = pca.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "space={ 'eta': hp.uniform(\"eta\", 0.001, 0.5),\n",
    "        'max_depth': hp.quniform(\"max_depth\", 3, 30, 1),\n",
    "        'gamma': hp.uniform ('gamma', 1,10),\n",
    "        'reg_alpha' : hp.uniform('reg_alpha', 0,20),\n",
    "        'reg_lambda' : hp.uniform('reg_lambda', 0,1),\n",
    "        'colsample_bytree' : hp.uniform('colsample_bytree', 0.01,1),\n",
    "        'min_child_weight' : hp.quniform('min_child_weight', 0, 10, 1),\n",
    "        'n_estimators': hp.quniform('n_estimators', 100, 400, 50),\n",
    "        'max_delta_step': hp.quniform('max_delta_step', 0, 10, 1),\n",
    "        'subsample': hp.uniform('subsample', 0.001, 1),\n",
    "        'colsample_bylevel': hp.uniform('colsample_bylevel', 0.01, 1),\n",
    "        'scale_pos_weight': hp.uniform('scale_pos_weaight', 0, 1),\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(space):\n",
    "    clf=xgb.XGBClassifier( eta = space['eta'],\n",
    "                    n_estimators =int(space['n_estimators']), max_depth = int(space['max_depth']), gamma = space['gamma'],\n",
    "                    reg_alpha = space['reg_alpha'], reg_lambda = space['reg_lambda'],\n",
    "                    min_child_weight=int(space['min_child_weight']), colsample_bytree=int(space['colsample_bytree']), \n",
    "                    max_delta_step = space['max_delta_step'], subsample = space['subsample'],\n",
    "                    colsample_bylevel = space['colsample_bylevel'], scale_pos_weight = space['scale_pos_weight'],\n",
    "                    objective = 'multi:softmax')\n",
    "    \n",
    "    evaluation = [( x_train_pca, y_train), ( x_test_pca, y_test)]\n",
    "    \n",
    "    clf.fit(x_train_pca, y_train_pca,\n",
    "            eval_set=evaluation, eval_metric=\"auc\",\n",
    "            early_stopping_rounds=10,verbose=False)\n",
    "    \n",
    "\n",
    "    pred = clf.predict(x_test_pca)\n",
    "    accuracy = accuracy_score(y_test_pca, pred)\n",
    "    print (\"SCORE:\", accuracy)\n",
    "    return {'loss': -accuracy, 'status': STATUS_OK }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                          | 0/100 [00:00<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ASUS\\anaconda3\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13:03:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5200433369447454                                                                                                     \n",
      "[13:04:21] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5512459371614301                                                                                                     \n",
      "[13:04:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5154929577464789                                                                                                     \n",
      "[13:05:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.628385698808234                                                                                                      \n",
      "[13:07:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5029252437703142                                                                                                     \n",
      "[13:07:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5971830985915493                                                                                                     \n",
      "[13:08:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5893824485373781                                                                                                     \n",
      "[13:09:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.638353196099675                                                                                                      \n",
      "[13:11:17] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.4413867822318527                                                                                                     \n",
      "[13:11:43] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.4515709642470206                                                                                                     \n",
      "[13:12:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5882990249187432                                                                                                     \n",
      "[13:13:41] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.40303358613217766                                                                                                    \n",
      "[13:20:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5315276273022752                                                                                                     \n",
      "[13:21:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5150595882990249                                                                                                     \n",
      "[13:22:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.49577464788732395                                                                                                    \n",
      "[13:22:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6539544962080174                                                                                                     \n",
      "[13:23:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.49014084507042255                                                                                                    \n",
      "[13:24:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.4403033586132178                                                                                                     \n",
      "[13:24:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.3733477789815818                                                                                                     \n",
      "[13:24:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.47930660888407367                                                                                                    \n",
      "[13:25:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6433369447453955                                                                                                     \n",
      "[13:26:58] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6431202600216684                                                                                                     \n",
      "[13:28:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6439869989165764                                                                                                     \n",
      "[13:29:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.6294691224268689                                                                                                     \n",
      "[13:30:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.29967497291440953                                                                                                    \n",
      "[13:30:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6697724810400867                                                                                                     \n",
      "[13:32:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5347778981581799                                                                                                     \n",
      "[13:32:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6643553629469122                                                                                                     \n",
      "[13:33:39] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6299024918743229                                                                                                     \n",
      "[13:34:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6013001083423619                                                                                                     \n",
      "[13:35:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6491874322860238                                                                                                     \n",
      "[13:36:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6736728060671723                                                                                                     \n",
      "[13:37:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6637053087757313                                                                                                     \n",
      "[13:39:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6316359696641387                                                                                                     \n",
      "[13:40:18] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5449620801733478                                                                                                     \n",
      "[13:40:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.5872156013001083                                                                                                     \n",
      "[13:42:07] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6563380281690141                                                                                                     \n",
      "[13:43:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.552762730227519                                                                                                      \n",
      "[13:44:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6559046587215601                                                                                                     \n",
      "[13:45:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.656988082340195                                                                                                      \n",
      "[13:46:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6117009750812568                                                                                                     \n",
      "[13:47:31] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6602383531960997                                                                                                     \n",
      "[13:48:46] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6459371614301191                                                                                                     \n",
      "[13:49:49] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5874322860238353                                                                                                     \n",
      "[13:50:46] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6398699891657638                                                                                                     \n",
      "[13:51:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6242686890574215                                                                                                     \n",
      "[13:52:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.476056338028169                                                                                                      \n",
      "[13:53:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.49100758396533045                                                                                                    \n",
      "[13:54:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6719393282773565                                                                                                     \n",
      "[13:56:03] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6400866738894908                                                                                                     \n",
      "[13:58:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6491874322860238                                                                                                     \n",
      "[13:59:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.581148429035753                                                                                                      \n",
      "[14:01:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6539544962080174                                                                                                     \n",
      "[14:02:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5302275189599134                                                                                                     \n",
      "[14:03:00] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6671722643553629                                                                                                     \n",
      "[14:04:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.58136511375948                                                                                                       \n",
      "[14:05:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.49100758396533045                                                                                                    \n",
      "[14:05:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5887323943661972                                                                                                     \n",
      "[14:06:59] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6043336944745396                                                                                                     \n",
      "[14:07:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.6338028169014085                                                                                                     \n",
      "[14:08:19] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6019501625135428                                                                                                     \n",
      "[14:09:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5891657638136512                                                                                                     \n",
      "[14:10:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6316359696641387                                                                                                     \n",
      "[14:11:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.542361863488624                                                                                                      \n",
      "[14:12:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.657421451787649                                                                                                      \n",
      "[14:13:20] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.66738894907909                                                                                                       \n",
      "[14:14:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6832069339111593                                                                                                     \n",
      "[14:16:25] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6762730227518959                                                                                                     \n",
      "[14:18:11] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.676489707475623                                                                                                      \n",
      "[14:19:37] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6686890574214518                                                                                                     \n",
      "[14:21:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6784398699891657                                                                                                     \n",
      "[14:22:23] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.6754062838569881                                                                                                     \n",
      "[14:23:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6598049837486457                                                                                                     \n",
      "[14:24:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6719393282773565                                                                                                     \n",
      "[14:25:29] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6067172264355363                                                                                                     \n",
      "[14:26:24] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6747562296858072                                                                                                     \n",
      "[14:27:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6114842903575298                                                                                                     \n",
      "[14:28:17] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6801733477789815                                                                                                     \n",
      "[14:29:34] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6853737811484291                                                                                                     \n",
      "[14:30:47] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6816901408450704                                                                                                     \n",
      "[14:32:05] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6903575297941495                                                                                                     \n",
      "[14:33:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6907908992416034                                                                                                     \n",
      "[14:35:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.68602383531961                                                                                                       \n",
      "[14:36:28] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.5061755146262189                                                                                                     \n",
      "[14:36:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5924160346695558                                                                                                     \n",
      "[14:38:26] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5687973997833152                                                                                                     \n",
      "[14:39:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5024918743228602                                                                                                     \n",
      "[14:40:33] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6242686890574215                                                                                                     \n",
      "[14:42:06] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6249187432286024                                                                                                     \n",
      "[14:43:03] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6186348862405201                                                                                                     \n",
      "[14:44:27] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6149512459371614                                                                                                     \n",
      "[14:45:30] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5371614301191766                                                                                                     \n",
      "[14:45:51] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6513542795232936                                                                                                     \n",
      "[14:47:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6535211267605634                                                                                                     \n",
      "[14:48:01] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.618851570964247                                                                                                      \n",
      "[14:49:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SCORE:                                                                                                                 \n",
      "0.5789815817984832                                                                                                     \n",
      "[14:50:10] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.628385698808234                                                                                                      \n",
      "[14:51:08] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5726977248104008                                                                                                     \n",
      "[14:51:40] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.6758396533044421                                                                                                     \n",
      "[14:52:45] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576:                   \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "SCORE:                                                                                                                 \n",
      "0.5534127843986999                                                                                                     \n",
      "100%|███████████████████████████████████████████| 100/100 [1:50:13<00:00, 66.14s/trial, best loss: -0.6907908992416034]\n",
      "{'colsample_bylevel': 0.02666068780802034, 'colsample_bytree': 0.45321531060628234, 'eta': 0.47167763532290863, 'gamma': 1.2185702651821244, 'max_delta_step': 9.0, 'max_depth': 14.0, 'min_child_weight': 10.0, 'n_estimators': 400.0, 'reg_alpha': 4.212588780824599, 'reg_lambda': 0.6884768863944317, 'scale_pos_weaight': 0.6286545416254485, 'subsample': 0.9720277525526299}\n"
     ]
    }
   ],
   "source": [
    "trials = Trials()\n",
    "\n",
    "best_hyperparams = fmin(fn = objective,\n",
    "                        space = space,\n",
    "                        algo = tpe.suggest,\n",
    "                        max_evals = 100,\n",
    "                        trials = trials)\n",
    "\n",
    "print(best_hyperparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bylevel': 0.02666068780802034,\n",
       " 'colsample_bytree': 0.45321531060628234,\n",
       " 'eta': 0.47167763532290863,\n",
       " 'gamma': 1.2185702651821244,\n",
       " 'max_delta_step': 9.0,\n",
       " 'max_depth': 14.0,\n",
       " 'min_child_weight': 10.0,\n",
       " 'n_estimators': 400.0,\n",
       " 'reg_alpha': 4.212588780824599,\n",
       " 'reg_lambda': 0.6884768863944317,\n",
       " 'scale_pos_weaight': 0.6286545416254485,\n",
       " 'subsample': 0.9720277525526299}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_hyperparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15:20:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:576: \n",
      "Parameters: { \"scale_pos_weight\" } might not be used.\n",
      "\n",
      "  This could be a false alarm, with some parameters getting used by language bindings but\n",
      "  then being mistakenly passed down to XGBoost core, or some parameter actually being used\n",
      "  but getting flagged wrongly here. Please open an issue if you find any such cases.\n",
      "\n",
      "\n",
      "[15:20:17] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Train accuracy:  0.978330353756975\n",
      "Test accuracy:  0.7982665222101842\n"
     ]
    }
   ],
   "source": [
    "clf = xgb.XGBClassifier(eta = 0.47, colsample_bytree = 0.45, gamma = 1.22, max_depth = 14, min_child_weight = 10, \n",
    "                        n_estimators = 400, reg_alpha = 4.21, reg_lambda = 0.688, \n",
    "                        colsample_bylevel = 0.0266, max_delta_step = 9, scale_pos_weight = 0.628, subsample = 0.97, \n",
    "                        objective = 'multi:softmax')\n",
    "\n",
    "clf.fit(x_train_pca, y_train)\n",
    "\n",
    "y_train_pred = clf.predict(x_train_pca)\n",
    "y_test_pred = clf.predict(x_test_pca)\n",
    "\n",
    "train_acc = accuracy_score(y_train, y_train_pred)\n",
    "test_acc = accuracy_score(y_test, y_test_pred)\n",
    "\n",
    "print(\"Train accuracy: \", train_acc)\n",
    "print(\"Test accuracy: \", test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average = Nonen:  [0.91449814 0.89591078 0.87445887 0.77083333 0.80851064 0.75847458\n",
      " 0.82758621 0.78082192 0.71186441 0.77130045 0.73469388 0.75390625\n",
      " 0.79692308 0.77011494 0.82269504 0.80970149 0.83898305 0.71014493]\n",
      "average = micro:  0.7982665222101842\n",
      "average = macro:  0.7973012209774503\n",
      "average = weighted:  0.7977374520705721\n"
     ]
    }
   ],
   "source": [
    "print(\"average = Nonen: \", precision_score(y_test, y_test_pred, average = None))\n",
    "print(\"average = micro: \", precision_score(y_test, y_test_pred, average = 'micro'))\n",
    "print(\"average = macro: \", precision_score(y_test, y_test_pred, average = 'macro'))\n",
    "print(\"average = weighted: \", precision_score(y_test, y_test_pred, average = 'weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average = Nonen:  [0.9010989  0.95256917 0.87826087 0.77083333 0.85393258 0.76824034\n",
      " 0.81509434 0.72765957 0.67741935 0.70781893 0.66666667 0.76284585\n",
      " 0.83548387 0.73897059 0.83154122 0.85770751 0.82157676 0.784     ]\n",
      "average = micro:  0.7982665222101842\n",
      "average = macro:  0.7973177704568214\n",
      "average = weighted:  0.7982665222101842\n"
     ]
    }
   ],
   "source": [
    "print(\"average = Nonen: \", recall_score(y_test, y_test_pred, average = None))\n",
    "print(\"average = micro: \", recall_score(y_test, y_test_pred, average = 'micro'))\n",
    "print(\"average = macro: \", recall_score(y_test, y_test_pred, average = 'macro'))\n",
    "print(\"average = weighted: \", recall_score(y_test, y_test_pred, average = 'weighted'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}