random_forest_pca.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.io import arff\n",
    "import os\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import scipy\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score, f1_score\n",
    "import xgboost as xgb\n",
    "from hyperopt import tpe, hp, fmin, STATUS_OK,Trials\n",
    "from hyperopt.pyll.base import scope\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for file at wisdm-dataset/arff_files/phone/accel/data_1600_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1601_accel_phone.arffshape is: (407, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1602_accel_phone.arffshape is: (424, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1603_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1604_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1605_accel_phone.arffshape is: (406, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1606_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1607_accel_phone.arffshape is: (383, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1608_accel_phone.arffshape is: (482, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1609_accel_phone.arffshape is: (306, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1610_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1611_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1612_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1613_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1615_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1616_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1617_accel_phone.arffshape is: (404, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1618_accel_phone.arffshape is: (303, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1619_accel_phone.arffshape is: (407, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1620_accel_phone.arffshape is: (555, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1621_accel_phone.arffshape is: (408, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1622_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1623_accel_phone.arffshape is: (455, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1624_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1625_accel_phone.arffshape is: (574, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1626_accel_phone.arffshape is: (375, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1627_accel_phone.arffshape is: (775, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1628_accel_phone.arffshape is: (541, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1629_accel_phone.arffshape is: (348, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1630_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1631_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1632_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1633_accel_phone.arffshape is: (429, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1634_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1635_accel_phone.arffshape is: (644, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1636_accel_phone.arffshape is: (321, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1637_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1638_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1639_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1640_accel_phone.arffshape is: (324, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1641_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1642_accel_phone.arffshape is: (714, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1643_accel_phone.arffshape is: (759, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1644_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1645_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1646_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1647_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1648_accel_phone.arffshape is: (803, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1649_accel_phone.arffshape is: (409, 93)\n",
      "for file at wisdm-dataset/arff_files/phone/accel/data_1650_accel_phone.arffshape is: (803, 93)\n",
      "(23074, 93)\n",
      "(23074, 91)\n",
      "(23074,)\n"
     ]
    }
   ],
   "source": [
    "phone_accel_df = pd.DataFrame()\n",
    "filenames = os.listdir('wisdm-dataset/arff_files/phone/accel')\n",
    "filenames.pop(0)\n",
    "for file in filenames:\n",
    "    path = \"wisdm-dataset/arff_files/phone/accel/\" + file\n",
    "    data = arff.loadarff(path)\n",
    "    df = pd.DataFrame(data[0])\n",
    "    print(\"for file at \" + path + \"shape is: \" + str(df.shape))\n",
    "    frames = [df, phone_accel_df]\n",
    "    phone_accel_df = pd.concat(frames, sort = False)\n",
    "    \n",
    "print(phone_accel_df.shape)\n",
    "    \n",
    "phone_accel_old_column_names = phone_accel_df.columns\n",
    "\n",
    "phone_accel_new_column_names = []\n",
    "for name in phone_accel_old_column_names:\n",
    "    phone_accel_new_column_names.append(name.replace('\"', ''))\n",
    "    \n",
    "replacement_dict = dict(zip(phone_accel_old_column_names, phone_accel_new_column_names))\n",
    "phone_accel_df = phone_accel_df.rename(columns = replacement_dict)\n",
    "#phone_accel_df.head()\n",
    "\n",
    "labelencoder = LabelEncoder()\n",
    "phone_accel_df['ACTIVITY'] = labelencoder.fit_transform(phone_accel_df['ACTIVITY'])\n",
    "#phone_accel_df.head()\n",
    "\n",
    "phone_accel_df.pop('class')\n",
    "\n",
    "phone_accel_df_shuffle = shuffle(phone_accel_df)\n",
    "\n",
    "phone_accel_data = phone_accel_df_shuffle.to_numpy()\n",
    "\n",
    "X_phone_accel = phone_accel_data[:,1:]\n",
    "Y_phone_accel = phone_accel_data[:,0]\n",
    "print(X_phone_accel.shape)\n",
    "print(Y_phone_accel.shape)\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X_phone_accel, Y_phone_accel, test_size = 0.35)\n",
    "#x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size = 0.40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAAFlCAYAAAAOIeUsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqF0lEQVR4nO3deZhU1Z3/8c+XZm03BFpk625GceHnEpG4JCrEXUgkmZkkaisuGEImLuNojEoSdRJMjE5iZqIyuIz+InFJXOIo4sQFl8FMRM24RFEi0CC4L1FQsenv/HG67KruutXVTVXXct6v56mn+557695Tt8/TfDh97jnm7gIAAABi1KfUFQAAAABKhTAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIVt9SXXjYsGHe2NhYtPOvW7dOm222WdHOj8pF20AS2gZyoX0gCW2j/D3xxBNvuntdtn0lC8ONjY1asmRJ0c6/aNEiTZ48uWjnR+WibSAJbQO50D6QhLZR/sxsZdI+hkkAAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADR6jIMm9m1Zva6mT2bsN/M7F/NbJmZPW1mEwpfTQCofvPnS42NUp8+4ev8+cXbx7W6v+/AAyeV7HPFcH8r+Vod20a5fK5yqUeu95QFd8/5knSApAmSnk3YP0XSPZJM0j6S/qerc7q79txzTy+mBx98sKjnR+WibSBJtrZxww3uDQ3uZuHrDTds2r5c5bW17lL7q7Y2lBd637e+xbUq6VrlUg+uVVnXKpd65LpWb5K0xD0h6ybtyDhIaswRhv9d0tFp20sljejqnIRhlAptI15dB9TWogXUQYPcjznGfeDAzPKBA92//3334cMzy1Ovujr3YcOy72tocB86NHlffX32fX36ZC/feuvwyrZv8ODw6u4+s+T6jRyZfd+oUWF/0rVGjeretXpa9+HDk/fV1CSfL1fde/P+DhmSvC+pjnV1yfsKfX9z7Uu6v9tum/v+jhhRuHuY6z0jRiTva2gI9exuuyn0Pcy1L+l3QK62MXRo7nuf9LsoVxvtTcUOw3dJ2i9t+35JExOOnSlpiaQl9fX1Rf3QBB4koW1Ut+72vObqtUj6xb/11sn7+vTpHHiL9Ur6Rya1L9f+Ur9y1a+rupfz5yqX+m1K2yiH+pf7/a3k+nVV995qG2a9+29DrjBciAfoLNvoi2wHuvs8d5/o7hPr6uoKcGkAMco1Lm3mTGnlyvDrduXKsH3NNdLZZ0vr12eeZ/166d//PXv57NlSc3P267/zTvK+1lbpo4+693nMpG22yb5vxIjwyqa+Xho5MnnfmDHZ99XUZC8fPTq8shkzJvl8ufYlXStX/caMCfsLda2e1r0n97Cnde/N+1tfn1zHXPsKfX97+rly3d/eajc9vb+9eQ/L/f4mnaskklJy+ksMk0AVoW2Ul0KMrR0wwH3GDPettipsr0VS7299ffK+XH9mTPoTaerzlcMYPq5Vftcql3pwrcq6VrnUI5Yxw1OV+QDdH/M5J2EYpULb6H2FGLrQr5/7/vuH4Jtenu8raTxbbwbUrv5RSLpPxdjHtXqyr/OY8t76XHHc30q+VmtZfq5yqUeu9/SWTQrDkm6UtFbSJ5JWS5ohaZakWW37TdLlkv4i6RkljBfu+CIMo1RoG8XR3cCbOr47gTYpuErhukkPjOUKtr0dUMvhHwX0DL87kIS2Uf42uWe4GC/CMEqFttFzPQm8o0dnD6jDhnX/YYxcQxe66sntqv5JvTtACr87kIS2Uf5yhWFWoAPQSbYH1JIeTvvVr6Szzkp+CO2VV7Jf4623uv+QSX29NGeOVFubWV5bG8qbmqR586SGhvBQWkND2G5qCsc1NUkrVoSH3Fas6Fz+wAMPZZQDAKofYRhAhqTQe+qpyYH31Vezn6u5OfcT10nBdubMwgdeAACyIQwDkUqanuzcc7OH3nfeyX6e1aulpJkS6+uliy7qfrC94goCLwCgdxCGgSrWnfl4jz8+hNpVq7p3jfp66ec/L97QBQIvAKCYCMNAhetO4D3pJOmLX5ROPrlz7+/GjaFs8ODs1xk6lKELAIDq07fUFQDQc6nAmwq2qfG9a9ZIF1/cOfBu2CDdfXfy+T78MITY9HNKIfT+4hfh+9TKbKkxv+mBl5ALAKg09AwDFSCp93f27Ozje88+O8zWkE2q5zab+vrcvbz08AIAqg1hGChz2YY7zJghfelLoYc2yYgR2cu7mp5MIvQCAOJhYR7i3jdxiy18yZ57Fu387777rgYnDX5E1Mq5bbz2mrR8ufTRx9LAAdLYse3b2fTtK7W0dC5PvXfpiyHQpvTpI+24gzR8ePZrDR9enM9VKcq5baD0aB9IQtsof/bQQ0+4+8Rs+xgzDJSJ117LDK8ffSy9sDT0BicZt332wJsebJMC7/DhhF8AAEoXhnfcUVq0qGin/9OiRZo8eXLRzo/KVeq2MX9+9ofQdqmT3mztcLCH1dg2bux8noYGacXz0n1ZzndA27CG4W0v5KfUbQPljfaBJLSNCmCWuIueYaAXJc3+IElvvpn9PRs3hvG8HWd3SB/fy5heAAB6hgfogCLINvvDa69Jp52WvKRx0rLFqdkckubwBQAAPUfPMFBgSb2/J50kvf129vc0N0u/+lX2+X1TwygIvwAAFB49w0APZev9bWmRzjwze+/v736Xe7qzrlZxAwAAhUcYBnog29y/M2dKdXVhOEQ2q1dLl1zC/L4AAJQTwjDQhWw9wOedl733V5KGDct+Hnp/AQAoP4wZBnLINv73hBOyL3QhSe+9l3vsr8T4XwAAygk9w0AO2XqAW1qSpyuk9xcAgMpCzzCgzgthnHVWCMHNzdmPd2fuXwAAqgE9w4hetofhTj1V+u53pQEDsr+HuX8BAKgOhGFEI/Ug3IEHTvr0QThJOvvszkMhJGnkSOmaa5Jnf2DmBwAAKh9hGFHI7P01rVwZFsEYP15asyb7e9auZfwvAADVjjHDiMLs2Z17fzdskF58URo8WHr33c7vqa8PXxn/CwBA9aJnGFUl25zA7skPwrW2Sr/8Ze6FMAAAQPUiDKNqZHsQ7qSTQih2z/4epkIDACBuhGFUjaShEGvXSjNmSIMGZe5jGWQAABBXGH7wQWm//UIqGjJEOu446bXXMo9ZsSJ0D2Z7pQ8sXb8+JKwhQ6TttpNuvrnz9X76U2n33ZOXKyumCy6QHnig8Oddty7ct222CffkH/8xlD/2mLT33tJmm4XyP/0p+/vvuEP62c86ly9aFN533315VSPbcIikoRAtLdLVV0tXXZXq/fXcvb8XXBDqUqifW+p8XUndg0WLCnNdAADQpXgeoHvkEenQQ6XDDpNuvVV66y3pe9+TDjpIeuKJzhPKnnuudOSRmWVbbNH+/U9+Iv3+99J110lPPy0de6w0YYI0blzYv3q19KMfSQsXSn1LcJsvvDB0lR54YGHPe/nl0o03StdeK+2wgzRiRChPdb3+53+GLtcddsj+/jvuCIH3n/6px1XItkTy9Om5h0JI7Q/CLVr0kCZPntzj6wMAgOoRTxi+8MLQLXjHHe3hdKedpL32CpPJ/sM/ZB7/N38j7bNP8vnuuUc65ZQQmI88MiS0++5rD8Onny597WvS5z5XlI9TMs8/HybgnT69vay1VVq6tDjhO4uOwyH6aKOs1VVb21fu0ocftu/jQTgAAJBLPMMk/vAH6ZBDMntpP/tZaehQ6fbbu3++DRsyB6HW1koffRS+X7hQeugh6eKLu3/eq64KPcyDBklbby1NmiQtXhz2Jf0Z/brrQvmKFWE79Sf5OXPah3hccEHX177hhjCsY+BAadiwMBxi7dr2/WbhWqtWtZ/3uuukmpoQiH/4w1DW2Jj9/CecIF1/vfTKK+3v73js+vXhPxnDhumjLet0++bHamt799OhEB9/LK1YafqRZuu7+ole1lhtUH/tqmf04YfS7/7pIT068CD9VVtonW2ml7Y7TE27P5txia3/+Efp85+XttpK2nxzaccdpX/+5871Xb5cmjo1HNPQEI5pbc08ZulS6StfCfOzDRoU/gO1cGHX9/qNN6RjjpG23DK8d/r07PO7AQCAooonDNfUSP37dy4fMEB69tnO5eeeG4LzVluFnt9nnsncv/feIditXSvde28YI7vPPiGtnXpqGEYxdGj36njWWeHv/xMmSLfcEsLpAQckD4ZN8thj4esJJ4TvH3tMOvnk3O+ZNy+E3513lm67LdT/3ntDGP/gg/bzHnaYtO227ef9whekRx8N+2fMCGVJ/7n4/velKVOkurr293c89vTTJTPdf/KvNfujH+jwdbfqMp2ulSul449vH5Vxgq7TVN2ts3SppupurdFIHT/sbh3yk4P0+UM31xZ33KDa23+tkZu/L+2/fwjwkvTyy9r1e98LIfzmm6U77wxDNtat61zfr3wl9HTfcYf05S9L558ffuYpa9aEMej/+79hfrZbbgnBdurU8JeDXP72b6W77pIuuijUo2/f0G4AAEDvcveSvPbcc08vpgcffDCz4LOfdd9rr8yyFSvczdz7928vW7PG/ZvfdL/1VveHH3afN8+9sdF9883d//zn9uNWr3bfdVf3MFTV/TvfCeUXXOC+777ura3dq/BLL7n36eN+xhm5PlS4VsfP9h//EcqXL28vk9xnz87v2i0t7tts4z55cmb5I4+E8/ziF+1lTU3uDQ2Zx33ySTju/PO7vtbxx7uPGtW5PPXZpk9393AJyf3f9G3/UANcanUp/Bhc8jU2wgdq/ae3v7bW/a/bbOd+4IGZ533vPfehQ91PPz1s/+Y34Q3vvZdcx/PPD8dce21m+S67uB9ySPv2mWe619SEn11KS4v7Dju477FH5/Ol/Nd/he0bb8w8/+GHZ//5otd0+r0BpKF9IAlto/xJWuIJmTSenuHTT5f++Mfw0Nzrr0svvBB6Qvv0Ca+UESOkuXNDz93++0vf+Ib08MPhT/rpg09HjQo9gsuWSW++GWaOePll6dJLpSuvDANXZ82Shg+Xxo6V/u3fctfvvvvCn+BnzizO55dCbmxpaX9t3BjKly4N96Tj1Ar77ReGBzz0UPHq1NHUqZLaO8Of0a4aqI81XGHWj1QH7vr9D9fwhkGfzgt84z+/pC1e/0v4DOmfsbZW2nff8DOUpM98Rq19+0pHHSX99rfhc3dRl0/tsktmL/3DD4e/Bmy/fXtZTY109NHhLwV//Wv28z72WDju7/4us/yoo3LcGAAAUAzxhOGmphCE/+VfQkAdPz4E2ilT2v/2nmTMmBAMH388s9wsTKuWGg5x6qlhOMLuu4fgvGRJGIJx++3SeedJ99+ffI233gpfR4/u+WfsyvXXS/36tb+22y6Uv/12+JrtPmy7bfv+IkhNkfaFL4Ttq28boiuuaJ8B4mOFWT4GKozHTpVvt9+IjHmBj9ynLdTOmJH5Gfv1C8MRUvd3++319E9/Gt543HHh8+29d/bAP2RI5vaAAe3jwqVwX5Lumbv0zjvZP/TatWE8eL9+meXDh2c/HgAAFE08s0lI4QGvc84JPbjbbBPCx847h6DbFffcc8XefnvoDbzpprC9cGEYs1tXF16HHhrKDjoo+/uHDQtfX3klPNCVzcCB4euGDZnlqaDXlS99KTPQp6aTS4W+V1/t/J5XX5UmTszv/N2UPkVaY1vZTTdLzzwoXXKJ9K1vSUqbNeLTmSGOVeefReo/JD/+sXTwwZ0vljZe/N099pDOOCOM7/7v/5Z+8IPQC7xiRfvPIR9DhiTfM7POYTplxIgQlD/5JDMQd5zzGgAAFF08PcMpm20m7bprCMILF4bhErNm5X5Pc3MITXvvnX3/+vVh8Ymf/zxzLuL0h7I++CB5IlwpBLg+fcKDbEkaGsLXjg/8LVjQ+dj+/TPnGJNCYJw4sf21666hfMcdw/1IBfmUxYvDJL6TJiXXqbsGDPi0XtlWjJNC5p8+PdyKYW0Zd/SoLpZI3nHH0MX83HOZnzH12m237HU58EDp7LPDz2r58u59lkmTwiwlqVk8pDD05OabpT32yGwL6fbdNxx3662Z5R3vPwAAKLp4eoafeio84T9hQth+9NHQ/Xj22ZlzAZ95ZvgT+r77hh7dpUtDb2OfPmGoQzY//GEIY1/7WnvZwQeHGQZ22inMOnD//eHcSbbbLvRW/uxn0vvvhxksamrCOOeddpK+/vXQozhpUqjPsGGhd/uGG6S//KXz+caPl+6+Wzr88PAn+ZEjwyubmpowbdg3vxkWDzn22NBDPXt2mDf5xBNz39suzJ8fTtXcLJ0/eLzOf+dtrbv0Sm2zcqK20EA9q10zjk9N/NDUJDV9IunEtgkrGnNcxCwsCDJtWug5/9rXwj167bUQ6uvrw6wRc+dq59/8JiyKMmZMGO/94x+He7PLLt37YGecEaaWO+SQMI/1lltKV1whvfhiuPdJDjkk/DXim98M1x83LgTobLOaAACA4kp6sq7Yr16fTeLZZ90//3n3rbZyHzgwPO3fcbYAd/drrnGfONF98OAwU8Dw4e5HH+3+wgvZL/T88+5bbOH+4ouZ5e+/737CCe5bb+0+cqT7JZfkV/ErrwyzVPTvH947aZL74sXt+1etcv/iF8PnGD7c/dxz3a+6qvNsEo8+6j5hgvuAAfnP9PCrX7nvtlu49pAh7sceG2bXSNfN2SRuuCHM9PDprA/6wH+to/y9msHuki9Xg0vuk/Sgu+QH6feZp+/uTBmLF7tPnRp+fgMGhLp+/evt93DxYn/jc59zHz06fM5tt3X/+7/P/PmmZn/45JPMcx9/fOfP/sIL7tOmuW+5Zbje3nu733NP5jEdZ5Nwd3/9dfejjgrTY2y1lftxx7nfcQezSZQYT4QjF9oHktA2yp9yzCZhnutP90U0ceJEX7JkSdHOv2jRIpbcLQONjWGkRUfDhkmXXZa5rLIUxgXnHA5RALQNJKFtIBfaB5LQNsqfmT3h7lkfgopvzDB6VdJ6IW+9FQLvvHlhKHRqirRiB2EAAIB0hGEUTGqatD59wvDmCy9snwqto1R5U5MypkgjCAMAgN5EGEZBpKZJW7kyjA5+9dXwTN6UKWHoQ7pPp0gDAAAoMcIwCuKcczpPk9baGmZ9YygEAAAoV4RhdEv6UIjGxrAthZnKsmluZigEAAAoX/HMM4xNlr5inBSGRJx0Uvi+vj77w3JJY4YBAADKAT3DyFu2FeM2bAjlF13E2GAAAFB5CMPIW9I0aamhEIwNBgAAlYZhEsjbsGHSG290Lk+fJo3wCwAAKgk9w+gk/SG5kSOlWbNC+c9/Lg0alHksQyEAAEAlyysMm9nhZrbUzJaZ2TlZ9m9lZv9pZv9rZs+Z2YmFryp6Q8f5gteuDcMdrr8+9PpedRVDIQAAQPXoMgybWY2kyyUdIWm8pKPNbHyHw74t6c/uvrukyZL+xcz6F7iu6AXZHpJzl84/P3zPNGkAAKCa5NMzvJekZe7+srtvkHSTpGkdjnFJW5iZSdpc0tuSWgpaU/SKXA/JAQAAVJt8wvAoSavStle3laX7paSdJa2R9Iyk0929tSA1RFGkjwtuaJDOOy+UJ80LzHzBAACgGuUThi1LmXfYPkzSnySNlPQZSb80sy07nchsppktMbMlb2SblgC9ouO44OZm6cc/Dg/CzZnDfMEAACAe+YTh1ZLGpG2PVugBTneipNs8WCZpuaSdOp7I3ee5+0R3n1hXV9fTOmMTZRsXLLU/DMd8wQAAIBb5zDP8uKRxZjZW0iuSjpJ0TIdjmiUdJOkRMxsuaUdJLxeyoiicpPG/q9oGwzBfMAAAiEWXPcPu3iLpFEn3Snpe0i3u/pyZzTKzthlo9UNJnzOzZyTdL+m77v5msSqNTcO4YAAAgCCvFejcfYGkBR3K5qZ9v0bSoYWtGgph/vwwLKK5OYz9Pe64MP535szMoRKMCwYAADFiBboq1vFBuXXrpGuuCfsYFwwAAJBnzzAq03nndX5Q7pNPQk8xC2YAAADQM1zVVq3KXs4CGgAAAAFhuAq99174yoNyAAAAuRGGq0D6anJDh0ojRkhLlrCABgAAQFcIwxWu40Nyb78tffyx9PjjLKABAADQFcJwhcu2mlxrq3TxxeH7pqbwsFxrKw/NAQAAdEQYrnBJD8PxkBwAAEDXCMMVbMMGHpIDAADYFIThCvXb30o77SSdcQYPyQEAAPQUYbhCpM8YMWSI9NWvSttuKx17LA/JAQAA9BQr0FWA1IwRqQfl3nlHqqmRvvGNMJVaUxPhFwAAoCfoGa4A2WaM2LhRuvDC0tQHAACgWhCGKwAzRgAAABQHYbjMtbZKW22VfR8zRgAAAGwawnAZa22VZs2S3n1X6t8/cx8zRgAAAGw6wnCZcpdOOUW66irpvPOka69lxggAAIBCYzaJMjJ/fnhYrrlZ2nxz6f33pbPPln70oxCCCb8AAACFRRguEx2nT3v/falvX2m33UIQBgAAQOExTKJMZJs+raUllAMAAKA4CMNlgunTAAAAeh9huEyMHp29nOnTAAAAiocwXCYaGzuXMX0aAABAcRGGy8BvfiM98oj05S8zfRoAAEBvYjaJEvvoI+m006S99pJuuUXq16/UNQIAAIgHYbjEBg6U7r03DIkgCAMAAPQuhkn0svnzw/jgPn2kUaPC9m67SdtvX+qaAQAAxIee4V7UcWGNNWukk04K3zM2GAAAoPfRM9yLsi2ssWEDC2sAAACUCmG4F7GwBgAAQHkhDPeipAU0WFgDAACgNAjDvWjOHGnAgMwyFtYAAAAoHcJwL2pqkq65JvQEs7AGAABA6TGbRC9ZsEBavlz61rcIvwAAAOWCnuFe8NZb0owZoRe4paXUtQEAAEAKPcO94NvfDoF44UKpf/9S1wYAAAAp9AwXSfpKczffLE2bJu2+e6lrBQAAgHSE4SJIrTS3cqXkHsoWLAjlAAAAKB+E4SLIttLc+vWsNAcAAFBuCMNFwEpzAAAAlYEwXASjRmUvZ6U5AACA8kIYLoLGxs5lrDQHAABQfgjDBfb730uPPhpmj2hoYKU5AACAcsY8wwW0fr00a5a0ww7STTdJAweWukYAAADIhTBcQIsXS6+8It17L0EYAACgEhCGC+jgg8PcwsOHl7omAAAAyAdjhgugpUW6//7wPUEYAACgchCGN0FqyeV+/UKv8Pnnl7pGAAAA6A7CcA+lL7mccumlLLkMAABQSQjDPcSSywAAAJUvrzBsZoeb2VIzW2Zm5yQcM9nM/mRmz5nZQ4WtZvlhyWUAAIDK12UYNrMaSZdLOkLSeElHm9n4DscMlnSFpCPd/f9J+mrhq1pekpZWZsllAACAypFPz/Bekpa5+8vuvkHSTZKmdTjmGEm3uXuzJLn764WtZvmZMycssZyOJZcBAAAqSz5heJSkVWnbq9vK0u0gaWszW2RmT5jZ9EJVsBy99560bp00dy5LLgMAAFSyfBbdsCxlnuU8e0o6SNIgSY+Z2R/c/cWME5nNlDRTkuoreDzBRRdJl1wiPfmktGJFqWsDAACAnsqnZ3i1pDFp26MlrclyzEJ3X+fub0p6WNLuHU/k7vPcfaK7T6yrq+tpnUtq+XLpssuk6dOlz3ym1LUBAADApsgnDD8uaZyZjTWz/pKOknRnh2N+J2l/M+trZrWS9pb0fGGrWh7OOUeqqWFsMAAAQDXocpiEu7eY2SmS7pVUI+lad3/OzGa17Z/r7s+b2UJJT0tqlXS1uz9bzIqXwuLF0i23hJXmRnUcNQ0AAICKk8+YYbn7AkkLOpTN7bB9iaRLCle18mMmHXqo9J3vlLomAAAAKARWoOvC/PlSY6PUp4909NFhrPBmm5W6VgAAACiEvHqGYzV/vjRzZvuyyytXhm2JKdQAAACqAT3DOcye3R6EU9avD+UAAACofIThHJqbu1cOAACAykIYziFpXZAKXi8EAAAAaQjDOcyZI/Xvn1lWW8scwwAAANWCMJxDU5M0bVpYZMNMamiQ5s3j4TkAAIBqwWwSXbjlFmnjxhCIAQAAUF3oGc7BPXwlCAMAAFQnwnAO3/uetN9+oWcYAAAA1YcwnMNdd0n9+tEzDAAAUK0IwwlWr5aeflqaMqXUNQEAAECxEIYTLFwYvhKGAQAAqhdhOMGCBWFxjfHjS10TAAAAFAtTqyX40pekgw8O8wsDAACgOhGGE5x4YqlrAAAAgGJjmEQWS5ZIr75a6loAAACg2AjDWUyfLh13XKlrAQAAgGIjDHewYoX0/PPS1KmlrgkAAACKjTDcwT33hK9MqQYAAFD9CMMdLFggbbedNG5cqWsCAACAYiMMp/n4Y+mBB0KvMFOqAQAAVD+mVkszYEAYLwwAAIA40DMsaf58qbFR6tNHOuAA6ZFHSl0jAAAA9Iboe4bnz5dmzpTWrw/bK1eGbUlqaipdvQAAAFB80fcMz57dHoRT1q8P5QAAAKhu0Yfh5ubulQMAAKB6RB+G6+u7Vw4AAIDqEX0YnjNHqq3NLKutDeUAAACobtGH4aYmad48qX//sN3QELZ5eA4AAKD6RT+bhBSCL+EXAAAgPtH3DAMAACBehGFJ3/++NG1aqWsBAACA3sYwCUmLFknupa4FAAAAelv0PcOtrdJTT0kTJpS6JgAAAOht0Yfhl16S1q0jDAMAAMQo+jD85JPh6x57lLYeAAAA6H3Rh+HBg6UpU6Tx40tdEwAAAPS26B+gO+KI8AIAAEB8ou4Zdg/jhQEAABCnqMPwypXSlltKv/51qWsCAACAUog6DD/5ZJhabfvtS10TAAAAlEL0YbimRtp111LXBAAAAKUQfRgeP14aNKjUNQEAAEApRB+GWWwDAAAgXtFOrbZxo3TeedLOO5e6JgAAACiVaMNwTY102mmlrgUAAABKKdphEkuXSitWlLoWAAAAKKVow/C550qHHVbqWgAAAKCUog3DPDwHAACAKMPwW2+F1ecIwwAAAHHLKwyb2eFmttTMlpnZOTmO+6yZbTSzvy9cFQvvqafCV8IwAABA3LoMw2ZWI+lySUdIGi/paDMbn3DcxZLuLXQlC+3JJ8PXPfYobT0AAABQWvlMrbaXpGXu/rIkmdlNkqZJ+nOH406VdKukzxa0hkXQ1CTtuKM0ZEipawIAAIBSyicMj5K0Km17taS90w8ws1GSviLpQOUIw2Y2U9JMSaqvr+9uXQtm1KjwAgAAQNzyGTNsWcq8w/Zlkr7r7htzncjd57n7RHefWFdXl2cVC+uDD6TLL5dWrer6WAAAAFS3fHqGV0sak7Y9WtKaDsdMlHSTmUnSMElTzKzF3e8oRCUL6amnpFNOkcaOlcaM6fp4AAAAVK98wvDjksaZ2VhJr0g6StIx6Qe4+9jU92Z2naS7yjEIS+0PzzGTBAAAALocJuHuLZJOUZgl4nlJt7j7c2Y2y8xmFbuChTR/vjR7dvh+n33CNgAAAOKVT8+w3H2BpAUdyuYmHHvCpler8ObPl2bOlNavD9srV4ZtKcwuAQAAgPhEswLd7NntQThl/fr2nmIAAADEJ5ow3NzcvXIAAABUv2jCcNK0xiWc7hgAAAAlFk0YnjNHqq3NLKutDeUAAACIUzRhuKlJmjdPamiQzMLXefN4eA4AACBmec0mUS2amgi/AAAAaBdNzzAAAADQEWEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEK68wbGaHm9lSM1tmZudk2d9kZk+3vRab2e6FryoAAABQWF2GYTOrkXS5pCMkjZd0tJmN73DYckmT3H03ST+UNK/QFQUAAAAKLZ+e4b0kLXP3l919g6SbJE1LP8DdF7v7O22bf5A0urDVBAAAAAovnzA8StKqtO3VbWVJZki6J9sOM5tpZkvMbMkbb7yRfy0BAACAIsgnDFuWMs96oNkXFMLwd7Ptd/d57j7R3SfW1dXlX0sAAACgCPrmccxqSWPStkdLWtPxIDPbTdLVko5w97cKUz0AAACgePLpGX5c0jgzG2tm/SUdJenO9APMrF7SbZKOc/cXC19NAAAAoPC67Bl29xYzO0XSvZJqJF3r7s+Z2ay2/XMl/UDSUElXmJkktbj7xOJVGwAAANh0+QyTkLsvkLSgQ9nctO9PlnRyYasGAAAAFBcr0AEAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEizAMAACAaBGGAQAAEC3CMAAAAKJFGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQLcIwAAAAokUYBgAAQLQIwwAAAIgWYRgAAADRIgwDAAAgWoRhAAAARIswDAAAgGgRhgEAABAtwjAAAACiRRgGAABAtAjDAAAAiBZhGAAAANEiDAMAACBahGEAAABEK68wbGaHm9lSM1tmZudk2W9m9q9t+582swmFryoAAABQWF2GYTOrkXS5pCMkjZd0tJmN73DYEZLGtb1mSrqywPUEAAAACi6fnuG9JC1z95fdfYOkmyRN63DMNEn/34M/SBpsZiMKXFcAAACgoPIJw6MkrUrbXt1W1t1jAAAAgLLSN49jLEuZ9+AYmdlMhWEUkvSBmS3N4/o9NUzSm0U8PyoXbQNJaBvIhfaBJLSN8teQtCOfMLxa0pi07dGS1vTgGLn7PEnz8rjmJjOzJe4+sTeuhcpC20AS2gZyoX0gCW2jsuUzTOJxSePMbKyZ9Zd0lKQ7Oxxzp6TpbbNK7CPpPXdfW+C6AgAAAAXVZc+wu7eY2SmS7pVUI+lad3/OzGa17Z8raYGkKZKWSVov6cTiVRkAAAAojHyGScjdFygE3vSyuWnfu6RvF7Zqm6xXhmOgItE2kIS2gVxoH0hC26hgFnIsAAAAEB+WYwYAAEC0qi4Md7V0NOJiZmPM7EEze97MnjOz09vKh5jZ783spbavW5e6ruh9ZlZjZk+Z2V1t27QLSJLMbLCZ/dbMXmj7/bEv7QOSZGZntP178qyZ3WhmA2kbla2qwnCeS0cjLi2SznT3nSXtI+nbbW3iHEn3u/s4Sfe3bSM+p0t6Pm2bdoGUX0ha6O47SdpdoZ3QPiJnZqMknSZporvvojCxwFGibVS0qgrDym/paETE3de6+5Nt37+v8A/aKIV2cX3bYddL+nJJKoiSMbPRkqZKujqtmHYBmdmWkg6QdI0kufsGd39XtA8EfSUNMrO+kmoV1lWgbVSwagvDLAuNRGbWKGkPSf8jaXhqLuy2r9uUsGoojcsknS2pNa2MdgFJ+htJb0j6j7ZhNFeb2WaifUTP3V+RdKmkZklrFdZV+C/RNipatYXhvJaFRnzMbHNJt0r6R3f/a6nrg9Iysy9Ket3dnyh1XVCW+kqaIOlKd99D0jrxZ29IahsLPE3SWEkjJW1mZseWtlbYVNUWhvNaFhpxMbN+CkF4vrvf1lb8mpmNaNs/QtLrpaofSuLzko40sxUKw6kONLMbRLtAsFrSanf/n7bt3yqEY9oHDpa03N3fcPdPJN0m6XOibVS0agvD+SwdjYiYmSmM+3ve3X+WtutOSce3fX+8pN/1dt1QOu5+rruPdvdGhd8TD7j7saJdQJK7vypplZnt2FZ0kKQ/i/aBMDxiHzOrbfv35SCFZ1FoGxWs6hbdMLMpCmMBU0tHzyltjVBKZrafpEckPaP2saHnKYwbvkVSvcIvt6+6+9slqSRKyswmSzrL3b9oZkNFu4AkM/uMwsOV/SW9LOlEhQ4k2kfkzOxCSV9XmK3oKUknS9pctI2KVXVhGAAAAMhXtQ2TAAAAAPJGGAYAAEC0CMMAAACIFmEYAAAA0SIMAwAAIFqEYQAAAESLMAwAAIBoEYYBAAAQrf8Do/DxWkb2Uy8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_pca = StandardScaler().fit_transform(x_train)\n",
    "\n",
    "pca = PCA().fit(x_pca)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "xi = np.arange(1, 92, step=1)\n",
    "yi = np.cumsum(pca.explained_variance_ratio_)\n",
    "\n",
    "plt.ylim(0.0,1.1)\n",
    "plt.plot(xi, yi, marker='o', linestyle='--', color='b')\n",
    "\n",
    "plt.axhline(y=0.95, color='r', linestyle='-')\n",
    "plt.text(0.5, 0.85, '95% cut-off threshold', color = 'red', fontsize = 16)\n",
    "\n",
    "ax.grid(axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 28)\n",
    "pca.fit(x_train)\n",
    "\n",
    "x_train_pca = pca.transform(x_train)\n",
    "x_test_pca = pca.transform(x_test)\n",
    "#x_val_pca = pca.transform(x_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "space = {\n",
    "    \"n_estimators\": hp.choice(\"n_estimators\", [100, 150, 200, 250, 300, 350, 400, 450, 500]),\n",
    "    \"max_depth\": hp.quniform(\"max_depth\", 1, 15,1),\n",
    "    \"criterion\": hp.choice(\"criterion\", [\"gini\", \"entropy\"]),\n",
    "    \"min_samples_split\": hp.choice(\"min_samples_split\", [2, 3, 4, 5, 6, 7, 8, 9, 10]),\n",
    "    \"max_features\": hp.choice(\"max_features\", [\"auto\",\"sqrt\",\"log2\"]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hyperparameter_tuning(space):\n",
    "    clf = RandomForestClassifier(n_estimators = int(space['n_estimators']), max_depth = space['max_depth'], \n",
    "                                 criterion = space['criterion'], min_samples_split = int(space['min_samples_split']),\n",
    "                                 max_features = space['max_features'],n_jobs=-1)\n",
    "    acc = cross_val_score(clf, x_train_pca, y_train,scoring=\"accuracy\").mean()\n",
    "    return {\"loss\": -acc, \"status\": STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 61%|█████████████████████████                | 61/100 [1:16:55<1:26:49, 133.57s/trial, best loss: -0.8287099477603647]"
     ]
    }
   ],
   "source": [
    "trials = Trials()\n",
    "\n",
    "best = fmin(\n",
    "    fn=hyperparameter_tuning,\n",
    "    space = space, \n",
    "    algo=tpe.suggest, \n",
    "    max_evals=100, \n",
    "    trials=trials\n",
    ")\n",
    "\n",
    "print(\"Best: {}\".format(best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = RandomForestClassifier(criterion = 'entropy', max_depth = 15, max_features = 'auto', min_samples_split = 4, \n",
    "                             n_estimators = 450, n_jobs = -1)\n",
    "\n",
    "clf.fit(x_train, y_train)\n",
    "\n",
    "y_train_pred = clf.predict(x_train)\n",
    "y_test_pred = clf.predict(x_test)\n",
    "\n",
    "train_acc = accuracy_score(y_train, y_train_pred)\n",
    "test_acc = accuracy_score(y_test, y_test_pred)\n",
    "\n",
    "print(\"Train accuracy: \", train_acc)\n",
    "print(\"Test accuracy: \", test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"average = Nonen: \", precision_score(y_test, y_test_pred, average = None))\n",
    "print(\"average = micro: \", precision_score(y_test, y_test_pred, average = 'micro'))\n",
    "print(\"average = macro: \", precision_score(y_test, y_test_pred, average = 'macro'))\n",
    "print(\"average = weighted: \", precision_score(y_test, y_test_pred, average = 'weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"average = Nonen: \", recall_score(y_test, y_test_pred, average = None))\n",
    "print(\"average = micro: \", recall_score(y_test, y_test_pred, average = 'micro'))\n",
    "print(\"average = macro: \", recall_score(y_test, y_test_pred, average = 'macro'))\n",
    "print(\"average = weighted: \", recall_score(y_test, y_test_pred, average = 'weighted'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}