13_ensemble/13_03_kaggle_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle In Class Competition\n",
    "\n",
    "<img src=\"http://cdn.alleywatch.com/wp-content/uploads/2013/06/Competition.jpg\" width=300/>\n",
    "<center>**Competition Time**</center>\n",
    "\n",
    "We're going to use all of the technique's we've learned in order to classify an old Kaggle competition. This competition data provides the scientists with 93 features (which seem to have been cleaned!), and asks users to classify each item/row into class_1 to class_9.\n",
    "\n",
    "At first, you will only have access to the training set. After some time, the test set will be released and we will have a chance to compare our best models!\n",
    "\n",
    "Things to try:\n",
    "* Start small and simple! Try simpler techniques such as logistic regression or SVMs.\n",
    "* Maybe you don't actually wan't ALL of the features. Maybe some are noisy. Would feature selection help?\n",
    "* This training dataset is large, so it may help to test your ideas out on subsets of the training data. Ask questions to others to help guide you!\n",
    "* Think about why you might want a decision tree or other classifier.\n",
    "* Build upon your base ideas with ensemble techniques."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn import ensemble\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train set - Use this data to build your model\n",
    "train = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>feat_10</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Class_6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 94 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   feat_1  feat_2  feat_3  feat_4  feat_5  feat_6  feat_7  feat_8  feat_9  \\\n",
       "0       0       0       0       0       0       0       0       0       0   \n",
       "1       0       0       0       0       0       0       0       0       0   \n",
       "2       0       0       0       0       0       0       0       1       0   \n",
       "3       2       0       0       0       0       0       0       0       1   \n",
       "4       1       0       0       0       0       0       0       1       0   \n",
       "\n",
       "   feat_10   ...     feat_85  feat_86  feat_87  feat_88  feat_89  feat_90  \\\n",
       "0        0   ...           0        0        0        0        0        0   \n",
       "1        0   ...           0        0        2        0        0        1   \n",
       "2        0   ...           0        4        1        4        0        2   \n",
       "3        0   ...           0        0        0        0        0        0   \n",
       "4        0   ...           0        1        0        0        0        0   \n",
       "\n",
       "   feat_91  feat_92  feat_93   target  \n",
       "0        0        0        0  Class_4  \n",
       "1        0        0        1  Class_6  \n",
       "2        0        0        0  Class_3  \n",
       "3        1        1        0  Class_8  \n",
       "4       34        0        0  Class_9  \n",
       "\n",
       "[5 rows x 94 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read your data\n",
    "X_train = train.drop(['target'],axis=1)\n",
    "y_train = train.target\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import ensemble\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "\n",
    "from sklearn import linear_model\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.744505168215\n"
     ]
    }
   ],
   "source": [
    "# Logistic Regression\n",
    "model_lr = linear_model.LogisticRegression()\n",
    "model_lr.fit(X_train,y_train)\n",
    "\n",
    "cv_lr = cross_val_score(model_lr,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_lr.mean() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#SVM - RBF kernel\n",
    "model_svm = svm.SVC(kernel='rbf')\n",
    "model_svm.fit(X_train,y_train)\n",
    "\n",
    "cv_svm = cross_val_score(model_svm,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_svm.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#SVM - Linear kernel\n",
    "model_svm_lin = svm.SVC(kernel='linear')\n",
    "model_svm_lin.fit(X_train,y_train)\n",
    "\n",
    "cv_svm_lin = cross_val_score(model_svm_lin,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_svm_lin.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.737909541913\n"
     ]
    }
   ],
   "source": [
    "# Bagging\n",
    "model_bag = ensemble.BaggingClassifier()\n",
    "\n",
    "cv_bag = cross_val_score(model_bag,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_bag.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.734270308228\n"
     ]
    }
   ],
   "source": [
    "# Random Forest\n",
    "model_rf = ensemble.BaggingClassifier()\n",
    "\n",
    "cv_rf = cross_val_score(model_rf,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_rf.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.771842012566\n"
     ]
    }
   ],
   "source": [
    "# Gradient Boosting\n",
    "model_gb = ensemble.GradientBoostingClassifier()\n",
    "\n",
    "cv_gb = cross_val_score(model_gb,\n",
    "                         X_train,\n",
    "                         y_train,\n",
    "                         cv=StratifiedKFold(y_train,n_folds=10,shuffle=True))\n",
    "print cv_gb.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(init=None, learning_rate=0.1, loss='deviance',\n",
       "              max_depth=3, max_features=None, max_leaf_nodes=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "              random_state=None, subsample=1.0, verbose=0,\n",
       "              warm_start=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gb.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Later on - Will upload test set during class\n",
    "test = pd.read_csv('test.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read your data\n",
    "X_test = test.drop(['target'],axis=1)\n",
    "y_test = test.target\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.77316652632423111"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_gb.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}