model_iteration_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    First we load all the Kaggle data for the titanic set and look at the general characteristics of the data and see the flaws and errors\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   PassengerId  Survived  Pclass  \\\n",
      "0            1         0       3   \n",
      "1            2         1       1   \n",
      "2            3         1       3   \n",
      "3            4         1       1   \n",
      "4            5         0       3   \n",
      "\n",
      "                                                Name     Sex  Age  SibSp  \\\n",
      "0                            Braund, Mr. Owen Harris    male   22      1   \n",
      "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female   38      1   \n",
      "2                             Heikkinen, Miss. Laina  female   26      0   \n",
      "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female   35      1   \n",
      "4                           Allen, Mr. William Henry    male   35      0   \n",
      "\n",
      "   Parch            Ticket     Fare Cabin Embarked  \n",
      "0      0         A/5 21171   7.2500   NaN        S  \n",
      "1      0          PC 17599  71.2833   C85        C  \n",
      "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
      "3      0            113803  53.1000  C123        S  \n",
      "4      0            373450   8.0500   NaN        S  \n",
      "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
      "count   891.000000  891.000000  891.000000  714.000000  891.000000   \n",
      "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
      "std     257.353842    0.486592    0.836071   14.526497    1.102743   \n",
      "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
      "25%     223.500000    0.000000    2.000000   20.125000    0.000000   \n",
      "50%     446.000000    0.000000    3.000000   28.000000    0.000000   \n",
      "75%     668.500000    1.000000    3.000000   38.000000    1.000000   \n",
      "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
      "\n",
      "            Parch        Fare  \n",
      "count  891.000000  891.000000  \n",
      "mean     0.381594   32.204208  \n",
      "std      0.806057   49.693429  \n",
      "min      0.000000    0.000000  \n",
      "25%      0.000000    7.910400  \n",
      "50%      0.000000   14.454200  \n",
      "75%      0.000000   31.000000  \n",
      "max      6.000000  512.329200  \n"
     ]
    }
   ],
   "source": [
    "import csv \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import thinkstats2\n",
    "import thinkplot\n",
    "%matplotlib inline\n",
    "titanic = pd.read_csv(\"train.csv\")\n",
    "\n",
    "# Print the first 5 rows of the dataframe.\n",
    "print(titanic.head(5))\n",
    "print(titanic.describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    Now that we see the categories(Age,Sex,Embarked) we have to change NaN values and recode characters to numbers for our model we will do so.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['male' 'female']\n",
      "['S' 'C' 'Q' nan]\n"
     ]
    }
   ],
   "source": [
    "titanic[\"Age\"] = titanic[\"Age\"].fillna(titanic[\"Age\"].median())\n",
    "\n",
    "print(titanic[\"Sex\"].unique())\n",
    "print(titanic[\"Embarked\"].unique())\n",
    "\n",
    "# Clean Up Data\n",
    "titanic[\"Age\"] = titanic[\"Age\"].fillna(titanic[\"Age\"].median())\n",
    "\n",
    "titanic.loc[titanic[\"Sex\"] == \"male\", \"Sex\"] = 0\n",
    "titanic.loc[titanic[\"Sex\"] == \"female\", \"Sex\"] = 1\n",
    "\n",
    "titanic[\"Embarked\"] = titanic[\"Embarked\"].fillna(\"S\")\n",
    "titanic.loc[titanic[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n",
    "titanic.loc[titanic[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n",
    "titanic.loc[titanic[\"Embarked\"] == \"Q\", \"Embarked\"] = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    After cleaning up our training data we can use scikit-learn to create a linear regression model and make predictions cross comparing our training model to evaluate the accuracy.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import the linear regression class\n",
    "from sklearn.linear_model import LinearRegression\n",
    "# Sklearn also has a helper that makes it easy to do cross validation\n",
    "from sklearn.cross_validation import KFold\n",
    "\n",
    "# The columns we'll use to predict the target\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\"]\n",
    "\n",
    "# Initialize our algorithm class\n",
    "alg = LinearRegression()\n",
    "# Generate cross validation folds for the titanic dataset.  It return the row indices corresponding to train and test.\n",
    "# We set random_state to ensure we get the same splits every time we run this.\n",
    "kf = KFold(titanic.shape[0], n_folds=3, random_state=1)\n",
    "\n",
    "predictions = []\n",
    "for train, test in kf:\n",
    "    # The predictors we're using the train the algorithm.  Note how we only take the rows in the train folds.\n",
    "    train_predictors = (titanic[predictors].iloc[train,:])\n",
    "    # The target we're using to train the algorithm.\n",
    "    train_target = titanic[\"Survived\"].iloc[train]\n",
    "    # Training the algorithm using the predictors and target.\n",
    "    alg.fit(train_predictors, train_target)\n",
    "    # We can now make predictions on the test fold\n",
    "    test_predictions = alg.predict(titanic[predictors].iloc[test,:])\n",
    "    predictions.append(test_predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    Now we can assess the accuracy of our predictions \n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.783389450056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wilson/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:11: FutureWarning: in the future, boolean array-likes will be handled as a boolean array index\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# The predictions are in three separate numpy arrays.  Concatenate them into one.  \n",
    "# We concatenate them on axis 0, as they only have one axis.\n",
    "predictions = np.concatenate(predictions, axis=0)\n",
    "\n",
    "def calculate_accuracy(predictions_data):\n",
    "    # Map predictions to outcomes (only possible outcomes are 1 and 0)\n",
    "    predictions_data[predictions_data > .5] = 1\n",
    "    predictions_data[predictions_data <=.5] = 0\n",
    "    accuracy = sum(predictions[predictions == titanic[\"Survived\"]]) / len(predictions)\n",
    "    return accuracy\n",
    "\n",
    "print calculate_accuracy(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    The accuracy of our model is 78.33% .To improve the linear regression model we can vary the output values using logistic regression.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.787878787879\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# Initialize our algorithm\n",
    "alg = LogisticRegression(random_state=1)\n",
    "# Compute the accuracy score for all the cross validation folds.  (much simpler than what we did before!)\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv=3)# Take the mean of the scores (because we have one for each fold)\n",
    "\n",
    "print(scores.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "Using LogisticRegression improved the model by a 0.45%. Now we can apply the algorithm to the test_data and generate a submission file to get a score!\n",
    "\n",
    "First, we need to clean up the test data like we did to the training data then apply the algorithm and store it in a csv\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "titanic_test = pd.read_csv(\"test.csv\")\n",
    "titanic_test[\"Age\"] = titanic_test[\"Age\"].fillna(titanic[\"Age\"].median())\n",
    "titanic_test[\"Fare\"] = titanic_test[\"Fare\"].fillna(titanic_test[\"Fare\"].median())\n",
    "titanic_test.loc[titanic_test[\"Sex\"] == \"male\", \"Sex\"] = 0 \n",
    "titanic_test.loc[titanic_test[\"Sex\"] == \"female\", \"Sex\"] = 1\n",
    "titanic_test[\"Embarked\"] = titanic_test[\"Embarked\"].fillna(\"S\")\n",
    "\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n",
    "titanic_test.loc[titanic_test[\"Embarked\"] == \"Q\", \"Embarked\"] = 2\n",
    "\n",
    "# Initialize the algorithm class\n",
    "alg = LogisticRegression(random_state=1)\n",
    "\n",
    "# Train the algorithm using all the training data\n",
    "alg.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "# Make predictions using the test set.\n",
    "predictions = alg.predict(titanic_test[predictors])\n",
    "\n",
    "# Create a new dataframe with only the columns Kaggle wants from the dataset.\n",
    "submission = pd.DataFrame({\n",
    "        \"PassengerId\": titanic_test[\"PassengerId\"],\n",
    "        \"Survived\": predictions\n",
    "    })\n",
    "\n",
    "submission.to_csv(\"kaggle_v1.csv\", index = False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    Submission Score: 75.120 For next time lets try improving logistic regression by generating better features to assess survivability. We will add two features: we have observed that richer people are more likely to survive so we can add: Titles in their names b/c unique titles will be richer. Next let's try adding the second category based on the cabin data (only first class cabins were known and first class had a higher chance to survive)\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mr          517\n",
      "Miss        182\n",
      "Mrs         125\n",
      "Master       40\n",
      "Dr            7\n",
      "Rev           6\n",
      "Col           2\n",
      "Major         2\n",
      "Mlle          2\n",
      "Countess      1\n",
      "Ms            1\n",
      "Lady          1\n",
      "Jonkheer      1\n",
      "Don           1\n",
      "Mme           1\n",
      "Capt          1\n",
      "Sir           1\n",
      "Name: Name, dtype: int64\n",
      "0.804713804714\n"
     ]
    }
   ],
   "source": [
    "#use data.io tutorial title code\n",
    "import re\n",
    "# A function to get the title from a name.\n",
    "def get_title(name):\n",
    "    # Use a regular expression to search for a title.  Titles always consist of capital and lowercase letters, and end with a period.\n",
    "    title_search = re.search(' ([A-Za-z]+)\\.', name)\n",
    "    # If the title exists, extract and return it.\n",
    "    if title_search:\n",
    "        return title_search.group(1)\n",
    "    return \"\"\n",
    "titles = titanic[\"Name\"].apply(get_title)\n",
    "print(pd.value_counts(titles))\n",
    "# Map each title to an integer.  Some titles are very rare, and are compressed into the same codes as other titles.\n",
    "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2}\n",
    "for k,v in title_mapping.items():\n",
    "    titles[titles == k] = v\n",
    "# Add in the title column.\n",
    "titanic[\"Title\"] = titles\n",
    "\n",
    "\n",
    "#after adding the titles we train and reassess the model.\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\", \"Title\"]\n",
    "# Initialize the algorithm class\n",
    "alg = LogisticRegression(random_state=1)\n",
    "\n",
    "# Compute the accuracy score for all the cross validation folds.  (much simpler than what we did before!)\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv=3)# Take the mean of the scores (because we have one for each fold)\n",
    "\n",
    "print(scores.mean())\n",
    "\n",
    "#we end it with testing out prediction score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "    To can generate a submission file for this version\n",
    "</p>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mr        240\n",
      "Miss       78\n",
      "Mrs        72\n",
      "Master     21\n",
      "Col         2\n",
      "Rev         2\n",
      "Dona        1\n",
      "Ms          1\n",
      "Dr          1\n",
      "Name: Name, dtype: int64\n",
      "[1 3 2 4 7 6 5 10]\n",
      "['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Title']\n",
      "Index([u'PassengerId', u'Pclass', u'Name', u'Sex', u'Age', u'SibSp', u'Parch',\n",
      "       u'Ticket', u'Fare', u'Cabin', u'Embarked', u'Title'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Add Title to the test data\n",
    "titles = titanic_test[\"Name\"].apply(get_title)\n",
    "print(pd.value_counts(titles))\n",
    "# Map each title to an integer.  Some titles are very rare, and are compressed into the same codes as other titles.\n",
    "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2, \"Dona\": 10}\n",
    "for k,v in title_mapping.items():\n",
    "    titles[titles == k] = v\n",
    "# Add in the title column.\n",
    "titanic_test[\"Title\"] = titles\n",
    "print(titles.unique())\n",
    "print(predictors)\n",
    "print(titanic_test.columns)\n",
    "# Initialize the algorithm class\n",
    "alg = LogisticRegression(random_state=1)\n",
    "\n",
    "# Train the algorithm using all the training data\n",
    "alg.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "# Make predictions using the test set.\n",
    "predictions = alg.predict(titanic_test[predictors])\n",
    "\n",
    "# Create a new dataframe with only the columns Kaggle wants from the dataset.\n",
    "submission = pd.DataFrame({\n",
    "        \"PassengerId\": titanic_test[\"PassengerId\"],\n",
    "        \"Survived\": predictions\n",
    "    })\n",
    "\n",
    "submission.to_csv(\"kaggle_v2.csv\", index = False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> \n",
    "Next we try a RandomForestClassifier using the predictors we created for the linear model\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.79797979798\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEpCAYAAABYyHNYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGShJREFUeJzt3X2UZHV95/H3B8YoKJDxgWkjKD5ERLNR2QR13RNaSY5o\nViGi+HRyENcke84aSNhkgd1VRhI9wtG4imYTV2PGxBghrhFdlQliEzVBVGAlyoOPqLtOE3kSMCrI\nd/+4t6Fn6J6u6a6aW7+e9+ucOnPvraq+31NT9alf/e7v/m6qCklSG/YaugBJ0ugMbUlqiKEtSQ0x\ntCWpIYa2JDXE0JakhqwY2kkem+TyJJf1/96S5KQkG5NsTXJNkguSHLA7CpakPVl2ZZx2kr2A7wBP\nAV4F3FBVZyc5FdhYVadNpkxJEux698gvA1+rqm8DxwBb+u1bgGPHWZgk6d52NbRfBPxVv7ypquYB\nqmobcOA4C5Mk3dvIoZ3kPsDzgPP6TTv2q3g+vCRN2IZdeOyzgS9U1ff69fkkm6pqPskMcP1ST0pi\nmEvSKlRVdty2K90jLwHet2j9fODl/fIJwId2suOpv51xxhmD12Cd1mid1rlwW85IoZ1kX7qDkP9r\n0eazgF9Jcg1wFPCGUf6WJGn1RuoeqaofAA/ZYduNdEEuSdpNPCOy99a3/glJBr3NzByyYp2zs7MT\nfy3GoYU6W6gRrHPcWqlzObt0cs2qdpDUpPcxDkkYfgBMdtqXJWnPkYRa44FISdLADG1JaoihLUkN\nMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNqS1BBD\nW5IaYmhLUkMMbUlqiKEtSQ0xtCWpIYa2JDVkpNBOckCS85JcleRLSZ6SZGOSrUmuSXJBkgMmXawk\n7elGbWm/BfhoVR0GPBG4GjgNuLCqDgUuAk6fTImSpAWpqp0/INkfuLyqHr3D9quBI6tqPskMMFdV\nj1vi+bXSPqZBEmDoOkMLr5WkyUtCVWXH7aO0tB8JfC/Ju5NcluQdSfYFNlXVPEBVbQMOHG/JkqQd\njRLaG4DDgbdX1eHA7XRdIzs2CW0iStKEbRjhMd8Bvl1Vn+/XP0AX2vNJNi3qHrl+uT+wefPmu5dn\nZ2eZnZ1ddcGStB7Nzc0xNze34uNW7NMGSHIx8BtVdW2SM4B9+7turKqzkpwKbKyq05Z4rn3ao1dh\nn7YkYPk+7VFD+4nAO4H7AF8HTgT2Bs4FDgauA46vqpuXeK6hPXoVhrYkYI2hvcYdG9qjV2FoSwLW\nNnpEkjQlDG1JaoihLUkNMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpiaEtSQwxtSWqIoS1JDTG0Jakh\nhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqiKEtSQ0xtCWpIYa2JDXE0JakhhjaktSQDaM8KMk3\ngVuAu4A7quqIJBuB9wOPAL4JHF9Vt0yoTkkSo7e07wJmq+rJVXVEv+004MKqOhS4CDh9EgVKku4x\namhnicceA2zpl7cAx46rKEnS0kYN7QL+Lsnnkryy37apquYBqmobcOAkCpQk3WOkPm3g6VX13SQP\nAbYmuYYuyBfbcV2SNGYjhXZVfbf/95+T/C1wBDCfZFNVzSeZAa5f7vmbN2++e3l2dpbZ2dm11CxJ\n687c3Bxzc3MrPi5VO28gJ9kX2Kuqbktyf2Ar8FrgKODGqjoryanAxqo6bYnn10r7mAZJGP7HQmjh\ntZI0eUmoqtxr+wih/Ujgg3SJtgF4b1W9IckDgXOBg4Hr6Ib83bzE8w3t0aswtCUBawjtMezY0B69\nCkNbErB8aHtGpCQ1xNCWpIYY2pLUEENbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNqS\n1BBDW5IaYmhLUkMMbUlqiKEtSQ0xtCWpIYa2JDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkN\nGTm0k+yV5LIk5/frG5NsTXJNkguSHDC5MiVJsGst7ZOBLy9aPw24sKoOBS4CTh9nYZKkexsptJMc\nBDwHeOeizccAW/rlLcCx4y1NkrSjUVvabwZ+H6hF2zZV1TxAVW0DDhxzbZKkHawY2kl+FZivqiuA\n7OShtZP7JEljsGGExzwdeF6S5wD7APsl+QtgW5JNVTWfZAa4frk/sHnz5ruXZ2dnmZ2dXVPRkrTe\nzM3NMTc3t+LjUjV6AznJkcB/qqrnJTkbuKGqzkpyKrCxqk5b4jm1K/sYShKG/7EQWnitJE1eEqrq\nXr0baxmn/QbgV5JcAxzVr0uSJmiXWtqr2oEt7V2pwpa2JGAyLW1J0m5maEtSQwxtSWqIoS1JDTG0\nJakhhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqiKEtSQ0xtCWpIYa2JDXE0JakhhjaktQQQ1uS\nGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1xNCWpIasGNpJ7pvks0kuT3JlkjP67RuTbE1yTZILkhww\n+XIlac+Wqlr5Qcm+VfWDJHsDnwFOAo4Dbqiqs5OcCmysqtOWeG6Nso+hJQGGrjO08FpJmrwkVFV2\n3D5S90hV/aBfvC+wgS7djgG29Nu3AMeOoU5J0k6MFNpJ9kpyObAN+Luq+hywqarmAapqG3Dg5MqU\nJMHoLe27qurJwEHAEUmewL37EvxdL0kTtmFXHlxV308yBxwNzCfZVFXzSWaA65d73ubNm+9enp2d\nZXZ2dlXFStJ6NTc3x9zc3IqPW/FAZJIHA3dU1S1J9gEuAN4AHAncWFVneSBybFV4IFISsPyByFFa\n2g8FtiTZi6475f1V9dEklwDnJnkFcB1w/FgrliTdy0hD/ta0A1vau1KFLW1JwBqH/EmSpoOhLUkN\nMbQlqSGGtiQ1xNCWpIYY2pL2KDMzh5Bk0NvMzCGrrt8hfz2H/El7hlY+6w75k6R1wNCWpIYY2pLU\nEENbkhpiaEtSQwxtSWrILl0EYbW6ITbD2bTpEWzb9s1Ba5Ckcdgt47RbGRPZQp2S1qaVz7rjtCVp\nHTC0JakhhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqyIqhneSgJBcl+VKSK5Oc1G/fmGRrkmuS\nXJDkgMmXK0l7thXPiEwyA8xU1RVJHgB8ATgGOBG4oarOTnIqsLGqTlvi+Z4ROTLPiJQmrZXP+qrP\niKyqbVV1Rb98G3AVcBBdcG/pH7YFOHYXq5Yk7aJd6tNOcgjwJOASYFNVzUMX7MCB4y5OkrS9kWf5\n67tG/gY4uapu67o9trOTtv7mRcuz/U2StGBubo65ubkVHzfSLH9JNgAfAT5WVW/pt10FzFbVfN/v\n/cmqOmyJ59qnPTL7tKVJa+WzvtZZ/v4M+PJCYPfOB17eL58AfGjEvyVJWqVRRo88Hfh74Eq6r6cC\n/gtwKXAucDBwHXB8Vd28xPNtaY/MlrY0aa181pdraXsRhIVHNPIfKWltWvmsexEESVoHDG1Jaoih\nLUkNMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNrS\nlJuZOYQkg99mZg4Z+qUQzqd9zyMamWNXe57peG/Cenl/Tsfr6XzakrRHMLQlqSGGtiQ1xNCWpIYY\n2pLUEENbkhpiaEtSQ1YM7STvSjKf5IuLtm1MsjXJNUkuSHLAZMuUJMFoLe13A8/aYdtpwIVVdShw\nEXD6uAuTJN3biqFdVZ8Gbtph8zHAln55C3DsmOuSJC1htX3aB1bVPEBVbQMOHF9JkqTlbBjT31nh\nRP7Ni5Zn+5skacHc3Bxzc3MrPm6kCaOSPAL4cFX9fL9+FTBbVfNJZoBPVtVhyzzXCaNGtj4m5NF4\nTcd7E9bL+3M6Xs/JTxiV/rbgfODl/fIJwIdG/DuSpDVYsaWd5K/o+jMeBMwDZwB/C5wHHAxcBxxf\nVTcv83xb2iNbHy0Zjdd0vDdhvbw/p+P1XH1L2/m0Fx7RyH+k9jzT8d6E9fL+nI7X0/m0JWmPYGhL\nUkMMbUlqiKEtSQ0xtCWpIYa2JDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1\nxNCWpIYY2pLUEENbkhpiaEtSQwxtSWMxM3MISQa/zcwcMvRLMVFeI3LhEY1cN057nul4b8JK70/r\n3BVeI1KS9giGtvZY/pxXi9YU2kmOTnJ1kmuTnDquoqTdYX7+OrqfycPeujqk0aw6tJPsBbwNeBbw\nBOAlSR43rsK0tAc+cKaJluHc3NzEXwtpT7SWlvYRwFeq6rqqugP4a+CY8ZSl5dx00zwttAwNbWky\n1hLaDwO+vWj9O/02iTe+8b838YtAas2GoQvQ+nT77bcw9LCq+fl7jZaSmreW0P6/wMMXrR/Ub1vC\n8B+ebmzmio+aeB0rVmCdY7VyncPXCNY5bi3UOdpnaInnrfZkjiR7A9cARwHfBS4FXlJVV63qD0qS\nVrTqlnZV/STJq4CtdH3j7zKwJWmyJn4auyRpfDwjUpIaYmhLWveS7JPk0KHrGIeJhHaSRye5b788\nm+SkJD89iX1peiSZSfK8JM9NMjN0PRJAkucCVwAf79eflOT8YatavYn0aSe5AvgF4BDgo8CHgCdU\n1XPGvrNVSvIHwGur6s5+fX/gLVV14rCVbS/JJuD1wM9U1bOTPB54WlW9a+DStpPklcBrgIvoxlMd\nCZxZVX82aGFLSPIw4BEsOhBfVX8/XEX3lm482MuAR1XVmUkeDsxU1aUDlwZAkg+zk4H4VfW83VjO\nTiX5AvBMYK6qntxvu7Kq/tWwla3OpE6uuauq7kzya8A5VXVOkssntK/V2gB8NsmJwCa6eVTOGbak\nJf058G7gv/br1wLvB6YqtIHfB55cVTcAJHkQ8A/AVIV2krOAFwFfBn7Sby5gqkIb+GPgLrqwORO4\nFfgA8ItDFrXIG/t/nw/MAH/Zr78EmB+kouXdUVW37DAuutkRGJMK7TuSvAQ4AXhuv+0+E9rXqlTV\n6UkuBD4L3AT8UlV9deCylvLgqjo3yekA/ZfhT1Z60gBuoAuWBbf226bNscChVfWjoQtZwVOq6vCF\nxk5V3ZTkp4YuakFVXQyQ5E1V9QuL7vpwks8PVNZyvpTkpcDeSX4WOImuQdGkSR2IPBF4GvC6qvpG\nkkcCfzGhfa1Kkl8C3krXipkDzknyM4MWtbTb+1ZrASR5KnDLsCUt6at0v1w2JzkDuAS4NskpSU4Z\nuLbFvs6UNSCWcUd/AtvC//tD6Fre0+b+SR61sNJ/1u8/YD1L+W26mUh/BLwP+D7wO4NWtAa743Jj\nG4GDq+qLE93RLkpyKfDyqvpyv/584PVVNVXTyyY5nK7b5ueAfwIeArxgCl/PM3Z2f1W9dnfVspQk\n59AF4MOAJwKfoPsQA1BVJw1U2pKSvIyuG+dwYAvwAuC/VdV5gxa2gyRHA++g+zIM3bGC36qqCwYt\nbB2b1IHIOeB5dN0vXwCuBz5TVVPT4kqyd1X9ZIdtD1rok50mSTYAh9J9KK7pp8KdWv0X9c01RWdu\nJTlhZ/dX1ZbdVcuo0s1PfxTd//snpvWM436k2EJj5+pp6Xpq6WDprphUaF9eVU/uRxQcXFVnJPli\nVf382He2SotGZTysqo6e4lEZz19i8y3AlVV1/e6uZ0dJXgOcW1VX9x/ejwFPAu4EXlpVFw5a4A6S\n3B/44cIXdt8Fcd+q+sGwld2jr+lL0/arbylJ9gVOAR5RVb/R9xkfWlUfGbg0khy5s/sX+uVbM6k+\n7Q1JHgocDwz+n7eMPwcuAB7ar1/LdPZz/XvgnXTDv14G/E/gVOAzSX59yMJ6L6KbOAy6A8970XXh\nHEn3pThtPgHss2h9H2Cqvlj6L5Rr+mF+0+7dwI/pjmFBN9PnHw5Xzj2q6uI+mJ+0sLx429D1rdak\nQvtMukD8alV9rj9Q8ZUJ7Wu1HlxV59If3OnHa0/jqIwNwGFVdVxVHQc8nu4n31PowntoP17UDfIs\n4H1V9ZP+p/w0ztd+v6q6bWGlX953wHqWs5Fu1MMnkpy/cBu6qCU8uqrOBu4A6H+xDD/v6faW6hp7\n+e4uYlwm8qHqD5act2j968Bxk9jXGrQyKuPgqlo87vX6ftuNSaahb/tHSX6ObmzuM4DfW3TfNIbh\n7UkOr6rLAJL8a+BfBq5pKa8euoAR/TjJPtzzOXo0iw7wDqkfdvxS4JE7fOHtB9w4TFVrN5HQTnI/\nup/1TwDut7C9ql4xif2t0inA+cCjk3yGflTGsCUtaS7JR7jnS/C4ftv9gZuHK+tuJwN/Q/f6vbmq\nvgGQ5DnAtJ1QBV295yX5f3Qtwhm6Lp6p0lB/6xl0p4cfnOS9wNOZnlbsP9DN9f9g4E2Ltt8KTNXo\nq10xqQOR5wFX033LnUnXF3tVVZ089p3toiS/CHy7qrb1ozJ+iy4Ivwy8pqqm6hu4P535+cC/7Tfd\nBGyqqv84XFVtSrIX8FTgc3SjcWBKR+P0v/zOAQ4DfgrYG7i9qvYftLAl9L9Yn0r3JXhJVX1v4JLW\ntUn1aT+mql5N9ybbAvwqXR/sNPhTugMnAP+G7vTwt9OF4TuGKmo5fX/x1+lGY/waXRfE1A39SvKg\nJG9NclmSLyR5S/9hnhpVdRfw9qq6o6r+qb9NXWD33kZ3SvhX6A6WvpLufTpVkpxZVTdU1f/uR4zc\n2Le4B5fk0/2/tyb5/qLbrUm+P3R9qzWp0F74INzc93ceABw4oX3tqr0XtaZfBLyjqj7Qf8k8ZsC6\ntpPksUnOSHI1XYvrW3S/jJ5RVW8buLyl/DXwz3S/Wl7QL79/0IqW9okkxyWrvEDfbtRPq7B3f2D3\n3cDRQ9e0hIMXpljoh3x+kOkZdHB/gKrar6r2X3Tbbxp/sYxqUkf339GfYPFqun7jB9DNADcN9k6y\noR8tchTwm4vum6bRDlcDnwL+3cKcKEl+d9iSduqhVfUHi9b/MMnU9RXTdYedAtyZ5Id0P+lrCj/E\nP+jnGrkiydl0fbPTOP/9K4D39sH9DOBjVfXmgWtaMDUnd43TpEaPvLNfvBh41M4eO4D3ARcn+R7d\nqIFPASR5DNM1euT5wIuBTyb5OF1Ldppbh1uTvBg4t19/Ad2wz6lSVfsNXcOIfp0upF8F/C5wMFM0\nAqufXmHBW+i6HT9D99m6e3TOwA7c2bw3VfVHu7OYcRnrgciVJgaalhepP8jzUGBrVd3eb3ss8IAp\nebPdrR8lcgxd/+YzgfcAH6yqrYMW1ktyK12LJnQ/RxfGuu8N3DaFLdiF0+x/lu1HNk3F1KxJHl5V\n3xq6jpUk+eRO7q6qeuZuK2YZSb4L/A+WaewMPR/Oao07tKd60qDW9WHzQuBFVXXU0PW0qJ9a4WTg\nILqrmTwV+MdpCBmAJJdV1eH98gf6E6qmUj8a54VVNY3HLrZ7LdcTr8auNUnyuH7ekSU/HFP4y+VK\nugsJXFJVT+onZXp9VS01x8tutzBvz47L0yrJ52v7+bSnRguv32pM6uSaLcDJVXVzv74ReNOUnVyj\n8TiF7mDu4pMXFrcEpqIFu8gPq+qHSUhy3/4LZ5ou+FrLLE+rC5P8Ht1IodsXNk7J+Q7r8tfoRGf5\nW2mb2pfkCOBbVbWtXz+B7oDZN4HNU/LhvVuSD9JdpON36L5QbgLuU1Ny/dJ0VyW6na4fdh9gYfbB\nqRzlkuQbS2yuqpq2AQjrxqRC+/8As1V1U7/+QODiavRCmlpeksuAX+7nQvklulEuv003i9phVTWN\nUwMAd0/deQDw8ar68UqPl6bBpMYlvwm4JMnC8K8XAq+b0L40rCVPVgI+kOSKAevaTj8fzn+gO4Hq\nSuBdDc3vMdX6E+gez/ajcd4zXEXr26TGab8n3cU9F/ozn1/9Zb207rRystIWujN1PwU8my5kBp8L\np3X9iLFZutfzo3Sv7afphqZqAsb6oVqiNfMn/YdZ61crJys9fqF7Lsm7gEsHrme9eAHdNTcvr6oT\n010R6i8HrmldG3dLaMfWzGFM59VgNCZV9bokn+Cek5UWDpLsRde3PS3unhiqqu5sYOqRVvxLVd2V\n5M4k+9PP9z50UevZuEPb1sweqKouWWLbtUPUshNPXDSzW4B9+vWpHJXRkM8n+Wm6y+B9AbgN+Mdh\nS1rfxn1G5HZnIK3XM5Ik3VuSQ4D9q6rZCwy0YNyhvTDGFLYfZ2prRlqnkixcpKOAT1fVBwcuaV3z\nNHZJq5bkj+kGHryv3/Qi4GteWWlyDG1Jq9ZfpOOwhQPQ/SRSX6qqw4atbP2axknVJbXjq8DDF60f\n3G/ThEzTyQ+SGpHkw3R92PsBVyW5tF9/Co4amyhDW9JqvHHoAvZU9mlLWrP+xJq7G4HTNrvjemJL\nW9KqJflN4Ezgh8Bd9MN7mb5rw64btrQlrVqSrwBPq6rvDV3LnsLRI5LW4mvcc6EG7Qa2tCWtWpIn\nA+8GPgv8aGF7VZ00WFHrnH3aktbiT4GL6KZivmvgWvYItrQlrZrXft39DG1Jq5bk9XQXcf4w23eP\nOORvQgxtSavm1dh3P0NbkhrikD9JuyzJf160/MId7nv97q9oz2FoS1qNFy9aPn2H+47enYXsaQxt\nSauRZZaXWtcYGdqSVqOWWV5qXWPkgUhJu2zR9WAXXwuWfv1+VXWfoWpb7wxtSWqI3SOS1BBDW5Ia\nYmhLUkMMbUlqiKEtSQ35/13r5OqIB8cMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc370032e90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.811447811448\n",
      "0.789001122334\n",
      "0.82379349046\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import cross_validation\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=10, min_samples_split=2, min_samples_leaf=1)\n",
    "\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv =3)\n",
    "\n",
    "print(scores.mean())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<p>\n",
    "    Now we can map the effectiveness of each category for our model\n",
    "    \n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEpCAYAAABYyHNYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFTdJREFUeJzt3X+0ZWV93/H3hxmioDCZqMzYMAJiQtAmKstCUrriTagr\naFaAJhWL/gFak3StptjaWoamhpE2LmHFZqmJbUwImaTGCrUqpKZMCB67NEWUHysshPEnVLuYS/kt\nkBoC3/5x9mUud+6PM3fOufs+97xfa501e++z993fteeez33Os5+9d6oKSVIbDuu7AEnS6AxtSWqI\noS1JDTG0JakhhrYkNcTQlqSGrBjaSX44ya1Jbun+fSTJhUm2JtmTZG+S65JsWYuCJWma5WDGaSc5\nDPgOcBrwK8ADVXV5kouArVW1czJlSpLg4LtH/j7wjar6NnA2sLtbvhs4Z5yFSZIOdLCh/Sbgj7vp\nbVU1C1BV+4BjxlmYJOlAI4d2ksOBs4Cru0UL+1W8Hl6SJmzzQaz7euDmqrq/m59Nsq2qZpNsB+5b\nbKMkhrkkrUJVZeGyg+keOQ/42Lz5a4ALuunzgU8vs2NfVVxyySW917BeXh4Lj4XHYvnXUkYK7SRH\nMjwJ+d/mLb4MeF2SvcAZwPtG+VmSpNUbqXukqp4AXrRg2YMMg1yStEa8InINffCD/4kkG+q1ffvx\nqzoWMzMzYz22LfNY7OexWNlBXVyzqh0kNel9tCIJG2+QTZbtf5O0OkmoQzwRKUnqmaEtSQ0xtCWp\nIYa2JDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpi\naEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGjBTaSbYkuTrJnUnuSHJakq1J9iTZm+S6JFsmXawk\nTbtRW9ofAD5TVScDrwTuAnYC11fVScANwMWTKVGSNCdVtfwKydHArVV14oLldwGvrarZJNuBQVX9\nyCLb10r7mBZJgI12LIL/v9L4JaGqsnD5KC3tE4D7k1yZ5JYkH0lyJLCtqmYBqmofcMx4S5YkLTRK\naG8GTgF+u6pOAR5n2DWysHllc0uSJmzzCOt8B/h2VX25m/8Ew9CeTbJtXvfIfUv9gF27dj0zPTMz\nw8zMzKoLlqSNaDAYMBgMVlxvxT5tgCSfA36xqr6a5BLgyO6tB6vqsiQXAVurauci29qn3bFPW9Ko\nlurTHjW0Xwn8HnA48E3grcAm4CpgB3APcG5VPbzItoZ2x9CWNKpDCu1D3LGh3TG0JY3qUEaPSJLW\nCUNbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqiKEtSQ0x\ntCWpIYa2JDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1ZPMoKyW5G3gEeBp4\nsqpOTbIV+DhwHHA3cG5VPTKhOiVJjN7SfhqYqapXV9Wp3bKdwPVVdRJwA3DxJAqUJO03amhnkXXP\nBnZ307uBc8ZVlCRpcaOGdgF/luRLSd7eLdtWVbMAVbUPOGYSBUqS9hupTxs4varuTfIiYE+SvQyD\nfL6F85KkMRsptKvq3u7f/5vkU8CpwGySbVU1m2Q7cN9S2+/ateuZ6ZmZGWZmZg6lZknacAaDAYPB\nYMX1UrV8AznJkcBhVfVYkucBe4D3AGcAD1bVZUkuArZW1c5Ftq+V9jEtkrDxvpAE/3+l8UtCVeWA\n5SOE9gnAJxmmzWbgo1X1viQ/AFwF7ADuYTjk7+FFtje0O4a2pFGtOrTHsGNDu2NoSxrVUqHtFZGS\n1BBDW5IaYmhLUkMMbUlqiKEtSQ0xtCWpIYa2JDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkN\nMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNWTk0E5yWJJbklzT\nzW9NsifJ3iTXJdkyuTIlSXBwLe13AF+ZN78TuL6qTgJuAC4eZ2GSpAONFNpJjgXeAPzevMVnA7u7\n6d3AOeMtTZK00Kgt7d8E3gXUvGXbqmoWoKr2AceMuTZJ0gIrhnaSnwVmq+o2IMusWsu8J0kag80j\nrHM6cFaSNwBHAEcl+SNgX5JtVTWbZDtw31I/YNeuXc9Mz8zMMDMzc0hFS9JGMxgMGAwGK66XqtEb\nyEleC/zLqjoryeXAA1V1WZKLgK1VtXORbepg9rGRJWHjfSEJ/v9K45eEqjqgd+NQxmm/D3hdkr3A\nGd28JGmCDqqlvaod2NJ+hi1tSaOaREtbkrTGDG1JaoihLUkNMbQlqSGGtiQ1xNCWpIYY2pLUEENb\nkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqiKEtSQ0xtCWp\nIYa2JDXE0JakhhjaktSQFUM7yXOSfDHJrUluT3JJt3xrkj1J9ia5LsmWyZcrSdMtVbXySsmRVfVE\nkk3AF4ALgV8AHqiqy5NcBGytqp2LbFuj7GMaJAE22rEI/v9K45eEqsrC5SN1j1TVE93kc4DNDJPn\nbGB3t3w3cM4Y6pQkLWOk0E5yWJJbgX3An1XVl4BtVTULUFX7gGMmV6YkCUZvaT9dVa8GjgVOTfIK\nDvye73dkSZqwzQezclU9mmQAnAnMJtlWVbNJtgP3LbXdrl27npmemZlhZmZmVcVK0kY1GAwYDAYr\nrrfiicgkLwSerKpHkhwBXAe8D3gt8GBVXeaJyNF4IlLSqJY6ETlKS/vFwO4khzHsTvl4VX0myY3A\nVUneBtwDnDvWiiVJBxhpyN8h7cCW9jNsaUsa1SEN+ZMkrQ+GtiQ1xNCWpIYY2pLUEENbkhpyUBfX\nSNIkbd9+PLOz9/Rdxlht23Yc+/bdPbaf55C/NeSQP2l5fkbmbeWQP0lqn6EtSQ0xtCWpIYa2JDXE\n0JakhhjaktSQNRmnPRzGs7GMe+ylJI1iTcZpb7xxl7CasZeOQZWW52dk3laO05ak9hnaktQQQ1uS\nGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1ZMXQTnJskhuS3JHk9iQXdsu3JtmTZG+S65JsmXy5kjTd\nVrwiMsl2YHtV3Zbk+cDNwNnAW4EHquryJBcBW6tq5yLbe0Xk3BZe7SUty8/IvK1We0VkVe2rqtu6\n6ceAO4FjGQb37m613cA5B12VJOmgHFSfdpLjgVcBNwLbqmoWhsEOHDPu4iRJzzbyXf66rpH/Cryj\nqh4bdns8yzLt/13zpme6lyRpzmAwYDAYrLjeSHf5S7IZ+BPgT6vqA92yO4GZqprt+r0/W1UnL7Kt\nfdpzW9hfJy3Lz8i8rQ7xLn+/D3xlLrA71wAXdNPnA58+6KokSQdllNEjpwP/E7id4Z/AAv4NcBNw\nFbADuAc4t6oeXmR7W9pzW9iKkJblZ2TeVku0tH0IwqoZ2kOGtsbHz8i8rXwIgiS1z9CWpIYY2pLU\nEENbkhpiaEtSQwxtSWqIoS1JDTG0JakhhrYkNcTQlqSGGNqS1BBDW5IaYmhLUkMMbUlqyMiPG5PG\nafv245mdvafvMsZq27bj2Lfv7r7L0Abn/bRXzftpD63+XsEeCy3k78W8rbyftiS1z9CWpIYY2pLU\nEENbkhpiaEtSQwxtSWqIoS1JDVkxtJNckWQ2yV/OW7Y1yZ4ke5Ncl2TLZMuUJMFoLe0rgZ9ZsGwn\ncH1VnQTcAFw87sIkSQdaMbSr6vPAQwsWnw3s7qZ3A+eMuS5J0iJW26d9TFXNAlTVPuCY8ZUkSVrK\nuG4YtcKF9bvmTc90L0nSnMFgwGAwWHG9kW4YleQ44Nqq+rFu/k5gpqpmk2wHPltVJy+xrTeMmtvC\nm+Hs38pjoUX4ezFvq0O8YVS615xrgAu66fOBTx90RZKkg7ZiSzvJHzPsz3gBMAtcAnwKuBrYAdwD\nnFtVDy+xvS3tuS1sRezfymOhRfh7MW+rJVra3k971QztIUN7P0P7UPl7MW8r76ctSe0ztCWpIYa2\nJDXE0JakhhjaktQQQ1uSGmJoS1JDDG1JaoihLUkNMbQlqSGGtiQ1xNCWpIYY2pLUEENbkhpiaEtS\nQwxtSWqIoS1JDTG0Jakhm/suQJp227cfz+zsPX2XMVbbth3Hvn13913GhuQzIlfNZ0QO+YzI/TwW\n+3ks9vMZkZI0tQxtSWrIIYV2kjOT3JXkq0kuGldRkqTFrTq0kxwG/BbwM8ArgPOS/Mi4CpMkHehQ\nWtqnAl+rqnuq6kngvwBnj6csSdJiDiW0fxD49rz573TLJEkT4olISWrIoVxc83+Al8ybP7ZbtogD\nhhpuCMMxpQe91djr6NvqjgN4LJ615VjrWA88Fvut/lgs8rNWe3FNkk3AXuAM4F7gJuC8qrpzbNVJ\nkp5l1S3tqnoqya8Aexh2s1xhYEvSZE38MnZJ0vh4IlKSGmJoS+pVkiOSnNR3Ha2YSGgnOTHJc7rp\nmSQXJvn+SexLUruS/BxwG/A/uvlXJbmm36rWt4n0aSe5DXgNcDzwGeDTwCuq6g1j39k6l+TfAe+p\nqr/p5o8GPlBVb+23srWX4bintwAvrapLk7wE2F5VN/Vc2ppJci3L3Hu0qs5aw3J6l+Rm4KeBQVW9\nult2e1X9aL+VrV+T6h55ugupfwB8qKreBbx4Qvta7zYDX0zyY0leB3wJuLnnmvryYeAngPO6+e8C\nv91fOb34DeD9wLeAvwJ+t3s9Bnyjx7r68mRVPbJgmaMjljGpJ9c8meQ84Hzg57plh09oX+taVV2c\n5Hrgi8BDwE9W1dd7Lqsvp1XVKUluBaiqh5J8X99FraWq+hxAkvdX1WvmvXVtki/3VFaf7kjyZmBT\nkh8CLgT+ouea1rVJtbTfyrBF9etV9a0kJwB/NKF9rWtJfhL4IHApMAA+lORv9VpUf57sLsoqgCQv\nAp7ut6TePC/JS+dmus/I83qspy//jOFdQr8HfAx4FPjnvVa0zq3F48a2Ajuq6i8nuqN1KslNwAVV\n9ZVu/ueB91bV1N3GNslbgDcBpwC7gX8I/NuqurrXwnqQ5EzgI8A3GV63fRzwy1V1Xa+Fad2b1InI\nAXAWw+6Xm4H7gC9U1TvHvrN1LsmmqnpqwbIXVNUDfdXUp+6e62cwDKo/n+araLsRVnN/vO+qqu/1\nWc9a8oTs6k2qT3tLVT2a5O3AH1bVJUmmsqUNvDDJe4EfrKozk7ycYdfRFT3Xtaa6bpE7um8Yd/Vd\nT9+SHAm8Eziuqn4xyQ8lOamq/qTv2tbIb/RdQKsm1ae9OcmLgXOBafklXMofANexf/TMV5nCPrvu\n28bebpif4Ergrxn+AYfhHTL/fX/lrK2q+lx3UvZVc9Pzl/Vd33o2qdC+lGFQfb2qvtSdcPnahPa1\n3r2wqq6iO+HWDYV8avlNNqytDEcL/HmSa+ZefRfVkxOr6nLgSYCqeoKNeE/SlZ2/yLIL1rqIlkyk\ne6Q7sXT1vPlvAr8wiX014PEkL2D/iIkfBxaOS50W7+67gHXkr5Mcwf7fixMZjqCYCt2Q4DcDJyz4\nw30U8GA/VbVhIqGd5LnAP2Y4lOe5c8ur6m2T2N86907gGuDEJF8AXsRw1MTUmRujLAAuYXjp9o4k\nHwVOZ7pamH/B8D78L2R4sdGc7wLTev5rJJMaPXI1w5NNb2bYVfIW4M6qesfYd7ZOJfk7wLeral+S\nzcAvM/y28RXg16pq6loT3beMDwEnA98HbAIer6qjey2sJ903sB9n2C1yY1Xd33NJasCk+rRfVlXv\nZviB3A38LHDahPa1Xv0OwxNNAH8X+FWGl2w/xHB87jT6LYaXsH8NOAJ4O9N3GTsASS6tqgeq6r93\nI0Ye7FrcUyHJ57t/v5vk0Xmv7yZ5tO/61rNJhfaT3b8PJ/nbwBbgmAnta73aNK81/SbgI1X1ie6P\n2ct6rKtX3SX8m6rqqaq6Ejiz75p6siPJxfDMeO1PMl0n658HUFVHVdXR815HTes3r1FNapz2R7or\nId/NsD/3+cCvTWhf69WmJJu70SJnAL80771JHff17onuXiO3JbmcYZ/mtN7T/W3AR7vg/ingT6vq\nN3uuaS15U6hV8nFjE5LkV4E3APczfGr9KVVVSV4G7K6q03stsAdJjgNmGfZn/wuG38A+PE030Epy\nyrzZwxl2o32B7mKrqrqlj7rWWpLvAP9hqferasn3pt1YQzvJspepT9t/RHfi7cXAnqp6vFv2w8Dz\np+XDCZDkJVX1v/uuYz1I8tll3q6q+uk1K6ZHSe4F/iNLjE2vqvesbUXtGPfX9KPG/POaVlU3LrLs\nq33U0rNPMbxJFEk+UVXTOmafqvqpJIcBb6yqj/ddT4/urapL+y6iRWMNbf86agnzW1MvXXKtKVFV\nTyd5FzDNoT2NV3+OxaSeEbl7/jMhk2xN8vuT2JeaUEtMT7Prk/yrJDuS/MDcq++i1tAZfRfQqkld\nXHPr3PPellum6ZDkKeBxhq2rI4An5t5i2I87dUO8knxrkcVVVVP/TUTLm9TQs8OSbK2qhwC6FsS0\nDnObelW1qe8a1puqOqHvGtSmSQXp+4Ebk1zVzb8R+PUJ7UtqUnfh2ct59v15/rC/itSCiY3T7m72\nPzd86Ya5x21JgiSXADMMQ/szwOuBz1fVVN5MTKMb9zjt5wL/hOFl2rcDV3RXBEqaJ8ntwCuBW6vq\nlUm2Af+5ql7Xc2la58Y9emQ38BqGgf16fKSQtJS/qqqngb9JcjTD56ju6LkmNWDcfdovr6ofBUhy\nBXDTmH++tFF8uRsW+7sMH379GPC/+i1JLRh398gtVXXKUvOSDpTkeODoqvLm/1rRuEN7bjwuPHtM\n7tSOx5WWkuTngb/H8IKjz1fVJ3suSQ3wLn9SD5J8mOEJ+491i94EfKOq/ml/VakFhrbUgyR3ASdX\n9wHsbiJ1R1Wd3G9lWu+m9Qb0Ut++zvA+63N2dMukZXlpubSGklzLsA/7KODOJDd186fhaCuNwNCW\n1pbXLuiQ2Kct9ai7sOaZxtO8h0FLi7KlLfUgyS8BlwL/D3iablgsPiRCK7ClLfUgydeAn6iq+/uu\nRW1x9IjUj2+w/2EQ0shsaUs9SPJq4Ergi8D35pZX1YW9FaUm2Kct9eN3gBsY3hHz6Z5rUUNsaUs9\n8JmpWi1DW+pBkvcCdwPX8uzuEYf8aVmGttQDn8au1TK0JakhDvmT1lCSfz1v+o0L3nvv2lek1hja\n0tr6R/OmL17w3plrWYjaZGhLaytLTC82Lx3A0JbWVi0xvdi8dABPREpraN5zVOc/Q5Vu/rlVdXhf\ntakNhrYkNcTuEUlqiKEtSQ0xtCWpIYa2JDXE0Jakhvx/itLm1Rsd54kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc370f426d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# in this case we are probably overfitting with useless columns so we can remove the less efficient ones \n",
    "# Perform feature selection\n",
    "selector = SelectKBest(f_classif, k=5)\n",
    "selector.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "# Get the raw p-values for each feature, and transform from p-values into scores\n",
    "scores = -np.log10(selector.pvalues_)\n",
    "\n",
    "# Plot the scores.  See how \"Pclass\", \"Sex\", \"Title\", and \"Fare\" are the best?\n",
    "plt.bar(range(len(predictors)), scores)\n",
    "plt.xticks(range(len(predictors)), predictors, rotation='vertical')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>\n",
    "We can try 3,4,5 of the best options to find the best amount\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 categories 0.789001122334\n",
      "4 categories 0.811447811448\n",
      "5 categories 0.82379349046\n"
     ]
    }
   ],
   "source": [
    "\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=150, min_samples_split=6, min_samples_leaf=3)\n",
    "\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv =3)\n",
    "\n",
    "print('3 categories ' + str(scores.mean()))\n",
    "\n",
    "\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=150, min_samples_split=8, min_samples_leaf=4)\n",
    "\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv =3)\n",
    "\n",
    "print('4 categories ' + str(scores.mean()))\n",
    "\n",
    "\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Embarked\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=150, min_samples_split=10, min_samples_leaf= 5)\n",
    "\n",
    "scores = cross_validation.cross_val_score(alg, titanic[predictors], titanic[\"Survived\"], cv =3)\n",
    "\n",
    "print('5 categories ' + str(scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> \n",
    "        We see either 4 or 5 is the best and since there might be overfitting we will submit both version and compare it to the test data\n",
    "        </p>\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#4 categories\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=150, min_samples_split=8, min_samples_leaf=4)\n",
    "\n",
    "alg.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "predictions = alg.predict(titanic_test[predictors])\n",
    "\n",
    "submission = pd.DataFrame({\n",
    "        \"PassengerId\": titanic_test[\"PassengerId\"],\n",
    "        \"Survived\": predictions\n",
    "    })\n",
    "\n",
    "submission.to_csv(\"kaggle_v3-1.csv\", index = False)\n",
    "\n",
    "#5 categories\n",
    "predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Embarked\", \"Title\"]\n",
    "\n",
    "alg = RandomForestClassifier(random_state=1, n_estimators=150, min_samples_split=10, min_samples_leaf= 5)\n",
    "\n",
    "alg.fit(titanic[predictors], titanic[\"Survived\"])\n",
    "\n",
    "predictions = alg.predict(titanic_test[predictors])\n",
    "\n",
    "submission = pd.DataFrame({\n",
    "        \"PassengerId\": titanic_test[\"PassengerId\"],\n",
    "        \"Survived\": predictions\n",
    "    })\n",
    "\n",
    "submission.to_csv(\"kaggle_v3-2.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> \n",
    "Using 5 leaves was the better model with a score of 0.78469\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}