Enron - Clean Def Version .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# some general settings\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "pd.set_option('display.float_format', lambda x: '%.5f' % x)\n",
    "np.set_printoptions(precision=6,threshold=5000,linewidth=200,suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to walk through directories and process spam-files\n",
    "# only takes text after the first newline!! (dus dit is mijn data-opschoondeel: de header wordt overgeslagen)\n",
    "import os\n",
    "\n",
    "def read_files(path):\n",
    "    emails_text = []\n",
    "    for (dirpath, dirs_in_path, file_names) in os.walk(path):\n",
    "        for file_name in file_names:\n",
    "            lines = []\n",
    "            file_to_open = os.path.join(dirpath,file_name)\n",
    "            with open(file_to_open, 'r') as spam_file:\n",
    "                indicator = False\n",
    "                for line in spam_file:\n",
    "                    if indicator:\n",
    "                        lines.append(line)\n",
    "                    elif line == '\\n':\n",
    "                        indicator = True\n",
    "            emails_text.append(\"\\n\".join(lines))\n",
    "    return emails_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#functie om data uit file in een dataframe te stoppen en df te returnen\n",
    "def df_to_append(location,class_spam,email_text):\n",
    "    df = pd.DataFrame(columns=['location','class_spam','email'])\n",
    "    df['email'] = email_text\n",
    "    df['location'] = location\n",
    "    df['class_spam'] = class_spam\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# location of data, and 0=ham, 1=spam\n",
    "input_emails = [\n",
    "                ['data\\\\beck-s',      0L],\n",
    "                ['data\\\\farmer-d',    0L],\n",
    "                ['data\\\\kaminski-v',  0L],\n",
    "                ['data\\\\kitchen-l',   0L],\n",
    "                ['data\\\\lokay-m',     0L],\n",
    "                ['data\\\\williams-w3', 0L],\n",
    "                ['data\\\\BG',          1L],\n",
    "                ['data\\\\GP',          1L],\n",
    "                ['data\\\\SH',          1L]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#aanmaken van spam/ham dataset:\n",
    "\n",
    "# 1. definieren kolomnamen\n",
    "column_names = ['location','class_spam','email']\n",
    "\n",
    "# 2. aanmaken dataframe\n",
    "data = pd.DataFrame(columns=column_names)\n",
    "\n",
    "# 3. loop alle locaties af, haal emailtext per locatie op, stop die in een df en append die dan in 1 final df\n",
    "for location, class_spam in input_emails:\n",
    "    email_text = read_files(location)\n",
    "    df_to_add = df_to_append(location,class_spam,email_text)\n",
    "    data = data.append(df_to_add,ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sla data in juiste vorm op, zodat ik dat niet telkens opnieuw hoef te doen\n",
    "from sklearn.externals import joblib\n",
    "joblib.dump(data, 'data.pkl')\n",
    "\n",
    "# om data weer te kunnen loaden moet regel hieronder geuncomment worden\n",
    "#data = joblib.load('data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# aanmaken van train- en testset, location heb ik er nu buitengehouden\n",
    "from sklearn.cross_validation import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data['email'], data['class_spam'], test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV \n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "\n",
    "count_vect = CountVectorizer(encoding=\"latin-1\") #utf-8 encoding werkte niet\n",
    "tfidf = TfidfTransformer()\n",
    "mnb = MultinomialNB()\n",
    "svc = LinearSVC()\n",
    "\n",
    "steps = [\n",
    "            ('count_vect', count_vect),\n",
    "            ('tfidf', tfidf),\n",
    "            ('mnb',mnb)\n",
    "            #('svc',svc)\n",
    "]\n",
    "\n",
    "pipeline = Pipeline(steps)\n",
    "\n",
    "# parameters die goed werkten op deze dataset:\n",
    "# ngram:(1,2), min_df: 1, max_df:0.2\n",
    "parameters = {\n",
    "                'count_vect__ngram_range':[(1,2)],                   #[(1,1),(1,2),(1,3)]\n",
    "                'count_vect__min_df':[1],                            #[1,5,10]\n",
    "                'count_vect__max_df':[0.2],                          #[0.2,0.5,1.0]\n",
    "                'tfidf__sublinear_tf':[False]                           #[True,False]\n",
    "                #,'svc__C':[0.1,1,10,1000],\n",
    "                #'svc__tol':[1e-3,1e-4,1e-5]\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters, cv=2, n_jobs=1, scoring='roc_auc', verbose=5) #scoring measure is hier AUC ipv accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 1 candidates, totalling 2 fits\n",
      "[CV] count_vect__max_df=0.2, count_vect__min_df=1, tfidf__sublinear_tf=False, count_vect__ngram_range=(1, 2) \n",
      "[CV]  count_vect__max_df=0.2, count_vect__min_df=1, tfidf__sublinear_tf=False, count_vect__ngram_range=(1, 2), score=0.999545 - 1.2min\n",
      "[CV] count_vect__max_df=0.2, count_vect__min_df=1, tfidf__sublinear_tf=False, count_vect__ngram_range=(1, 2) \n",
      "[CV]  count_vect__max_df=0.2, count_vect__min_df=1, tfidf__sublinear_tf=False, count_vect__ngram_range=(1, 2), score=0.999689 - 1.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:  2.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2, error_score='raise',\n",
       "       estimator=Pipeline(steps=[('count_vect', CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding='latin-1', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "   ...False,\n",
       "         use_idf=True)), ('mnb', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))]),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid={'count_vect__max_df': [0.2], 'count_vect__min_df': [1], 'tfidf__sublinear_tf': [False], 'count_vect__ngram_range': [(1, 2)]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring='roc_auc', verbose=5)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('count_vect', CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding='latin-1', input=u'content',\n",
       "        lowercase=True, max_df=0.2, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "   ...False,\n",
       "         use_idf=True)), ('mnb', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#bij countvect: min_df = 1, max_df = 0.2 en ngrams = (1,2)\n",
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#opslaan van best_estimator van de gridsearch\n",
    "#bij countvect: min_df = 1, max_df = 0.2 en ngrams = (1,2)\n",
    "joblib.dump(grid_search.best_estimator_, 'countvect_tfidf_mnbayes_best_est.pkl')\n",
    "# grid_search_best = joblib.load('countvect_tfidf_mnbayes_best_est.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6580    6]\n",
      " [  76 3754]]\n"
     ]
    }
   ],
   "source": [
    "# hoe is de accuracy: printen confusion matrix\n",
    "from sklearn.metrics import confusion_matrix\n",
    "conf_matrix = confusion_matrix(y_test, grid_search.predict(X_test),labels=[1,0])\n",
    "print conf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x353514a8>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD9CAYAAABp2RZmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGhhJREFUeJzt3Xl8FfW9//HXORAISwiigiC4y8elVkGlLlRwK4q7rcvF\n64pr63pv7aNSqYgXrf21WsUKKrjB1ar9Xa2C4lo1kVYRuFrUfmRxZRELgbCFLef+MQNElmQScnJm\nhvfTx3kkZ85kzufgyTuf853vzGRyuRwiIlJY2UIXICIiCmMRkVhQGIuIxIDCWEQkBhTGIiIxoDAW\nEYmB5vnceKseV2venGyiYtJ9hS5BYqi4OZmt3UZ9MmfF1Pu2+vkaU17DWESkSWWS+2FfYSwi6ZGJ\nVbNbLwpjEUkPdcYiIjGQbVboChpMYSwi6aFhChGRGNAwhYhIDKgzFhGJAXXGIiIxoM5YRCQG1BmL\niMSApraJiMSAOmMRkRjIasxYRKTw1BmLiMRAI86mMLNfAqcCRcD9wNvAo0A1MM3dfxaudxlwObAa\nGObu482sGBgLdAQqgQvdfUFtz5fcPyMiIhvLZKPfamFmfYDD3f0IoC+wC3AXMMjd+wBZMzvNzDoB\n1wCHAycAd5hZEXAV8KG7HwWMAQbXVbrCWETSI5OJfqtdP2CamT0HPA+MA3q6e1n4+EvA8UAvoNzd\n17h7JTAdOBDoDUyose5xdT2hhilEJD0ab2rbDgTd8MnAHgSBXLN5XQK0A0qAxTWWLwVKN1q+bt1a\nKYxFJD0abwfeAuATd18DfGpmVUDXGo+XAIsIxoPbbbS8IlxestG6tdIwhYikR+MNU5QTjAFjZl2A\nNsDr4VgywIlAGTAJ6G1mLcysFNgHmAZMBPqH6/YP162VOmMRSY9G6ozDGRE/NLP3gAzBDrnPgVHh\nDrpPgD+7e87M7iUI7wzBDr5VZjYCeMzMyoCVwIA6S8/l8ncBZ10dWjZHV4eWzWmUq0OfdG/0q0OP\nvzZWR4ioMxaR9NBBHyIiMaAwFhGJAZ21TUQkBnRyeRGRGNAwhYhIDKgzFhEpvIzCWESk8BTGIiIx\nkNGVPkRECk+dsYhIDKQ+jM2sHVC87r67z89bRSIiDZTqMDazxwnOWr+I4KxEOaBnnusSEam/5GZx\npM7Y3H2PvFciIrKVktwZRzlc5T0zs7xXIiKylTKZTORb3ETpjBcDk8xsKeEwhbt3yW9ZIiL1l82m\n+3DoY4AO4bWgRETiK34Nb2RRwvhToBMwO8+1iIhslTgOP0QVJYyPBD43s3+F9zVMISKxlOowdve9\nm6IQEZGtleowNrPDgIuBIoIRmS7u3i/fhYmI1FtyszjS1LYRwJtAKfAF8K9a1xYRKZAkT22LEsb/\ncvcngUp3HwJ0zW9JIiINk81mI9/iJsoOvGoz2x9oHR780SHPNYmINEgcO96oooTxfwD7A/cCTwAP\n57UiEZGGSm4WR5pN8ZGZFQEGXOTu/8h/WSIi9ZfkzrjOgRMzuw34I3AY8ICZ3Zj3qkREGiDtO/BO\nBH7o7jcAPwTOym9JIiINk/Yw/hooCb8vAr7JXzkiIlshU49bzETZgdcF+NTMPgD2A1aZ2UQAdz8i\nn8XF3c8vPp6T+hxAUfNmPPB0GWOe//v6x64e0JeLzjiCbxcuCe4P+xOfz17AqKHns2uXDqxZW81P\nhz7JjC/ns3vXHXjo1n+nOpfjoxlzueE3TxfqJUkTGv3Qg7z15husWb2as/9tAKef8eNCl5R4cZyy\nFlWUMNawxGb0Pngven1/d46+6C5aF7fguguO+c7jPfbdhYE3P84H/vX6ZSf1OYBmzbIcc/HdHP0D\nY+g1pzDgxtHc+Z9ncst9L/DO1JncM+gcTu57AOPe1H7SNHt/0nt8+MFUHv/vP7F8+XIef1STlBpD\nHIcfoooSxqVAG6AauB243d1fz2tVCXD84fvy8Yy5PH3XZbRtXcygPzz7ncd77NeNGy/5ETvt0I4J\n5R/xu0deZfoX82neLPjLXdq2FatWrwWg577deGfqTABeeedjjj1sH4Vxyk18p5y99u7O9df8lGXL\nlvEfP/9FoUtKhbSH8UjgauBW4FfAb4FtPoy3b9+Wbjttx5nXjWT3nXfgz3+4nIPO/K/1jz89YTIP\nPPU2lcuqePquy+nXezbTPp3Drl2254NnB9OhtA0/vm7kJttdsqyKdm2LN1ku6bKoooK5c+cw/P4H\n+Prrr7ju6qv4y7gJhS4r+ZKbxZHCuAr4CGjh7n83s7V5rikRFi5ehn82j7Vrq5nx5XyqVq1h+/Zt\nWLBoGQB/fOJNliyrAmBC+TQO2qcbR/cyXp34CUP++AI7d2zPSw9eyyFn3U4ut2G7JW2KWbxkRQFe\nkTSl0vbt2X2PPWnevDm77bY7LVu0pKJiIdttpwNct0aSO+Moo9054HHgRTM7G1id35KSYeLUmRx/\nxH4AdN6xlNbFLdYHcUmbYiY/M4hWxUUA9D20O1M+/pKFi5dTuTQI2orK5TRvliWbzTD1n19xZM89\nAfjRkfvxzpSZBXhF0pR69DyYd8rLAJg//xtWVFXRvv12Ba4q+ZI8tS1KZ3wO0At4CegLnJvPgpJi\nQvlHHNlzT8rG/JxMJsP1dzzNeaf8gFwuxxPj3uPXw5/nlYeuo2rVGv76rvPqxE94Z8pMHhhyHq+O\nvp6i5s349fDnqVq5mpvufpb7Bw+gqHmWf372Df/z2tRCvzzJs6P69GXK5PcZcM5PIJfjV4NviWVA\nJE2S/wkzuZqfkRtZqx5X52/jMbT/Xl3osW83xr7wbqFLibWKSfcVugSJoeLmWz/i2/0XEyJnzqe/\nPSFW0R2lM5aIFi5epiAWKaAkf7pQGDeiud8uLnQJItu0BGdxpMsufY/gah/bAWOBae4+Lt+FiYjU\nVzab3DSOMpviHoJr4H0LjAaG5LMgEZGGymSi3+Im0oHc7j4DyLn7t8CS/JYkItIwaZ/attDMrgDa\nmNm5wKI81yQi0iBJHqaIEsYDgUEEV4U+JLwvIhI7jd3xmllH4H3gOKA1MA74NHx4hLs/Y2aXAZcT\nHBA3zN3Hm1kxwT62jkAlcKG7L6jtuaKEcXvg/hr32wIL6/F6RESaRGNmsZk1Jzg3z/Jw0cHA7939\n7hrrdAKuAXoShHW5mb0CXAV86O5DzewcYDBwfW3PFyWMnyI4JDoL7A5MB3rX50WJiDSFRu6Mf0cw\nk+ym8P7BQHczO52gO76B4OjkcndfA1Sa2XTgQIKMvDP8uZcIwrhWde7Ac/fD3f0Idz+M4KKkc+r3\nekREmkZjzaYws4uA+e7+KhuuDfIucKO79wFmAbcA7YCaBxgsJTjtcEmN5UvC9WpV34M+FgN71PNn\nRESaRCN2xhcD1WZ2PHAQ8BhwqrvPDx9/DrgXeIvvBm0JUEEwTlxSY1mdEx+iHPTxN4JhCggGo1+t\n82WIiBRAY2Vx2P0CYGZvAFcCz5vZNe4+CTgWmAxMAoaZWQugFbAPMA2YCPQn2PnXHyir6zmjdMYX\nA+tOsFvl7rogqYjEUp6ntl0J3Gdmq4B5wOXuvtTM7gXKCYYyBrn7KjMbATxmZmXASmBAXRuPEsaj\n3F077EQk9vJxMIe717zA5SZZ6O6jCY5OrrlsBXB2fZ4nShgvM7O7ASe4Dh7u/mB9nkREpCnE8MC6\nyKKE8cTwa6fw6zZ1jmIRSY44HuYcVZQwXuvu66+0aWZ35LEeEZEGS3AWbzmMzWwgcCmwr5n1Dxdn\ngRZsmAQtIhIbae2MxwKvE5yXYli4rBqYv8WfEBEpoARn8ZbD2N1XAp8TnABDRCT2stlIZwWOJV12\nSURSI5WdsYhI0qR1zFhEJFESnMUKYxFJD3XGIiIxkOAsVhiLSHpkE5zGCmMRSY20X5BURCQREpzF\nCmMRSQ/twBMRiYEEZ7HCWETSI0Ny01hhLCKpoTFjEZEY0JixiEgMNEtwa6wwFpHUSHBjrDAWkfTQ\nMIWISAwkOIsVxiKSHjo3hYhIDCQ3ihXGIpIimk0hIhID2oEnIhIDCc5ihbGIpIc6YxGRGEjwkLHC\nWETSQ52xiEgMJDeKFcYikiKa2iYiEgMaphARiYEEZ7HCWETSQ+emEBGJgQRnscJYRNJDY8ZbsOC9\n4fncvCTU+WOnFLoEiaFnLuq51dvINkIdhaLOWERSQ1PbRERiIMFZrDAWkfTQmLGISAyoMxYRiYHG\naozNLAs8BBhQDVwJrAQeDe9Pc/efheteBlwOrAaGuft4MysGxgIdgUrgQndfUNtzJnnno4jId2Qz\nmci3OpwC5Ny9NzAYuB24Cxjk7n2ArJmdZmadgGuAw4ETgDvMrAi4CvjQ3Y8CxoTbqL32hr5oEZG4\nydbjVht3/wtBtwuwK1AB9HT3snDZS8DxQC+g3N3XuHslMB04EOgNTKix7nF11a5hChFJjcac2ubu\n1Wb2KHA6cBZB+K6zBGgHlACLayxfCpRutHzdurVSZywiqZHJRL9F4e4XAd2BUUCrGg+VAIsIxoPb\nbbS8IlxestG6tVIYi0hqZDPRb7Uxs383s1+Gd6uAtcD7ZtYnXHYiUAZMAnqbWQszKwX2AaYBE4H+\n4br9w3VrpWEKEUmNRjxr2/8Aj5jZWwQ5eS3wT2BUuIPuE+DP7p4zs3uBcoILjQxy91VmNgJ4zMzK\nCGZhDKjrCRXGIpIajZXF7r4cOGczD/XdzLqjgdEbLVsBnF2f51QYi0hq6KAPEZEYyCT4kqQKYxFJ\njeYJnpKgMBaR1NCJgkREYkBjxiIiMZDgxlhhLCLpoatDi4jEgIYpRERioJk6YxGRwktwFiuMRSQ9\nNEwhIhID2oEnIhIDCc5ihbGIpIc6YxGRGEhwFiuMRSQ9NLVNRCQGkhvFCmMRSRGNGYuIxEByo1hh\nLCIpkuDGWGEsIumhk8uLiMRAgq+6pDAWkfTQDjwRkRhI9TCFmQ0DBgLVBDsrc+7eJd+FiYjUV9qH\nKfoDu7r7ynwXIyKyNVLdGQP/CxQDCmMRibXkRnG0MJ4GzDWzeWwYptgjv2WJiNRfghvjSGF8DrA7\nsCjPtYiIbJVsgnvjKGH8BbBMY8YiEndpn9rWDZhpZrPC+zl3PyKPNYmINEiCszjyMIWISOylfZii\nCDgr/JoBugBX5LMoEZGGSHJnHGWO9BPh194EO/K2z185IiINl8lEv8VNlDBe6u53AF+7+0VAp/yW\nJCLSMJl6/Bc3UYYpcma2E1BiZm2AtnmuSUSkQbLxy9jIooTxrcAZwBhgVvhVRCR2Uj21zd3fBt4O\n7z6f33JERBoujsMPUUU5a9stwNXA6nXLdNa22j3/l2d54blnyWQyVK2sYro74195g9uGDGZJZSXV\n1dXcdvud7Ny1a6FLlUaWy1Xz9YsPsXLhXDKZDDufMJDiHYL/z6uXLeLL54aHJxWAqvlfsFPff2P7\nHscy/ZFBZFu2AqBFaUe6nbRhwlLFR++wYPLL7HXB0EK8pERJ+zDFycAu7r4i38WkxamnncGpp50B\nwG+GDeX0M37CPXf9P/qffArH/+gE3n/vXT77bJbCOIUqp0+BTIa9zh/C0i8/Zt6bT7HbT/4TgKI2\n7dnzvMEALJs9nW/efpoOBx1D9Zqgz9lzwOBNtrdi3mdUfPhmk9WfdEnujKPMpphPja5Yovto2j+Y\nNXMmZ/7kLP536hTmf/MNV156CS+9OI5DDu1V6PIkD0q7H0LXEy8FYNWib2lW3Gaz68155VF27jcw\n+PQ0/wuqV61k1p/uYNaTw1g+ewYAa1YsZd7bT9PluAuarP6kS+XUNjN70syeIJjKNnXd/XCZRPDw\nqAe54qdXAzBn9hzatStl5KiH6bRTZx4Z9WCBq5N8yWSyfDVuBHNee5z2+x+5yeOV0ydTvGM3WnbY\nCYBsUUt2POxk9jj3JnbudwlfPn8f1WvX8PWLD9D52PPJFhVDrqlfRTJl6nGLm9qGKUY2WRUptGTJ\nEr784nMOPuRQANpv154+fY8GoE/fo/njvX8oZHmSZ91OvorOyxYz/bGbsct+T7aoxfrHKj4qZ4dD\nTlx/v0WHzrTYLpi+37JDZ5q1asuKOTNYVfENs18eTW71aqoWzGbOa2Poctz5Tf5akqRZHFveiLYY\nxu7+VlMWkjZTJk+i1w8OX3+/R8+DKS97m/4nn8Lk9yex5157F7A6yZeKaWWsXrKQjoefRqZ5EZlM\ndpPPxCvmzqJN1+4bfuaDv1L17Vfs3O8SVi9ZSPWqKlp37U73S38LwKrF3/LlX4YriKNo5Cw2sx8A\nv3H3o83sIGAc8Gn48Ah3f8bMLgMuJxjOHebu482sGBgLdAQqgQvdfUFtz6ULkubJF5999p0ddDf8\n/BcM/fXNPPPUk7RtW8Ltv/1dAauTfCm1Xnw1fiQzx95KrrqaLsddwKKPJwLQ4ft9WLO8kmzL1t/5\nmQ4HHs1X40cyY8wQMpkMXU+6IghxqbfG3IFnZjcC5wNLw0UHA79397trrNMJuAboCbQGys3sFeAq\n4EN3H2pm5wCDgetrez6FcZ5ccPHA79zv3LkLIx56uEDVSFPJFrVk19Ov+86yFfO/ZMW8zwBo3rod\n3S+54zuPZ5o1Z5dTr97iNluU7qhpbRE18ijFDDYc8AZBGHc3s9MJuuMbgF5AubuvASrNbDpwIMG5\nfO4Mf+4lgjCuVZ1/fs3se2ZWZmbTzOyXZnZyfV+RyLaseau2dPh+n0KXsU1ozB147v4ssKbGoneB\nG929D8HRyLcA7YDFNdZZCpQCJTWWLwnXq1WUz0L3ABcD3wKjgSERfkZEQkUlHQpdwrYjv9MpnnP3\nqeu+Bw4iCNyaQVsCVBCME5fUWFbnZesiDUy5+wyCK3x8S5DyIiKxk+eztr1sZoeE3x8LTAYmAb3N\nrIWZlQL7EFzEeSLQP1y3P1BW18ajjBkvNLMrgDZmdi66MKmIxFSeD4e+ChhuZquAecDl7r7UzO4F\nygn67UHuvsrMRgCPmVkZsBIYUNfGM7lc7bPJzawdMAg4APgEuN3dF0apfPnqOjYu26QL/3tq3SvJ\nNueZi3pudZRO+aIycub03LVdrCYlR+mM2wP317jfFogUxiIiTSnJ56aIEsZPERyMmSW47NJ0gmkb\nIiKxkuAD8CKdz3j9YWRm1h7QSRVEJJYSnMX1PuhjMbBHPgoREdlqCU7jKCeX/xsbzhnVEXg1rxWJ\niDRQ2seMLwbWnVi+yt2/yWM9IiINlvYrfYxyd+2wE5H4S3kYLzOzuwEHqgHcXTvxRCR20j5MMTH8\n2in8qgM5RCSWUj21DVjr7v+17o6Z3VHbyiIihZLgLN5yGJvZQOBSYF8zW3fCiyzQAripCWoTEamf\nBKdxbZ3xWOB1gvNSDAuXVRNcLVpEJHZSOWbs7iuBzwmu7SQiEntpn9omIpIMCmMRkcJL5TCFiEjS\npH1qm4hIIiQ4ixXGIpIiCU5jhbGIpIbGjEVEYkBT20RE4kBhLCJSeBqmEBGJAU1tExGJgQRnscJY\nRNJDnbGISCwkN40VxiKSGpraJiISAxqmEBGJAU1tExGJg+RmscJYRNIjwVmsMBaR9NCYsYhIDGQS\nnMYKYxFJjeRGscJYRFIkwY2xwlhE0kNT20REYiDJnXG20AWIiIg6YxFJkSR3xgpjEUmNbILTWGEs\nIqmR3ChWGItImiQ4jRXGIpIamtomIhIDCR4yVhiLSHokOIsVxiKSIglOY4WxiKRGkqe2ZXK5XKFr\nEBHZ5ulwaBGRGFAYi4jEgMJYRCQGFMYiIjGgMBYRiQGFsYhIDGiecURm9iQwAmgFdHP3UVtY7zLg\nYXdfG2GbVwCd3H1ooxYrTa6p3h9mdiFg7j6ocSqXuFAY15O7v1zHKoOAx4A6f9kkffT+kIZKfRiH\nncTpQAmwPTDU3Z81s38AnwIrgSuB0UCH8MeudfePzOxnwEBgLrBjje3t4+43mdnNwGlAM2AksAbY\nCfgTcKaZ3Q70Dh+/y93/v5n1Bv4ALCT4hfzbRvXuDTwCrCYYRhoA7AX8CqgGOgEPufv9ZnYUcAvB\nQaBtw3VXA08BXwG7ht9/DzgIeNHdf7XV/6gpkrT3R+hwM3sZ2AEY4e6jzOzHwM8IfqdzwBnAAcBN\n4WvoCjwAHAN8H7jH3R/Y6n9AaTTbyphxa3c/DugH3GVmzQjC61Z3H0DQrbzm7scCVwAjzawjcC3Q\ni+AXqkWN7eXM7CCgn7sfGq6zt7s/TPCLeY6ZnQDs5u5HEfwC3GxmpcD9wDnu/iPgs83UejzwLnAc\nMAQoDZd3AU4GDgduMLMdgP2B89z9GOBZ4Kxw3d2Bi4FTgNuA64HDCIJDNpWk9wfAKnfvB5wJ3BAu\n6w70D7f3SfhaAHYmCOafEvxBPw/oT/AHRmJkWwnjtwDcfT5QQdjFEHQ+EHQQl5jZG8BDwHbAnsA0\nd1/j7muASRtt04D3wu2ucfcbazyWCbd5SLjNCQQdy25AR3efGa73zmZqHQ0sBl4m6HTWfZydGD5P\nFTAtrG82MNzMHgaOBorCdWe5+1JgETDP3Re7+0qCzlo2laT3B8CU8Os8gjFqgPnAY+F74QA2vBem\nuXs1wXthZjhWXQG0rPVfRJrcthLGBwOYWSegHcEbFzaE0yfA3WGHeTYwFpgO7G9mLcNOqcdG2/wn\n0DPcbpGZvWJmLcJtNgsffyPc5jHA08BMYLaZWbiNQzdT62lAWdip/Rn4Rbi8h5llzKw1sF9Y30PA\nRe5+CTCHzZ+zKrOF72WDJL0/IBiGWM/M2gG3AucClwJVbPh/XXNdvRdibFsJ485m9hrwAnBV2CnU\nfJPeTvDR8a/ASwTdxL+AOwnG7MYDS2tu0N0/AF42s4nA28AYd18FlAPj3f0FYJmZvQ28D+TCbvVK\nYIyZvQrsspla3weGmtnrBB+Jh4fLi8La3gJuc/eFwBig3MzKCD5WdwnXrfnatvS9bJCk98cm3L0y\n3O7fgTJgORveCzXpvRBjqT9rWxqmAplZH+CKcPxSGlEa3h+SDttKZywiEmup74xFRJJAnbGISAwo\njEVEYkBhLCISAwpjEZEYUBiLiMSAwlhEJAb+D9NqcfUY7rPBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x350085f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#visualiseren confusion matrix met seaborn\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "sns.heatmap(conf_matrix, annot=True, fmt=',.0f', cmap=\"Blues\", xticklabels=['predicted spam','predicted ham'], yticklabels=['true spam', 'true ham'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True spam, maar predicted ham: 6 = False Negatives\n",
      "True ham, maar predicted spam: 76 = False Positives\n"
     ]
    }
   ],
   "source": [
    "# hoeveel ga er nu echt fout, zowel FN als FP\n",
    "truespam_predham = conf_matrix[0,1]\n",
    "truespam_predspam = conf_matrix[0,0]\n",
    "trueham_predspam = conf_matrix[1,0]\n",
    "trueham_predham = conf_matrix[1,1]\n",
    "print \"True spam, maar predicted ham: %s = False Negatives\" %(truespam_predham)\n",
    "print \"True ham, maar predicted spam: %s = False Positives\" %(trueham_predspam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Datascience\\Anaconda\\lib\\site-packages\\matplotlib\\axes\\_axes.py:519: UserWarning: No labelled objects found. Use label='...' kwarg on individual plots.\n",
      "  warnings.warn(\"No labelled objects found. \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAFtCAYAAAANqrPLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtgVNWBx/HfJJMJ5AGREmBRoMhDUFheiqWUUqIRBJaA\nEIlisrQ+qlSpLlqNvB8hFC2sgFTZLigvUSpRSaFFSvCJGpH4QECloiioLGAgk0kmk7n7R8yQSIaZ\nDDOTDPf7+acz9945c+6pzC/n3nPPsRiGYQgAAFzwohq6AgAAIDwIfQAATILQBwDAJAh9AABMgtAH\nAMAkCH0AAEzC2tAVAFA/3bp1U9euXRUVFSWLxSKHw6HExETNnDlTPXr0kCQ5HA4tWbJEBQUFstls\nslgsGjJkiO666y7FxsZ6ysrLy9Ozzz6r8vJyVVRUqG/fvnrggQeUmJjYUKcHIIQsPKcPRJbu3bvr\nrbfeUvPmzT3bVq5cqW3btmnDhg2qrKzUhAkT1KdPH917772KjY1VeXm5Hn30Ue3bt0+rV69WVFSU\nnnjiCb3++utasmSJWrRoocrKSuXk5OiTTz7R2rVrG/AMAYQKl/eBCGMYhmr+rV5ZWakjR44oKSlJ\nkrR161YZhqEHH3zQ06uPjY3V1KlTVVJSopdfflkOh0NPPvmkcnNz1aJFC0lSdHS0HnzwQWVkZMjl\ncp31vZWVlcrNzdXQoUM1cuRITZ8+XRUVFVq2bJnmzZvnOa7m+8zMTN1zzz0aOXKkVq9erZ/97Gee\nst1utwYPHqyDBw+qpKRE2dnZGjt2rNLS0rRgwQK53e7QNCBgYoQ+EIGysrKUlpamQYMGaejQobJY\nLJo/f74kqaioSFdeeWWdnxswYIB2796tf/3rX4qLi1O7du1q7Y+NjdXIkSNltZ5952/dunX6+OOP\ntXnzZuXn58tut2vr1q0+69q8eXPl5+crKytLXbp00Y4dOyRJr732mi655BJ16tRJ8+fPV48ePfT8\n888rLy9PJ06c0MqVK+vbLAB84J4+EIHWrFmj5s2ba9++fbr99tvVp08fT49dUp09dUlyOp2yWq2K\nioqqd096165dSktLk81mkyQtWrRIUlXP/lxq/gGSnp6uTZs26brrrlNeXp7S09MlSTt37tSHH36o\njRs3SpLKy8tlsVjqVT8AvhH6QASqvrzfvXt3ZWdna+rUqerdu7fatm2rvn376i9/+UudnyksLNTv\nfvc7derUSS6XS4cPH67V23c6nbr77ruVk5OjqVOn6rvvvpPFYtHkyZNltVprBfHx48c9fzjUvN1Q\nUVFR63vj4uI8r4cNG6YFCxbo4MGDevfdd7Vw4UJJVZf6H3vsMV166aWSpJKSkvNtIgB14PI+EOFG\njBihvn37KicnR5I0dOhQxcXFKScnR+Xl5ZKksrIyzZ07V/Hx8br22mtls9l0++236+GHH9bx48cl\nVQV+Tk6OHA6HkpOTtWLFCr3wwgvKy8vTkCFDNGDAAOXn58vpdMrtdmvWrFn629/+phYtWmjv3r2S\npNLSUr3++ute62qz2TR8+HBlZ2fruuuu81w1+MUvfqGnnnrKU48777xT69atC1WTAaZFTx+IMHVd\n9p42bZrS0tL0xhtvaODAgVq5cqUef/xx3XDDDZ5L+SkpKVq1apWio6MlSXfccYeaNm2qW2+9VRaL\nReXl5erfv7/+/Oc/1/m9GRkZOnLkiMaOHStJuvrqq5WVlaXS0lK99tprGjp0qFq1aqU+ffqcs67p\n6elat26d5syZ49k2depUzZ8/X//xH/8hl8ulgQMH6rbbbjuvdgJwNh7ZAwDAJEJ+ef/9999XZmbm\nWdt37NihcePGKSMjwzN4BwAAhE5IL+//5S9/0Ysvvqj4+Pha210ulxYsWKBNmzYpNjZWN910k665\n5ppao48BAEBwhbSn36FDBz3++ONnbT948KA6dOighIQExcTEqF+/fiosLAxlVQAAML2Qhn5qaqpn\n0FBNJSUlteb2jo+P1+nTp0NZFQAATK9BRu8nJCTUeg7XbrerWbNmPj9nGAYTdiDiVLgq9c3xUkm1\nn2evNYLWqPOlvI2zrbnZvzKr3hSXOOV2V03jW7276uOG57VR4/NGje3VOwwZPzruHGXVKMzwfL5q\n+3cnS5UYZzursrXPre4TMs6nvepo7INfF+uixFj9WJ2l1bHR22hof8dJezusPuOs6zrUqKNm9Rm6\nXdf3e/14Xd/vZ528ft7POnmtl5/f//G/TigpMVY148XfdvK3jeouz7//f/784DX+VcYPYQn9HzdK\np06d9MUXX+jUqVNq0qSJCgsLdeutt/osx2Kx6NgxrgiEUnJyYqNt4wqXW24//yWWOSv13clSWRT8\nPxK/PVmqok//TzFW/y6UvfXxt0GvA4Dg+r6kXE1jo6Uavxl1/Xr42+/0t4Na12Gh7NqGJfSrTz4/\nP18Oh0Pp6enKzs7Wb37zGxmGofT0dLVq1SocVUGEKvrs//T4pg9V6Y7cJ0x/1ediSVX/oJs0jVGZ\no6LWv+6a/9AtXnbU+jGotb3GD5WXX4zq7SWlFbo4OUEWS40iLDVKqLG95g9XncdbzlSl1rG1yrKc\nVa/q1+UVbiXF2855Pl7byEtjVG9v3rypThU7vJTzo0INyRYTpbjYOn4S62jQ8wmDqmP9K9PbDu/H\n+lnXen1XHWX+sKlFiwSdOHHu2RPrc3XW7wD0Umbdx/p3XNNYq6zRF/58dRH3nH5j7YVeKBprT3/L\nW1/orzsPqkObRDXzXBI+t2J7uXpe+pOQ1Cc6yqJf/Pu/KTrKvx+JxLiYWj8ojbWdLyS0cejRxuGR\nnJzo+yA/MSMfGsxXx0p04lSZX8cePW6XJI0ZdKn+vVNoghwALnSEPoLmg4P/p7f9vH9tL3Ppg4PH\n6/0dNj/vowMAzkboXwD8HdwmSceLy3TydLnX/d+ddur776tGmh/48qTyXvv8rPu1wahHtfgmVg3/\nWQf/jm0aoy7tmtf7OwAAVQj9CLdx52fa+taXIf0OQ1Kni30/UilJrS9qqjGDLvXrWIvFoqQEG49h\nAkCYEPqNjKvSrY8PnZSzotKv4z/4rOoSebf2SX4d73ZXzXXQpV3dx8fH2WQvdXrex8VadU2/ixVj\nPXuSJQBAZCH0JVW63frw4AmVOV1+Hf/Z18UqLXcpyo8eqmEY2rX3WyXGxfj17OWp0gq/6lCTzRql\n+2/q41d9fGE0LgBcuAh9VfWWl276MKTfcbq0Qv/2kzifx8U3jdHp0gr9vEcbJSc19avsS5LjgxL4\nAIALG6EvyfFDD3/AFW38HigWF2vVpW39u89tkUUtmsVy7xoA0KAu2NAv2PO1vvjGv8vU356oGq1+\nWfsk/bJX21BWCwCABnNBhn6l2621/zjgfWEIL37SrElI6gMAQGNwQYZ+9QpgnS5upltHXO7XZ2Jj\noutcZQsAgAvFBRn61WJjotWmhe/BcwAAmAFzmgIAYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGAShD4A\nACZB6AMAYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGAShD4AACZB6AMAYBKEPgAAJkHoAwBgEoQ+AAAm\nQegDAGAShD4AACZB6AMAYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGAShD4AACZB6AMAYBKEPgAAJkHo\nAwBgEoQ+AAAmQegDAGAShD4AACZB6AMAYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGAShD4AACZB6AMA\nYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGASIQ19wzA0c+ZMZWRkKCsrS4cPH661/6WXXtINN9yg9PR0\nPfPMM6GsCgAApmcNZeHbt2+X0+nUhg0b9P777ys3N1fLly/37F+4cKG2bt2qJk2aaMSIERo5cqQS\nExNDWSUAAEwrpKG/e/duDRo0SJLUq1cvffTRR7X2d+vWTcXFxbJYLJLk+V8AABB8IQ39kpKSWj13\nq9Uqt9utqKiquwpdunTR2LFjFRcXp9TUVCUkJISyOgAAmFpIQz8hIUF2u93zvmbgHzhwQDt37tSO\nHTsUFxen+++/X//4xz80dOjQc5aZnOz78r+r0i1JssVY/ToetdFm4UE7hx5tHHq0cWQJaej37dtX\nBQUFGjZsmIqKitS1a1fPvsTERDVt2lQ2m00Wi0UtWrTQqVOnfJZ57Nhpn8dUh76zwuXX8TgjOTmR\nNgsD2jn0aOPQo43DI5h/WIU09FNTU/XGG28oIyNDkpSbm6v8/Hw5HA6lp6frxhtv1M033yybzab2\n7dtrzJgxoawOAACmFtLQt1gsmj17dq1tHTt29LzOyMjw/EEAAABCi8l5AAAwCUIfAACTIPQBADAJ\nQh8AAJMg9AEAMAlCHwAAkyD0AQAwCUIfAACTIPQBADAJQh8AAJMg9AEAMAlCHwAAkyD0AQAwCUIf\nAACTIPQBADAJQh8AAJMg9AEAMAlCHwAAkyD0AQAwCUIfAACTIPQBADAJQh8AAJMg9AEAMAlCHwAA\nkyD0AQAwCUIfAACTIPQBADAJQh8AAJMg9AEAMAlCHwAAkyD0AQAwCUIfAACTIPQBADAJQh8AAJMg\n9AEAMAlCHwAAkyD0AQAwCUIfAACTIPQBADAJQh8AAJMg9AEAMAlCHwAAkyD0AQAwCUIfAACT8Cv0\nS0tLtX//fhmGodLS0lDXCQAAhIDP0N+1a5fS0tI0adIkHTt2TCkpKXr99dfDUTcAABBEPkN/0aJF\nWr9+vZo1a6ZWrVpp7dq1WrhwYTjqBgAAgshn6LvdbiUnJ3ved+7cOaQVAgAAoWH1dUCbNm1UUFAg\ni8WiU6dOad26dWrbtm046gYAAILIZ09/zpw52rx5s44eParU1FTt27dPc+fODUfdAABAEPns6e/f\nv1+LFi2qtW3btm267rrrQlYpAAAQfF5Df8uWLXI6nVqyZIkmT57s2e5yufTkk08S+gAARBivoV9S\nUqI9e/bIbrfr7bff9myPjo7WfffdF5bKAQCA4PEa+jfeeKNuvPFG7dq1SwMGDAiocMMwNGvWLB04\ncEA2m005OTlq166dZ/8HH3ygP/7xj5Kkli1b6pFHHpHNZgvouwAAwLn5vKcfExOju+66S6WlpTIM\nQ263W0eOHNGOHTt8Fr59+3Y5nU5t2LBB77//vnJzc7V8+XLP/hkzZmjp0qVq166d/vrXv+rIkSP6\n6U9/el4nBAAA6uZz9P60adN07bXXqrKyUhMmTFCHDh107bXX+lX47t27NWjQIElSr1699NFHH3n2\nff7550pKStKqVauUmZmp4uJiAh8AgBDyGfpNmjTR2LFj1b9/fzVr1kzz5s1TYWGhX4WXlJQoMTHR\n895qtcrtdkuSTp48qaKiImVmZmrVqlV68803a40dAAAAweUz9GNjY/X999+rY8eOev/992WxWPxe\ndCchIUF2u93z3u12Kyqq6iuTkpLUvn17dezYUVarVYMGDap1JQAAAASXz3v6EydO1H333aelS5dq\n3Lhx2rx5s3r06OFX4X379lVBQYGGDRumoqIide3a1bOvXbt2Ki0t1eHDh9WuXTvt3r1b48aN81lm\ncnKiz2NclVVXE2wxVr+OR220WXjQzqFHG4cebRxZLIZhGL4OMgzD08M/dOiQ2rdvr4SEBJ+F1xy9\nL0m5ubnau3evHA6H0tPT9fbbb+vRRx+VJPXp00cPP/ywzzKPHTvt8xhXpVt3PLJTl//0It2f0cfn\n8TgjOTnRrzbG+aGdQ482Dj3aODyC+YeV157+iRMntGrVKjVv3lwTJ06U1WpVkyZNtGfPHt122216\n8803fRZusVg0e/bsWts6duzoeX311Vdr48aN51F9AADgL6+hf//99ys+Pl4nT55URUWFBg8erD/8\n4Q9yOBzKzs4OZx0BAEAQeA39L7/8Utu3b1dJSYkyMjK0fv16ZWZmauLEiUygAwBABPIa+tX37BMS\nEvT9999r6dKl6tOH++MAAEQqr4/sWSwWz+uWLVsS+AAARDivPX273a53331XbrdbDodD7777rmoO\n9L/qqqvCUkEAABAcXkO/devWeuyxxyRJrVq18ryWqq4CrF69OvS1AwAAQeM19NesWRPOegAAgBDz\nOQ0vAAC4MBD6AACYBKEPAIBJ+Az94uJiTZs2TVlZWTp58qSys7NVXFwcjroBAIAg8hn606dPV8+e\nPfX9998rPj5erVq10gMPPBCOugEAgCDyGfpfffWVxo8fr6ioKNlsNt1333365ptvwlE3AAAQRD5D\nPzo6WqdPn/bM0Hfo0CFFRTEUAACASOP1Of1q99xzjzIzM3X06FFNmjRJRUVFmj9/fjjqFjBHuUuS\n1MTm8/QAADANn6k4cOBA9ejRQx988IEqKys1Z84ctWzZMhx1C5i9rCr045sQ+gAAVPOZir/61a+U\nmpqqUaNGqXfv3uGo03mzl1VIkuKbxjRwTQAAaDx83pzPz89X9+7dtXjxYg0bNkxLly7VF198EY66\nBczuoKcPAMCP+Qz95s2bKz09XU8//bQeeeQRFRQU6Prrrw9H3QJGTx8AgLP57AqfOHFCW7du1ZYt\nW1RcXKyRI0dq2bJl4ahbwOyOqtBPaELoAwBQzWfop6Wl6frrr1d2drZ69OgRjjqdNwbyAQBwNp+p\n+Morr0Tcc/nVPf04evoAAHh4Df0xY8YoLy9Pl19+uWdiHkkyDEMWi0X79u0LSwUDceaePj19AACq\neU3FvLw8SdL+/fvP2ud0OkNXoyA4c3mfnj4AANV8XrcfP358rfdut1tjx44NWYWCwV5Woegoi5rY\nohu6KgAANBpee/pZWVl65513JEndunU78wGrVSkpKaGv2XmwO1yKb2KtdVsCAACz8xr6q1evliTN\nmzdP06ZNC1uFgsFeVqEEntEHAKAWr6FfUFCgIUOG6IorrtALL7xw1v7Ro0eHtGKBMgxDdodLrS+K\na+iqAADQqHgN/Q8//FBDhgzxXOL/scYa+mXOSrkNQ3E8ow8AQC1ek3Hy5MmSpNzcXM+2kpISHT16\nVF26dAl9zQJU/Yw+I/cBAKjN5+j9jRs3Kjs7WydOnNDw4cM1efJkLV68OBx1C4jncT2e0QcAoBaf\nof/MM8/owQcfVH5+vq655hpt3rxZr732WjjqFpDqiXmYdx8AgNr8ml83KSlJr7zyin71q1/JarWq\nvLw81PUK2JmePqEPAEBNPkO/c+fO+u1vf6uvvvpKAwYM0O9//3v17NkzHHULyJl7+lzeBwCgJp/J\nOH/+fO3Zs0ddu3aVzWZTWlqafvnLX4ajbgE5M+8+PX0AAGry2dOvqKhQQUGBfv3rXystLU1vvfVW\no5573+6ourzPI3sAANTmM/TnzJmjsrIyzZ8/X3/84x/lcrk0c+bMcNQtICUM5AMAoE4+u8N79+7V\nSy+95Hk/Y8YMDR8+PKSVOh+lDOQDAKBOPnv6hmHo1KlTnvenTp1SdHTjXb2ueiBfXCyX9wEAqMln\nMk6cOFHjxo3zrKy3Y8cO3XHHHSGvWKDsZRWKi7UqKooV9gAAqMln6I8dO1Y9e/ZUYWGh3G63li5d\nqssuuywcdQuIvczFbHwAANTBazq63W6tW7dOhw4dUr9+/TRhwoRw1itgdkeF2raMb+hqAADQ6Hi9\npz9r1iz9/e9/V9OmTfXEE09o2bJl4axXQJwVlXK63EzMAwBAHbyGfmFhodauXav7779fTz/9tLZt\n2xbOegWEKXgBAPDOa+jHxsbKYqkaDHfRRRd5XjdmpWUsqwsAgDdeQ//HIR8V5dfaPA2KZXUBAPDO\nazoeOXJE2dnZXt/n5uaGtmYBOLPYDj19AAB+zGvoP/TQQ7Xe9+/fP+SVOV8lXN4HAMArr6E/ZsyY\ncNYjKKoX2+HyPgAAZ2v8N+rrwU5PHwAAry6w0P+hp89z+gAAnMWv0C8tLdX+/ftlGIZKS0tDXaeA\neQby8Zw+AABn8Rn6u3btUlpamiZNmqRjx44pJSVFr7/+ejjqVm9nntOnpw8AwI/5DP1FixZp/fr1\natasmVq1aqW1a9dq4cKF4ahbvZWUuWSLiVKMtfEu/QsAQEPxGfput1vJycme9507d/a7cMMwNHPm\nTGVkZCgrK0uHDx+u87gZM2Zo0aJFfpfrjd1RwSA+AAC88Bn6bdq0UUFBgSwWi06dOqU///nPatu2\nrV+Fb9++XU6nUxs2bNCUKVPqnNBnw4YN+uSTT+pf8zrYy1yEPgAAXvgM/Tlz5mjz5s06evSorr32\nWu3bt09z5szxq/Ddu3dr0KBBkqRevXrpo48+qrV/z549+vDDD5WRkRFA1WurdLvlKHdxPx8AAC98\nJuRPfvKTgC+9l5SUKDEx8cyXWa1yu92KiorSsWPHtGzZMi1fvlxbtmwJqPyaSllhDwCAc/IZ+ikp\nKXWusPfPf/7TZ+EJCQmy2+2e99WBL0l///vf9f333+v222/XsWPHVF5erksvvVSjR48+Z5nJyYl1\nbnceK5Ek/SSpqddj4B/aLzxo59CjjUOPNo4sPkN/zZo1ntcul0svv/yynE6nX4X37dtXBQUFGjZs\nmIqKitS1a1fPvszMTGVmZkqS8vLy9Pnnn/sMfEk6dux0ndsPHymWJEVbvB8D35KTE2m/MKCdQ482\nDj3aODyC+YeVz9C/+OKLa72/7bbbdMMNN2jSpEk+C09NTdUbb7zhuWefm5ur/Px8ORwOpaenB1jl\nunnm3eeePgAAdfKZkIWFhZ7XhmHo008/VXl5uV+FWywWzZ49u9a2jh07nnVcMBb38cy7zz19AADq\n5DP0lyxZ4nltsVh00UUXacGCBSGtVCCqp+BN4JE9AADq5DP0r7/+et18883hqMt5qV5sJ47L+wAA\n1Mnnc/rr168PRz3Om2exHXr6AADUyWe3uE2bNsrKylKvXr0UGxvr2X733XeHtGL1deaePj19AADq\n4jMhe/fuHY56nLfqy/v09AEAqJvX0M/Ly9OYMWMaXY/eG3tZhaKjLGpiY4U9AADq4vWe/urVq8NZ\nj/Nmd1TNu1/X7IEAAMCPgXyRwl5WwTP6AACcg9fL+59++qmuueaas7YbhiGLxeLX3PvhYhiG7A6X\nWl8U19BVAQCg0fIa+h06dNCKFSvCWZeAlTkr5TYMntEHAOAcvKZkTEzMWfPuN1Y8ow8AgG9e7+n3\n7ds3nPU4L57H9XhGHwAAr7yG/owZM8JZj/NSPTEP8+4DAODdBTF6/0xPn9AHAMCbCyP0Pff0ubwP\nAIA3F0boe+bdp6cPAIA3F0boO1hWFwAAXy6I0C9hIB8AAD5dEKFfykA+AAB8uiBCv3ogX1wsl/cB\nAPDmwgj9sgrFxVoVFcUKewAAeHOBhL6L2fgAAPDhwgh9RwXz7gMA4EPEh76zolJOl5uJeQAA8CHi\nQ58peAEA8M8FEPosqwsAgD8iPvRLWVYXAAC/RHzon1lsh54+AADnEvGhX8LlfQAA/BLxoV+92A6X\n9wEAOLfID316+gAA+OUCCP0fevo8pw8AwDlFfuhXD+TjOX0AAM4p4kO/1HN5n54+AADnEvGhX1Lm\nki0mSjHW6IauCgAAjVrEhz6L7QAA4J/ID/0yF6EPAIAfIjr0K91uOcpd3M8HAMAPER36paywBwCA\n3yI69HlGHwAA/0V46POMPgAA/ors0HfQ0wcAwF+RHfr09AEA8Ftkh/4PU/Am8MgeAAA+RXbo/zCQ\nL47L+wAA+BTZoe9gWV0AAPwV2aHvuadPTx8AAF8iPPSrR+/T0wcAwJcID/0KRUdZ1MTGCnsAAPgS\n2aHvqJp332KxNHRVAABo9CI79MsqeEYfAAA/RWzoG4Yhu8PF43oAAPgpYkO/zFkpt2EwiA8AAD+F\ntJtsGIZmzZqlAwcOyGazKScnR+3atfPsz8/P1+rVq2W1WtW1a1fNmjXL77J5Rh8AgPoJaU9/+/bt\ncjqd2rBhg6ZMmaLc3FzPvvLyci1ZskRr167V+vXrdfr0aRUUFPhdtudxPZ7RBwDALyEN/d27d2vQ\noEGSpF69eumjjz7y7LPZbNqwYYNsNpskyeVyKTY21u+yqyfmYd59AAD8E9LQLykpUWJioue91WqV\n2+2WJFksFrVo0UKStGbNGjkcDv385z/3u+wzPX1CHwAAf4T02nhCQoLsdrvnvdvtVlTUmb8zDMPQ\nwoUL9cUXX2jZsmV+lZmcXPVHhOWz45KkNq0SPdsQHLRneNDOoUcbhx5tHFlCGvp9+/ZVQUGBhg0b\npqKiInXt2rXW/unTp6tJkyZavny532UeO3ZakvTtD/9b6XR5tuH8JScn0p5hQDuHHm0cerRxeATz\nD6uQhn5qaqreeOMNZWRkSJJyc3OVn58vh8OhK664Qps2bVK/fv2UmZkpi8WirKwsXXvttX6VbXcw\nkA8AgPoIaWJaLBbNnj271raOHTt6Xn/88ccBl13CQD4AAOolYifn8Tynz0A+AAD8ErGhX/rD6P24\nWC7vAwDgj4gNfXtZheJirYqKYoU9AAD8EcGh72IQHwAA9RC5oe+oUByD+AAA8FtEhr6zolJOl1sJ\nLKsLAIDfIjL0mYIXAID6i9DQZ1ldAADqKyJDv5RldQEAqLeIDH3PxDz09AEA8FtEhn4Jl/cBAKi3\niAx9z2I7jN4HAMBvkRn6Zcy7DwBAfUVo6NPTBwCgviIz9FlhDwCAeovI0C/1DOSjpw8AgL8iMvRL\nylyyxUQpxhrd0FUBACBiRGTo2x0VPK4HAEA9RWbol7kIfQAA6iniQr/S7Zaj3MX9fAAA6iniQr+U\nFfYAAAhIxIU+z+gDABCYCAx9ntEHACAQkRf6zLsPAEBAIi/06ekDABCQyAv9H6bgTeCRPQAA6iXy\nQv+HgXxxXN4HAKBeIi/0qxfboacPAEC9RF7oe+7p09MHAKA+IjD0q0fv09MHAKA+IjD0KxQdZVET\nGyvsAQBQH5EX+o6qefctFktDVwUAgIgSeaFfVsEz+gAABCCiQt8wDNkdLh7XAwAgABEV+o5yl9yG\nwSA+AAACEFGhf7qUZ/QBAAhURIV+SalTEs/oAwAQiAgLfebdBwAgUBEV+qcd1T19Qh8AgPqKrND3\n3NPn8j4GyaC6AAAMe0lEQVQAAPUVUaFffU8/jsv7AADUW0SFvqenz0A+AADqLaJCv7qnz0A+AADq\nL6JC/3QpA/kAAAhURIV+iaPq8n5cLJf3AQCor8gK/dIKxcVaFRXFCnsAANRXRIX+6VIng/gAAAhQ\nhIV+BY/rAQAQoIgKfWdFpRKYmAcAgIBEVOhLjNwHACBQkRf6XN4HACAgkRf6DOQDACAgkRf69PQB\nAAgIoQ8AgEmENPQNw9DMmTOVkZGhrKwsHT58uNb+HTt2aNy4ccrIyNDGjRv9KpNldQEACExIQ3/7\n9u1yOp3asGGDpkyZotzcXM8+l8ulBQsW6KmnntKaNWv07LPP6sSJEz7LZPQ+AACBCWno7969W4MG\nDZIk9erVSx999JFn38GDB9WhQwclJCQoJiZG/fr1U2Fhoc8y6ekDABCYkIZ+SUmJEhMTPe+tVqvc\nbned++Lj43X69GmfZdLTBwAgMCHtNickJMhut3veu91uRUVFefaVlJR49tntdjVr1uyc5W3+U1po\nKopakpMTfR+E80Y7hx5tHHq0cWQJaU+/b9++euWVVyRJRUVF6tq1q2dfp06d9MUXX+jUqVNyOp0q\nLCxU7969Q1kdAABMzWIYhhGqwg3D0KxZs3TgwAFJUm5urvbu3SuHw6H09HTt3LlTy5Ytk2EYGjdu\nnG666aZQVQUAANMLaegDAIDGI+Im5wEAAIEh9AEAMAlCHwAAk2jQ0A9kml5vn/nyyy91880365Zb\nbtHs2bPDfi6NVTDbeN++fZowYYKysrJ02223+TWDolkEs52rbd68WRkZGWE7h8YumG184sQJTZo0\nSZmZmbr55pvPKsusgtnG+/fv1/jx4zVhwgRNnTo17OfSWJ3P9PTvv/++MjMzPe8Dyj2jAW3bts14\n6KGHDMMwjKKiIuOuu+7y7KuoqDBSU1ON06dPG06n0xg7dqxx/Phxr5+58847jcLCQsMwDGPGjBnG\nyy+/HOazaZyC2ca33HKLsX//fsMwDGPDhg1Gbm5umM+m8QpmOxuGYezdu9f4z//8T2P8+PHhPZFG\nLJht/NBDDxlbt241DMMw3nrrLWPnzp1hPpvGKZht/Lvf/c549dVXDcMwjClTphgFBQXhPZlGKpA2\nNgzD+J//+R9j5MiRtX4TAsm9Bu3p12ea3iuvvFLvvPPOWZ/Zu3evJGnv3r268sorJUm//OUvtWvX\nrjCfTeMUzDZevHixLrvsMklVayfExsaG+Wwar2C0c/VnTp48qf/+7/+md/Qjwfxv+b333tM333yj\nX//618rPz9fVV18d/hNqhILZxt27d9fJkydlGIbsdrusVqZQlwKfnr5Dhw56/PHHa5UVSO41aOjX\nZ5reuLg4nT59Wna7vdb26OhoVVZWyqjx5KG/U/qaQbDa2O12q2XLlpKqfjDXr1+viRMnhuckIkAw\n2tlqtcrpdGratGl66KGH1LRp01r/XZtdMH8vvv76ayUlJWnVqlVq06aNVqxYEb4TacSC+Xvx05/+\nVDk5ORoxYoROnDih/v37h+9EGrFAp6dPTU1VdHS013L9zb0GDf36TtPbvHnzOj8THR3t+Vz1sb6m\n9DWLYLVx9We2bNmi2bNna8WKFbrooovCdBaNX7Da+cCBA/ryyy81a9YsTZkyRQcPHqy1OqWZBfP3\nIikpSUOGDJEkpaSkeHqnZhfM34ucnBytX79eW7Zs0ahRo7RgwYLwnUgjFszp6QPJvQYN/fpM0/vu\nu++qd+/e6tOnT52fufzyyz2XQV599VX169cvzGfTOAWzjV988UWtW7dOa9as0cUXXxz+k2nEgtXO\nPXv21ObNm7V69WotWrRInTt3VnZ2doOcU2MTzP+W+/Xr59leWFiozp07h/lsGqdgtnFSUpISEhIk\nSa1bt9apU6fCfDaN0/lOT1/z6l/37t3rnXsNOiOfEcA0vXV9pmPHjjp06JCmT5+uiooKderUSfPm\nzZPFYmmoU2s0gtHGCxYsUPv27TVgwAC1bdtWCQkJslgs6t+/v+6+++6GPL1GI5j/LVf7+uuvNWXK\nFG3YsKFBzqmxCWYbHzlyRNOmTZPD4VBiYqL+9Kc/1bqsalbBbOP33ntPjzzyiKxWq2w2m+bOnau2\nbds25Ok1CoG0cbUf/yYEkntMwwsAgEkwOQ8AACZB6AMAYBKEPgAAJkHoAwBgEoQ+AAAmQegDAGAS\nTIYMhMnXX3+toUOHqkuXLpKqnte1WCx64okn1Lp16zo/s2zZMkk6r/kQ8vLytGDBArVt21aGYai8\nvFxXXXWVZs2aVWtGL38sWbJEPXv21JAhQ5SVlaXVq1dLksaMGaO8vLyA6yhJmZmZ+vbbbxUfHy/D\nMFRSUqL27dvr0UcfVYsWLbx+7rnnnlNCQoKGDx9+Xt8PmAGhD4RR69atzzscA5GSkuKZztcwDN1y\nyy1at25drWU6/TF58mTP63feecfzOljnNH/+fM8CIpJ0zz33aNWqVZoyZYrXz+zZs4cFcwA/EfpA\nI/Dpp59q7ty5cjgcOn78uH7zm9/olltu8ex3uVx6+OGH9dlnn0mSbrrpJqWnp+v48eOaMWOGvvnm\nG0VFRem//uu/NGDAgHN+l8ViUZ8+fXTo0CFJ0vPPP6+nnnpKFotFV1xxhWbMmKGYmJg6vy87O1v9\n+/f3zFU/fvx4Pfvss+rWrZs+/vhjDR48WC+++KJatGih4uJijRw5Ujt37tQbb7yhpUuXqrKyUpdc\nconmzp2r5s2bn1W36oVHpKrFR06ePKlevXpJkrZu3aqnnnpK5eXlKisr07x581RRUaEdO3bo7bff\nVnJysrp161bv9gDMhHv6QBh9++23GjNmjEaPHq0xY8Zo5cqVkqSNGzdq0qRJ2rhxo55++mktWrSo\n1uf27Nmj4uJibdq0SStXrtR7770nScrJydG4ceP0/PPPa/ny5ZoxY4ZKS0vPWYeTJ0965un+5JNP\n9OSTT2rdunV66aWX1LRpUy1dutTr90lVfzRMmzZNkvTss896tkVFRen666/X1q1bJUnbtm1Tamqq\niouLtWjRIq1cuVKbNm3SwIED9cgjj9RZt+nTp2v06NH6xS9+oYyMDA0cOFATJ06UYRh67rnn9OST\nT+qFF17Q7bffrv/93//VgAEDlJKSosmTJ2vgwIEBtQdgJvT0gTDydnn/oYce0muvvaYVK1bowIED\ncjgctfZ36dJFhw4d0q233qrBgwfrgQcekCS9+eab+vzzz/XYY49JkiorK/Xll1+qW7dutT6/Y8cO\njRkzRm63W4Zh6LrrrtPw4cO1bt06paSkeFbnuvHGG/Xwww/rt7/9bZ3f58uoUaOUm5urCRMmKD8/\nX/fdd58++OADHT16VFlZWTIMQ263W0lJSXV+PicnR1deeaX27NmjyZMna/DgwZ512JcuXaqCggJ9\n/vnneuedd+pcZtTf9gDMitAHGoHf//73nuVehw8fri1bttTan5SUpM2bN2vXrl3auXOnRo8erb/9\n7W8yDENPP/20J7S/++47JScnn1V+zXv6NdW8nF6tsrJSzZs3P+v7flynuvTo0UPFxcX68MMP9e23\n36p379765z//qX79+mn58uWSJKfTWWtp0ZqqlwLp06ePMjMz9Yc//EEvvfSSysrKNG7cOI0ePVpX\nXXWVLrvsMq1bt67Oz/vTHoBZcXkfCCNv61vt2rVLkydPVkpKimeAXM1jd+zYoQceeECDBw/W1KlT\nFR8fr2+++UZXX321J/w+++wzjRo16qyrBOfSv39/7dixw7Ps6XPPPaerr766zu87evRorc9arVbP\nHw016zpy5EjNnDlTI0aMkCT16tVLRUVFnjEEjz/+uBYuXOizbhMnTpTD4dAzzzyjQ4cOKTo6Wnfe\nead+9rOf6dVXX/V8d3R0tFwulySdd3sAFzp6+kAYeVv28u6779ZNN92kZs2aqWPHjrrkkkv01Vdf\nefYPHjxY//jHPzRixAjFxsbquuuuU5cuXTRt2jTNmDFDo0aNkiQ9+uijiouL87s+l112me644w5N\nmDBBlZWVuuKKKzR79mzZbDZt27btrO+rKSUlRWlpaXr++edrndeoUaO0ZMkSLV68WJLUsmVLzZ8/\nX/fee6/cbrfatGlT5z39H7eNzWbTvffeq9zcXG3btk3dunXT0KFDFRcXp6uuukpHjhyRJP385z/X\n4sWL1axZM02fPl3Tp08PuD2ACx1L6wIAYBJc3gcAwCQIfQAATILQBwDAJAh9AABMgtAHAMAkCH0A\nAEyC0AcAwCQIfQAATOL/AXklRrxZr9eRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x14483278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# printen roc curve\n",
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, grid_search.predict_proba(X_test)[:,1])\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlim([0, 0.01])\n",
    "plt.ylim([0, 1.00])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC-curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}