machine-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json \n",
    "import pyspark as ps\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n",
      "-1\r\n"
     ]
    }
   ],
   "source": [
    "!head data/churn/upsell.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Var1\tVar2\tVar3\tVar4\tVar5\tVar6\tVar7\tVar8\tVar9\tVar10\tVar11\tVar12\tVar13\tVar14\tVar15\tVar16\tVar17\tVar18\tVar19\tVar20\tVar21\tVar22\tVar23\tVar24\tVar25\tVar26\tVar27\tVar28\tVar29\tVar30\tVar31\tVar32\tVar33\tVar34\tVar35\tVar36\tVar37\tVar38\tVar39\tVar40\tVar41\tVar42\tVar43\tVar44\tVar45\tVar46\tVar47\tVar48\tVar49\tVar50\tVar51\tVar52\tVar53\tVar54\tVar55\tVar56\tVar57\tVar58\tVar59\tVar60\tVar61\tVar62\tVar63\tVar64\tVar65\tVar66\tVar67\tVar68\tVar69\tVar70\tVar71\tVar72\tVar73\tVar74\tVar75\tVar76\tVar77\tVar78\tVar79\tVar80\tVar81\tVar82\tVar83\tVar84\tVar85\tVar86\tVar87\tVar88\tVar89\tVar90\tVar91\tVar92\tVar93\tVar94\tVar95\tVar96\tVar97\tVar98\tVar99\tVar100\tVar101\tVar102\tVar103\tVar104\tVar105\tVar106\tVar107\tVar108\tVar109\tVar110\tVar111\tVar112\tVar113\tVar114\tVar115\tVar116\tVar117\tVar118\tVar119\tVar120\tVar121\tVar122\tVar123\tVar124\tVar125\tVar126\tVar127\tVar128\tVar129\tVar130\tVar131\tVar132\tVar133\tVar134\tVar135\tVar136\tVar137\tVar138\tVar139\tVar140\tVar141\tVar142\tVar143\tVar144\tVar145\tVar146\tVar147\tVar148\tVar149\tVar150\tVar151\tVar152\tVar153\tVar154\tVar155\tVar156\tVar157\tVar158\tVar159\tVar160\tVar161\tVar162\tVar163\tVar164\tVar165\tVar166\tVar167\tVar168\tVar169\tVar170\tVar171\tVar172\tVar173\tVar174\tVar175\tVar176\tVar177\tVar178\tVar179\tVar180\tVar181\tVar182\tVar183\tVar184\tVar185\tVar186\tVar187\tVar188\tVar189\tVar190\tVar191\tVar192\tVar193\tVar194\tVar195\tVar196\tVar197\tVar198\tVar199\tVar200\tVar201\tVar202\tVar203\tVar204\tVar205\tVar206\tVar207\tVar208\tVar209\tVar210\tVar211\tVar212\tVar213\tVar214\tVar215\tVar216\tVar217\tVar218\tVar219\tVar220\tVar221\tVar222\tVar223\tVar224\tVar225\tVar226\tVar227\tVar228\tVar229\tVar230\r",
      "\r\n",
      "\t\t\t\t\t1526\t7\t\t\t\t\t\t184\t\t\t\t\t\t\t\t464\t580\t\t14\t128\t\t\t166.56\t\t\t\t\t\t\t0\t\t\t3570\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t4.076907\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t36\t35\t\t1350864\t\t0\t\t\t7333.11\t\t5\t\t12\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t104\t\t\t168\t117625.6\t\t\t\t\t\t1175\t\t\t\t6\t\t720\t8\t\t\t\t\t\t0\t1212385\t69134\t\t\t\t\t\t185\t\t\t0\t9\t\t\t\t\t397579\t\t\t\t1812252\t\t\t\t\t\t\t142\t\t\t38418\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t462\t\t\tbZkvyxLkBI\tRO12\t\ttaul\t1K8T\tlK27\tka_ns41\tnQUveAzAF7\t\t\tdXGu\t9_Y1\tFbIm\tVpdQ\thaYg\tme75fM6ugJ\tkIsH\t\tuKAI\tL84s\tXfqtO3UdzaXh_\t\t\t\tXTbPUYD\tsH5Z\tcJvF\tFzaX\t1YVfGrO\toslk\tfXVEsaq\tjySVZNlOJy\t\t\txb3V\tRAYp\tF2FyR07IdsN7I\t\t\r",
      "\r\n",
      "\t\t\t\t\t525\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t168\t210\t\t2\t24\t\t\t353.52\t\t\t\t\t\t\t0\t\t\t4764966\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t5.408032\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t3\t26\t0\t\t2872928\t\t3\t\t\t151098.9\t\t25\t\t2\t\t\t\t\t\t\t\t\t58158\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t40\t\t\t40\t-356411.6\t\t\t\t\t\t590\t\t\t\t72\t\t0\t\t\t\t\t\t\t8\t4136430\t357038\t\t\t\t\t\t0\t\t\t0\t9\t\t\t\t\t278334\t\t\t\t10439160\t\t\t\t\t\t\t32\t\t\t238572\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\tCEat0G8rTN\tRO12\t\ttaul\t1K8T\t2Ix5\tqEdASpP\ty2LIM01bE1\t\t\tlg1t\t9_Y1\tk13i\tsJzTlal\tzm5i\tme75fM6ugJ\tkIsH\t\tuKAI\tL84s\tNhsEn4L\t\t\t\tkZJyVg2\t\t\tFzaX\t0AJo2f2\toslk\t2Kb5FSF\tLM8l689qOp\t\t\tfKCe\tRAYp\tF2FyR07IdsN7I\t\t\r",
      "\r\n",
      "\t\t\t\t\t5236\t7\t\t\t\t\t\t904\t\t\t\t\t\t\t\t1212\t1515\t\t26\t816\t\t\t220.08\t\t\t\t\t\t\t0\t\t\t5883894\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t6.599658\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t130\t518\t\t1675776\t\t0\t\t\t16211.58\t\t40\t\t58\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t312\t\t\t336\t405104\t\t\t\t\t\t3230\t\t\t\t114\t\t5967\t-28\t\t\t\t\t\t0\t3478905\t248932\t\t\t\t\t\t800\t\t\t0\t36\t\t\t\t\t320565\t\t\t\t9826360\t\t\t\t\t\t\t206\t\t\t434946\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\teOQt0GoOh3\tAERks4l\tSEuy\ttaul\t1K8T\tffXs\tNldASpP\ty4g9XoZ\tvynJTq9\tsmXZ\t4bTR\t9_Y1\tMGOA\tVpdQ\thaYg\tDHn_WUyBhW_whjA88g9bvA64_\tkIsH\t\tuKAI\tL84s\tUbxQ8lZ\t\tTTGHfSv\t\tpMWAe2U\tbHR7\tUYBR\tFzaX\tJFM1BiF\tAl6ZaUT\tNKv4yOc\tjySVZNlOJy\t\tkG3k\tQu4f\t02N6s8f\tib5G6X1eUxUn6\tam7c\t\r",
      "\r\n",
      "\t\t\t\t\t\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t0\t\t\t0\t\t\t22.08\t\t\t\t\t\t\t0\t\t\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t1.98825\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t12\t0\t\t0\t\t0\t\t\t\t\t0\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t0\t-275703.6\t\t\t\t\t\t\t\t\t\t0\t\t0\t-14\t\t\t\t\t\t0\t0\t0\t\t\t\t\t\t0\t\t\t0\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t0\t\t\t0\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\tjg69tYsGvO\tRO12\t\ttaul\t1K8T\tssAy\t_ybO0dd\t4hMlgkf58mhwh\t\t\tW8mQ\t9_Y1\tYULl\tVpdQ\t\tme75fM6ugJ\tkIsH\t\tuKAI\tMtgm\tNhsEn4L\t\t\t\tkq0dQfu\teKej\tUYBR\tFzaX\tL91KIiz\toslk\tCE7uk3u\tLM8l689qOp\t\t\tFSa2\tRAYp\tF2FyR07IdsN7I\t\t\r",
      "\r\n",
      "\t\t\t\t\t1029\t7\t\t\t\t\t\t3216\t\t\t\t\t\t\t\t64\t80\t\t4\t64\t\t\t200\t\t\t\t\t\t\t0\t\t\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t4.552446\t\t\t\t\t\t\t\t18\t\t\t\t\t\t\t3\t82\t224\t\t784448\t\t0\t\t\t37423.5\t\t0\t\t0\t\t\t\t\t\t\t\t\t89754\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t32\t\t\t56\t10714.84\t\t\t\t\t\t215\t\t\t\t0\t\t15111\t58\t\t\t\t\t\t0\t150650\t66046\t\t\t\t\t\t3255\t\t\t0\t9\t\t\t\t\t267162\t\t\t\t644836\t\t\t\t\t\t\t2\t\t\t0\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\tIXSgUHShse\tRO12\tSEuy\ttaul\t1K8T\tuNkU\tEKR938I\tThrHXVS\t0v21jmy\tsmXZ\txklU\t9_Y1\tRVjC\tsJzTlal\t6JmL\tme75fM6ugJ\tkIsH\t\tuKAI\tL84s\tXfqtO3UdzaXh_\t\tSJs3duv\t\t11p4mKe\tH3p7\tUYBR\tFzaX\tOrnLfvc\toslk\t1J2cvxe\tLM8l689qOp\t\tkG3k\tFSa2\tRAYp\tF2FyR07IdsN7I\tmj86\t\r",
      "\r\n",
      "\t\t\t\t\t658\t7\t\t\t\t\t\t3156\t\t\t\t\t\t\t\t224\t280\t\t2\t72\t\t\t200\t\t\t\t\t\t\t5\t\t\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t0.1664174\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t126\t196\t\t209048\t\t0\t\t\t11370.72\t\t0\t\t10\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t32\t\t\t64\t369814\t\t\t\t\t\t755\t\t\t\t0\t\t1935\t\t\t\t\t\t\t8\t641020\t43684\t\t\t\t\t\t355\t\t\t0\t9\t\t\t\t\t2639\t\t\t\t745620\t\t\t\t\t\t\t68\t\t\t61368\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\tm9SrEy7Rm6\t2Knk1KF\t\ttaul\t1K8T\tlK27\tfayYfhR\tetM739XNb0Rf0\t\t\tae6C\t9_Y1\tyrDU\tVpdQ\twMei\tDHn_WUyBhW_whjA88g9bvA64_\tkIsH\t\tuKAI\tL84s\t3vzwTT0wY25GE\t\t\t\tteAHwXo\tykzL\tUYBR\tFzaX\tKbkKEj0\tzCkv\tQqVuch3\tLM8l689qOp\t\t\tQcbd\t02N6s8f\tZy3gnGM\tam7c\t\r",
      "\r\n",
      "\t\t\t\t\t1680\t7\t\t\t\t\t\t2952\t\t\t\t\t\t\t\t308\t385\t\t4\t128\t\t\t176.56\t\t\t\t\t\t\t0\t\t\t13158\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t5.448622\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t114\t392\t\t1030800\t\t0\t\t\t31655.4\t\t5\t\t16\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t88\t\t\t176\t-808528\t\t\t\t\t\t1080\t\t\t\t24\t\t13194\t-24\t\t\t\t\t\t0\t1664450\t104978\t\t\t\t\t\t1665\t\t\t0\t18\t\t\t\t\t47019\t\t\t\t2345424\t\t\t\t\t\t\t52\t\t\t417438\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t228\t\t\tQu0qrQKzJV\t2Knk1KF\tlvza\ttaul\t1K8T\tEJC9\tofiZR7x\tLJF4fPp\tQYxAlFM\tsmXZ\t5Rb0\t9_Y1\t15m3\tVpdQ\thaYg\tme75fM6ugJ\tkIsH\t\tuKAI\tMtgm\tXfqtO3UdzaXh_\t\tRQAGE01\t\t11p4mKe\tNvHl\tUYBR\tFzaX\tJO03372\toslk\tXlgxB9z\tLM8l689qOp\t\tkG3k\tFSa2\tRAYp\t55YFVY9\tam7c\t\r",
      "\r\n",
      "\t\t\t\t\t77\t0\t\t\t\t\t\t0\t\t\t\t\t\t\t\t32\t40\t\t2\t16\t\t\t230.56\t\t\t\t\t\t\t0\t\t\t3776496\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t5.067507\t\t\t\t\t\t\t\t9\t\t\t\t\t\t\t\t18\t0\t\t0\t\t3\t\t\t402441\t\t0\t\t2\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t8\t\t\t0\t101923.6\t\t\t\t\t\t215\t\t\t\t12\t\t0\t-8\t\t\t\t\t\t8\t3839825\t1284128\t\t\t\t\t\t0\t\t\t0\t0\t\t\t\t\t4233600\t\t\t\t10577000\t\t\t\t\t\t\t6\t\t\t3628800\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\teOQgUHShse\tRO12\t\tLfvqpCtLOY\t1K8T\tBxva\tXlthli9\tGaiUdPAZp_\t\t\tMx5G\t9_Y1\tRVjC\tVpdQ\tIYzP\tme75fM6ugJ\tkIsH\t\tuKAI\tMtgm\tNhsEn4L\t\t\t\t7WwuNea\t9haV\tUYBR\t\tU8IKsQe\toslk\tR2LdzOv\t\t\t\tFSa2\tRAYp\tF2FyR07IdsN7I\t\t\r",
      "\r\n",
      "\t\t\t\t\t1176\t7\t\t\t\t\t\t2912\t\t\t\t\t\t\t\t200\t250\t\t2\t64\t\t\t300.32\t\t\t\t\t\t\t0\t\t\t6014460\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t\t\t2.045717\t\t\t\t\t\t\t\t27\t\t\t\t\t\t\t3\t72\t91\t\t1933712\t\t0\t\t\t117761.7\t\t40\t\t2\t\t\t\t\t\t\t\t\t45645\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t56\t\t\t40\t161314.4\t\t\t\t\t\t790\t\t\t\t90\t\t2754\t\t\t\t\t\t\t0\t3830510\t203586\t\t\t\t\t\t3080\t\t\t0\t0\t\t\t\t\t114695\t\t\t\t10405680\t\t\t\t\t\t\t26\t\t\t98310\t\t\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t0\t\t\t\t\t\t\t\t\t\t\t4e7Wq69R_D\tRO12\tSEuy\ttaul\t1K8T\t0Xwj\t6CXYbuk\tokUBQrgaYWgG0\tz1Qe5zX\tsmXZ\t1G9T\tHLqf\tPx52\tVpdQ\tIYzP\tKxdu\tsBgB\t\t7A3j\tL84s\t9pUnzWLbztKTo\t\tyBN8Pcy\t\tkq0YABQ\t\t\tFzaX\tROeipLp\tzCkv\tK2SqEo9\tjySVZNlOJy\t\tkG3k\tPM2D\t6fzt\tam14IcfM7tWLrUmRT52KtA\tam7c\t\r",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!head data/churn/orange_small_train.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   50001 data/churn/orange_small_train.data\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l data/churn/orange_small_train.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sc = ps.SparkContext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# need to get local path since we are reading local files\n",
    "cwd = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "churn_rdd = sc.textFile('file://' + cwd + '/data/churn/small_combined.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "header = churn_rdd.first()\n",
    "\n",
    "# first 190 numeric, last 40 categorical\n",
    "churn_rows = churn_rdd.filter(lambda x: x != header).map(lambda x: x.split('\\t')[:190])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'-1',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'1526',\n",
       " u'7',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'184',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'464',\n",
       " u'580',\n",
       " u'',\n",
       " u'14',\n",
       " u'128',\n",
       " u'',\n",
       " u'',\n",
       " u'166.56',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'',\n",
       " u'',\n",
       " u'3570',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'4.076907',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'9',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'36',\n",
       " u'35',\n",
       " u'',\n",
       " u'1350864',\n",
       " u'',\n",
       " u'0',\n",
       " u'',\n",
       " u'',\n",
       " u'7333.11',\n",
       " u'',\n",
       " u'5',\n",
       " u'',\n",
       " u'12',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'104',\n",
       " u'',\n",
       " u'',\n",
       " u'168',\n",
       " u'117625.6',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'1175',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'6',\n",
       " u'',\n",
       " u'720',\n",
       " u'8',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'1212385',\n",
       " u'69134',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'185',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'9',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'397579',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'1812252',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'142',\n",
       " u'',\n",
       " u'',\n",
       " u'38418',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'0',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'462']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "churn_rows.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n",
    "* convert types\n",
    "* filter out null columns\n",
    "* imputing missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rdd_typed = churn_rows.map(lambda row: dict(zip(header.split('\\t')[:190], \\\n",
    "                                                           map(lambda x: None if x == '' else float(x), row))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'Var1': None,\n",
       " u'Var10': None,\n",
       " u'Var100': None,\n",
       " u'Var101': None,\n",
       " u'Var102': None,\n",
       " u'Var103': None,\n",
       " u'Var104': None,\n",
       " u'Var105': None,\n",
       " u'Var106': None,\n",
       " u'Var107': None,\n",
       " u'Var108': None,\n",
       " u'Var109': 104.0,\n",
       " u'Var11': None,\n",
       " u'Var110': None,\n",
       " u'Var111': None,\n",
       " u'Var112': 168.0,\n",
       " u'Var113': 117625.6,\n",
       " u'Var114': None,\n",
       " u'Var115': None,\n",
       " u'Var116': None,\n",
       " u'Var117': None,\n",
       " u'Var118': None,\n",
       " u'Var119': 1175.0,\n",
       " u'Var12': None,\n",
       " u'Var120': None,\n",
       " u'Var121': None,\n",
       " u'Var122': None,\n",
       " u'Var123': 6.0,\n",
       " u'Var124': None,\n",
       " u'Var125': 720.0,\n",
       " u'Var126': 8.0,\n",
       " u'Var127': None,\n",
       " u'Var128': None,\n",
       " u'Var129': None,\n",
       " u'Var13': 184.0,\n",
       " u'Var130': None,\n",
       " u'Var131': None,\n",
       " u'Var132': 0.0,\n",
       " u'Var133': 1212385.0,\n",
       " u'Var134': 69134.0,\n",
       " u'Var135': None,\n",
       " u'Var136': None,\n",
       " u'Var137': None,\n",
       " u'Var138': None,\n",
       " u'Var139': None,\n",
       " u'Var14': None,\n",
       " u'Var140': 185.0,\n",
       " u'Var141': None,\n",
       " u'Var142': None,\n",
       " u'Var143': 0.0,\n",
       " u'Var144': 9.0,\n",
       " u'Var145': None,\n",
       " u'Var146': None,\n",
       " u'Var147': None,\n",
       " u'Var148': None,\n",
       " u'Var149': 397579.0,\n",
       " u'Var15': None,\n",
       " u'Var150': None,\n",
       " u'Var151': None,\n",
       " u'Var152': None,\n",
       " u'Var153': 1812252.0,\n",
       " u'Var154': None,\n",
       " u'Var155': None,\n",
       " u'Var156': None,\n",
       " u'Var157': None,\n",
       " u'Var158': None,\n",
       " u'Var159': None,\n",
       " u'Var16': None,\n",
       " u'Var160': 142.0,\n",
       " u'Var161': None,\n",
       " u'Var162': None,\n",
       " u'Var163': 38418.0,\n",
       " u'Var164': None,\n",
       " u'Var165': None,\n",
       " u'Var166': None,\n",
       " u'Var167': None,\n",
       " u'Var168': None,\n",
       " u'Var169': None,\n",
       " u'Var17': None,\n",
       " u'Var170': None,\n",
       " u'Var171': None,\n",
       " u'Var172': None,\n",
       " u'Var173': 0.0,\n",
       " u'Var174': None,\n",
       " u'Var175': None,\n",
       " u'Var176': None,\n",
       " u'Var177': None,\n",
       " u'Var178': None,\n",
       " u'Var179': None,\n",
       " u'Var18': None,\n",
       " u'Var180': None,\n",
       " u'Var181': 0.0,\n",
       " u'Var182': None,\n",
       " u'Var183': None,\n",
       " u'Var184': None,\n",
       " u'Var185': None,\n",
       " u'Var186': None,\n",
       " u'Var187': None,\n",
       " u'Var188': None,\n",
       " u'Var189': 462.0,\n",
       " u'Var19': None,\n",
       " u'Var2': None,\n",
       " u'Var20': None,\n",
       " u'Var21': 464.0,\n",
       " u'Var22': 580.0,\n",
       " u'Var23': None,\n",
       " u'Var24': 14.0,\n",
       " u'Var25': 128.0,\n",
       " u'Var26': None,\n",
       " u'Var27': None,\n",
       " u'Var28': 166.56,\n",
       " u'Var29': None,\n",
       " u'Var3': None,\n",
       " u'Var30': None,\n",
       " u'Var31': None,\n",
       " u'Var32': None,\n",
       " u'Var33': None,\n",
       " u'Var34': None,\n",
       " u'Var35': 0.0,\n",
       " u'Var36': None,\n",
       " u'Var37': None,\n",
       " u'Var38': 3570.0,\n",
       " u'Var39': None,\n",
       " u'Var4': None,\n",
       " u'Var40': None,\n",
       " u'Var41': None,\n",
       " u'Var42': None,\n",
       " u'Var43': None,\n",
       " u'Var44': 0.0,\n",
       " u'Var45': None,\n",
       " u'Var46': None,\n",
       " u'Var47': None,\n",
       " u'Var48': None,\n",
       " u'Var49': None,\n",
       " u'Var5': None,\n",
       " u'Var50': None,\n",
       " u'Var51': None,\n",
       " u'Var52': None,\n",
       " u'Var53': None,\n",
       " u'Var54': None,\n",
       " u'Var55': None,\n",
       " u'Var56': None,\n",
       " u'Var57': 4.076907,\n",
       " u'Var58': None,\n",
       " u'Var59': None,\n",
       " u'Var6': 1526.0,\n",
       " u'Var60': None,\n",
       " u'Var61': None,\n",
       " u'Var62': None,\n",
       " u'Var63': None,\n",
       " u'Var64': None,\n",
       " u'Var65': 9.0,\n",
       " u'Var66': None,\n",
       " u'Var67': None,\n",
       " u'Var68': None,\n",
       " u'Var69': None,\n",
       " u'Var7': 7.0,\n",
       " u'Var70': None,\n",
       " u'Var71': None,\n",
       " u'Var72': None,\n",
       " u'Var73': 36.0,\n",
       " u'Var74': 35.0,\n",
       " u'Var75': None,\n",
       " u'Var76': 1350864.0,\n",
       " u'Var77': None,\n",
       " u'Var78': 0.0,\n",
       " u'Var79': None,\n",
       " u'Var8': None,\n",
       " u'Var80': None,\n",
       " u'Var81': 7333.11,\n",
       " u'Var82': None,\n",
       " u'Var83': 5.0,\n",
       " u'Var84': None,\n",
       " u'Var85': 12.0,\n",
       " u'Var86': None,\n",
       " u'Var87': None,\n",
       " u'Var88': None,\n",
       " u'Var89': None,\n",
       " u'Var9': None,\n",
       " u'Var90': None,\n",
       " u'Var91': None,\n",
       " u'Var92': None,\n",
       " u'Var93': None,\n",
       " u'Var94': None,\n",
       " u'Var95': None,\n",
       " u'Var96': None,\n",
       " u'Var97': None,\n",
       " u'Var98': None,\n",
       " u'Var99': None,\n",
       " u'label': -1.0}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd_typed.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "class CounterAccumulatorParam(ps.accumulators.AccumulatorParam):\n",
    "    def zero(self, initialValue):\n",
    "        return Counter()\n",
    "\n",
    "    def addInPlace(self, v1, v2):\n",
    "        v1 += v2\n",
    "        return v1\n",
    "\n",
    "accum = sc.accumulator(Counter(), CounterAccumulatorParam())\n",
    "\n",
    "def count_null(record):\n",
    "    global accum\n",
    "    \n",
    "    c = Counter()\n",
    "    \n",
    "    for key, value in record.items():\n",
    "            # since columns are initially strings\n",
    "            # empty string signals missing value\n",
    "            if value == None:\n",
    "                c[key] += 1\n",
    "                \n",
    "    accum.add(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rdd_typed.foreach(count_null)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({u'Var20': 50000, u'Var185': 50000, u'Var52': 50000, u'Var55': 50000, u'Var48': 50000, u'Var42': 50000, u'Var141': 50000, u'Var79': 50000, u'Var8': 50000, u'Var15': 50000, u'Var167': 50000, u'Var169': 50000, u'Var32': 50000, u'Var31': 50000, u'Var39': 50000, u'Var175': 50000, u'Var118': 49829, u'Var92': 49829, u'Var64': 49762, u'Var45': 49656, u'Var102': 49549, u'Var62': 49442, u'Var98': 49442, u'Var12': 49442, u'Var56': 49354, u'Var89': 49354, u'Var178': 49354, u'Var136': 49306, u'Var63': 49306, u'Var66': 49306, u'Var156': 49306, u'Var116': 49298, u'Var110': 49298, u'Var29': 49298, u'Var186': 49298, u'Var187': 49298, u'Var180': 49298, u'Var121': 49298, u'Var129': 49298, u'Var58': 49298, u'Var50': 49298, u'Var53': 49298, u'Var137': 49298, u'Var131': 49298, u'Var41': 49298, u'Var47': 49298, u'Var77': 49298, u'Var142': 49298, u'Var154': 49298, u'Var1': 49298, u'Var9': 49298, u'Var90': 49298, u'Var86': 49298, u'Var87': 49298, u'Var168': 49298, u'Var100': 49298, u'Var108': 49298, u'Var30': 49298, u'Var115': 49180, u'Var59': 49180, u'Var104': 49180, u'Var105': 49180, u'Var33': 49153, u'Var61': 49153, u'Var151': 49153, u'Var165': 49127, u'Var101': 49127, u'Var158': 49127, u'Var127': 48917, u'Var128': 48917, u'Var88': 48917, u'Var171': 48917, u'Var111': 48871, u'Var71': 48871, u'Var157': 48871, u'Var91': 48871, u'Var130': 48760, u'Var3': 48760, u'Var14': 48760, u'Var11': 48760, u'Var84': 48760, u'Var176': 48760, u'Var114': 48759, u'Var68': 48759, u'Var188': 48759, u'Var184': 48759, u'Var183': 48759, u'Var122': 48759, u'Var54': 48759, u'Var43': 48759, u'Var40': 48759, u'Var46': 48759, u'Var75': 48759, u'Var2': 48759, u'Var95': 48759, u'Var96': 48759, u'Var159': 48759, u'Var49': 48759, u'Var162': 48759, u'Var36': 48759, u'Var34': 48759, u'Var170': 48759, u'Var177': 48759, u'Var26': 48513, u'Var27': 48513, u'Var23': 48513, u'Var120': 48513, u'Var139': 48513, u'Var148': 48513, u'Var70': 48513, u'Var146': 48513, u'Var147': 48513, u'Var60': 48513, u'Var67': 48513, u'Var5': 48513, u'Var97': 48513, u'Var93': 48513, u'Var16': 48513, u'Var10': 48513, u'Var69': 48513, u'Var80': 48513, u'Var166': 48513, u'Var107': 48513, u'Var103': 48513, u'Var172': 48513, u'Var117': 48421, u'Var182': 48421, u'Var124': 48421, u'Var138': 48421, u'Var135': 48421, u'Var145': 48421, u'Var150': 48421, u'Var152': 48421, u'Var155': 48421, u'Var4': 48421, u'Var99': 48421, u'Var17': 48421, u'Var19': 48421, u'Var18': 48421, u'Var82': 48421, u'Var164': 48421, u'Var161': 48421, u'Var106': 48421, u'Var37': 48421, u'Var174': 48421, u'Var179': 48421, u'Var51': 46253, u'Var189': 28978, u'Var72': 22380, u'Var94': 22380, u'Var126': 13920, u'Var24': 7230, u'Var149': 7230, u'Var109': 7230, u'Var125': 5539, u'Var74': 5539, u'Var140': 5539, u'Var65': 5539, u'Var7': 5539, u'Var13': 5539, u'Var119': 5529, u'Var21': 5529, u'Var144': 5529, u'Var6': 5529, u'Var81': 5529, u'Var28': 5011, u'Var112': 5009, u'Var25': 5009, u'Var22': 5009, u'Var181': 5009, u'Var123': 5009, u'Var134': 5009, u'Var133': 5009, u'Var44': 5009, u'Var76': 5009, u'Var143': 5009, u'Var78': 5009, u'Var153': 5009, u'Var85': 5009, u'Var83': 5009, u'Var160': 5009, u'Var163': 5009, u'Var132': 5009, u'Var35': 5009, u'Var38': 5009, u'Var173': 5009})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accum.value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_to_drop = [ k for k, v in accum.value.items() if v > 15000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns_to_keep = set(header.split('\\t')[:190]) - set(columns_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(columns_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import *\n",
    "\n",
    "sqlContext = ps.HiveContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = sqlContext.jsonRDD(rdd_typed.map(lambda row: json.dumps(row)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Var1: double (nullable = true)\n",
      " |-- Var10: double (nullable = true)\n",
      " |-- Var100: double (nullable = true)\n",
      " |-- Var101: double (nullable = true)\n",
      " |-- Var102: double (nullable = true)\n",
      " |-- Var103: double (nullable = true)\n",
      " |-- Var104: double (nullable = true)\n",
      " |-- Var105: double (nullable = true)\n",
      " |-- Var106: double (nullable = true)\n",
      " |-- Var107: double (nullable = true)\n",
      " |-- Var108: double (nullable = true)\n",
      " |-- Var109: double (nullable = true)\n",
      " |-- Var11: double (nullable = true)\n",
      " |-- Var110: double (nullable = true)\n",
      " |-- Var111: double (nullable = true)\n",
      " |-- Var112: double (nullable = true)\n",
      " |-- Var113: double (nullable = true)\n",
      " |-- Var114: double (nullable = true)\n",
      " |-- Var115: double (nullable = true)\n",
      " |-- Var116: double (nullable = true)\n",
      " |-- Var117: double (nullable = true)\n",
      " |-- Var118: double (nullable = true)\n",
      " |-- Var119: double (nullable = true)\n",
      " |-- Var12: double (nullable = true)\n",
      " |-- Var120: double (nullable = true)\n",
      " |-- Var121: double (nullable = true)\n",
      " |-- Var122: double (nullable = true)\n",
      " |-- Var123: double (nullable = true)\n",
      " |-- Var124: double (nullable = true)\n",
      " |-- Var125: double (nullable = true)\n",
      " |-- Var126: double (nullable = true)\n",
      " |-- Var127: double (nullable = true)\n",
      " |-- Var128: double (nullable = true)\n",
      " |-- Var129: double (nullable = true)\n",
      " |-- Var13: double (nullable = true)\n",
      " |-- Var130: double (nullable = true)\n",
      " |-- Var131: double (nullable = true)\n",
      " |-- Var132: double (nullable = true)\n",
      " |-- Var133: double (nullable = true)\n",
      " |-- Var134: double (nullable = true)\n",
      " |-- Var135: double (nullable = true)\n",
      " |-- Var136: double (nullable = true)\n",
      " |-- Var137: double (nullable = true)\n",
      " |-- Var138: double (nullable = true)\n",
      " |-- Var139: double (nullable = true)\n",
      " |-- Var14: double (nullable = true)\n",
      " |-- Var140: double (nullable = true)\n",
      " |-- Var141: string (nullable = true)\n",
      " |-- Var142: double (nullable = true)\n",
      " |-- Var143: double (nullable = true)\n",
      " |-- Var144: double (nullable = true)\n",
      " |-- Var145: double (nullable = true)\n",
      " |-- Var146: double (nullable = true)\n",
      " |-- Var147: double (nullable = true)\n",
      " |-- Var148: double (nullable = true)\n",
      " |-- Var149: double (nullable = true)\n",
      " |-- Var15: string (nullable = true)\n",
      " |-- Var150: double (nullable = true)\n",
      " |-- Var151: double (nullable = true)\n",
      " |-- Var152: double (nullable = true)\n",
      " |-- Var153: double (nullable = true)\n",
      " |-- Var154: double (nullable = true)\n",
      " |-- Var155: double (nullable = true)\n",
      " |-- Var156: double (nullable = true)\n",
      " |-- Var157: double (nullable = true)\n",
      " |-- Var158: double (nullable = true)\n",
      " |-- Var159: double (nullable = true)\n",
      " |-- Var16: double (nullable = true)\n",
      " |-- Var160: double (nullable = true)\n",
      " |-- Var161: double (nullable = true)\n",
      " |-- Var162: double (nullable = true)\n",
      " |-- Var163: double (nullable = true)\n",
      " |-- Var164: double (nullable = true)\n",
      " |-- Var165: double (nullable = true)\n",
      " |-- Var166: double (nullable = true)\n",
      " |-- Var167: string (nullable = true)\n",
      " |-- Var168: double (nullable = true)\n",
      " |-- Var169: string (nullable = true)\n",
      " |-- Var17: double (nullable = true)\n",
      " |-- Var170: double (nullable = true)\n",
      " |-- Var171: double (nullable = true)\n",
      " |-- Var172: double (nullable = true)\n",
      " |-- Var173: double (nullable = true)\n",
      " |-- Var174: double (nullable = true)\n",
      " |-- Var175: string (nullable = true)\n",
      " |-- Var176: double (nullable = true)\n",
      " |-- Var177: double (nullable = true)\n",
      " |-- Var178: double (nullable = true)\n",
      " |-- Var179: double (nullable = true)\n",
      " |-- Var18: double (nullable = true)\n",
      " |-- Var180: double (nullable = true)\n",
      " |-- Var181: double (nullable = true)\n",
      " |-- Var182: double (nullable = true)\n",
      " |-- Var183: double (nullable = true)\n",
      " |-- Var184: double (nullable = true)\n",
      " |-- Var185: string (nullable = true)\n",
      " |-- Var186: double (nullable = true)\n",
      " |-- Var187: double (nullable = true)\n",
      " |-- Var188: double (nullable = true)\n",
      " |-- Var189: double (nullable = true)\n",
      " |-- Var19: double (nullable = true)\n",
      " |-- Var2: double (nullable = true)\n",
      " |-- Var20: string (nullable = true)\n",
      " |-- Var21: double (nullable = true)\n",
      " |-- Var22: double (nullable = true)\n",
      " |-- Var23: double (nullable = true)\n",
      " |-- Var24: double (nullable = true)\n",
      " |-- Var25: double (nullable = true)\n",
      " |-- Var26: double (nullable = true)\n",
      " |-- Var27: double (nullable = true)\n",
      " |-- Var28: double (nullable = true)\n",
      " |-- Var29: double (nullable = true)\n",
      " |-- Var3: double (nullable = true)\n",
      " |-- Var30: double (nullable = true)\n",
      " |-- Var31: string (nullable = true)\n",
      " |-- Var32: string (nullable = true)\n",
      " |-- Var33: double (nullable = true)\n",
      " |-- Var34: double (nullable = true)\n",
      " |-- Var35: double (nullable = true)\n",
      " |-- Var36: double (nullable = true)\n",
      " |-- Var37: double (nullable = true)\n",
      " |-- Var38: double (nullable = true)\n",
      " |-- Var39: string (nullable = true)\n",
      " |-- Var4: double (nullable = true)\n",
      " |-- Var40: double (nullable = true)\n",
      " |-- Var41: double (nullable = true)\n",
      " |-- Var42: string (nullable = true)\n",
      " |-- Var43: double (nullable = true)\n",
      " |-- Var44: double (nullable = true)\n",
      " |-- Var45: double (nullable = true)\n",
      " |-- Var46: double (nullable = true)\n",
      " |-- Var47: double (nullable = true)\n",
      " |-- Var48: string (nullable = true)\n",
      " |-- Var49: double (nullable = true)\n",
      " |-- Var5: double (nullable = true)\n",
      " |-- Var50: double (nullable = true)\n",
      " |-- Var51: double (nullable = true)\n",
      " |-- Var52: string (nullable = true)\n",
      " |-- Var53: double (nullable = true)\n",
      " |-- Var54: double (nullable = true)\n",
      " |-- Var55: string (nullable = true)\n",
      " |-- Var56: double (nullable = true)\n",
      " |-- Var57: double (nullable = true)\n",
      " |-- Var58: double (nullable = true)\n",
      " |-- Var59: double (nullable = true)\n",
      " |-- Var6: double (nullable = true)\n",
      " |-- Var60: double (nullable = true)\n",
      " |-- Var61: double (nullable = true)\n",
      " |-- Var62: double (nullable = true)\n",
      " |-- Var63: double (nullable = true)\n",
      " |-- Var64: double (nullable = true)\n",
      " |-- Var65: double (nullable = true)\n",
      " |-- Var66: double (nullable = true)\n",
      " |-- Var67: double (nullable = true)\n",
      " |-- Var68: double (nullable = true)\n",
      " |-- Var69: double (nullable = true)\n",
      " |-- Var7: double (nullable = true)\n",
      " |-- Var70: double (nullable = true)\n",
      " |-- Var71: double (nullable = true)\n",
      " |-- Var72: double (nullable = true)\n",
      " |-- Var73: double (nullable = true)\n",
      " |-- Var74: double (nullable = true)\n",
      " |-- Var75: double (nullable = true)\n",
      " |-- Var76: double (nullable = true)\n",
      " |-- Var77: double (nullable = true)\n",
      " |-- Var78: double (nullable = true)\n",
      " |-- Var79: string (nullable = true)\n",
      " |-- Var8: string (nullable = true)\n",
      " |-- Var80: double (nullable = true)\n",
      " |-- Var81: double (nullable = true)\n",
      " |-- Var82: double (nullable = true)\n",
      " |-- Var83: double (nullable = true)\n",
      " |-- Var84: double (nullable = true)\n",
      " |-- Var85: double (nullable = true)\n",
      " |-- Var86: double (nullable = true)\n",
      " |-- Var87: double (nullable = true)\n",
      " |-- Var88: double (nullable = true)\n",
      " |-- Var89: double (nullable = true)\n",
      " |-- Var9: double (nullable = true)\n",
      " |-- Var90: double (nullable = true)\n",
      " |-- Var91: double (nullable = true)\n",
      " |-- Var92: double (nullable = true)\n",
      " |-- Var93: double (nullable = true)\n",
      " |-- Var94: double (nullable = true)\n",
      " |-- Var95: double (nullable = true)\n",
      " |-- Var96: double (nullable = true)\n",
      " |-- Var97: double (nullable = true)\n",
      " |-- Var98: double (nullable = true)\n",
      " |-- Var99: double (nullable = true)\n",
      " |-- label: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_subset = df.select(list(columns_to_keep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+---------+------+----------+---------+---------+------+--------+------+-----+------+-----+-----+---------+------+-----+----+------+-----+-----+------+------+------+-----+------+---------+---------+-----+-------+------+------+-------+---------+------+-----+------+------+---------+-----+------+\n",
      "|Var65|   Var113|Var112|    Var153|   Var134|   Var133|Var132|   Var81|Var123|Var74| Var28|Var24|Var25|    Var38| Var21|Var44|Var7|  Var6|Var83|label|Var160| Var22|Var143|Var35|Var181|    Var76|   Var149|Var73| Var125|Var126|Var119| Var140|   Var163|Var144|Var78| Var13|Var109|    Var57|Var85|Var173|\n",
      "+-----+---------+------+----------+---------+---------+------+--------+------+-----+------+-----+-----+---------+------+-----+----+------+-----+-----+------+------+------+-----+------+---------+---------+-----+-------+------+------+-------+---------+------+-----+------+------+---------+-----+------+\n",
      "|  9.0| 117625.6| 168.0| 1812252.0|  69134.0|1212385.0|   0.0| 7333.11|   6.0| 35.0|166.56| 14.0|128.0|   3570.0| 464.0|  0.0| 7.0|1526.0|  5.0| -1.0| 142.0| 580.0|   0.0|  0.0|   0.0|1350864.0| 397579.0| 36.0|  720.0|   8.0|1175.0|  185.0|  38418.0|   9.0|  0.0| 184.0| 104.0| 4.076907| 12.0|   0.0|\n",
      "|  9.0|-356411.6|  40.0|1.043916E7| 357038.0|4136430.0|   8.0|151098.9|  72.0|  0.0|353.52|  2.0| 24.0|4764966.0| 168.0|  0.0| 0.0| 525.0| 25.0| -1.0|  32.0| 210.0|   0.0|  0.0|   0.0|2872928.0| 278334.0| 26.0|    0.0|  null| 590.0|    0.0| 238572.0|   9.0|  3.0|   0.0|  40.0| 5.408032|  2.0|   0.0|\n",
      "|  9.0| 405104.0| 336.0| 9826360.0| 248932.0|3478905.0|   0.0|16211.58| 114.0|518.0|220.08| 26.0|816.0|5883894.0|1212.0|  0.0| 7.0|5236.0| 40.0| -1.0| 206.0|1515.0|   0.0|  0.0|   0.0|1675776.0| 320565.0|130.0| 5967.0| -28.0|3230.0|  800.0| 434946.0|  36.0|  0.0| 904.0| 312.0| 6.599658| 58.0|   0.0|\n",
      "|  9.0|-275703.6|   0.0|       0.0|      0.0|      0.0|   0.0|    null|   0.0|  0.0| 22.08| null|  0.0|      0.0|  null|  0.0| 0.0|  null|  0.0| -1.0|   0.0|   0.0|   0.0|  0.0|   0.0|      0.0|     null| 12.0|    0.0| -14.0|  null|    0.0|      0.0|  null|  0.0|   0.0|  null|  1.98825|  0.0|   0.0|\n",
      "| 18.0| 10714.84|  56.0|  644836.0|  66046.0| 150650.0|   0.0| 37423.5|   0.0|224.0| 200.0|  4.0| 64.0|      0.0|  64.0|  0.0| 7.0|1029.0|  0.0| -1.0|   2.0|  80.0|   0.0|  0.0|   0.0| 784448.0| 267162.0| 82.0|15111.0|  58.0| 215.0| 3255.0|      0.0|   9.0|  0.0|3216.0|  32.0| 4.552446|  0.0|   0.0|\n",
      "|  9.0| 369814.0|  64.0|  745620.0|  43684.0| 641020.0|   8.0|11370.72|   0.0|196.0| 200.0|  2.0| 72.0|      0.0| 224.0|  0.0| 7.0| 658.0|  0.0| -1.0|  68.0| 280.0|   0.0|  5.0|   0.0| 209048.0|   2639.0|126.0| 1935.0|  null| 755.0|  355.0|  61368.0|   9.0|  0.0|3156.0|  32.0|0.1664174| 10.0|   0.0|\n",
      "|  9.0|-808528.0| 176.0| 2345424.0| 104978.0|1664450.0|   0.0| 31655.4|  24.0|392.0|176.56|  4.0|128.0|  13158.0| 308.0|  0.0| 7.0|1680.0|  5.0| -1.0|  52.0| 385.0|   0.0|  0.0|   0.0|1030800.0|  47019.0|114.0|13194.0| -24.0|1080.0| 1665.0| 417438.0|  18.0|  0.0|2952.0|  88.0| 5.448622| 16.0|   0.0|\n",
      "|  9.0| 101923.6|   0.0|  1.0577E7|1284128.0|3839825.0|   8.0|402441.0|  12.0|  0.0|230.56|  2.0| 16.0|3776496.0|  32.0|  0.0| 0.0|  77.0|  0.0| -1.0|   6.0|  40.0|   0.0|  0.0|   0.0|      0.0|4233600.0| 18.0|    0.0|  -8.0| 215.0|    0.0|3628800.0|   0.0|  3.0|   0.0|   8.0| 5.067507|  2.0|   0.0|\n",
      "| 27.0| 161314.4|  40.0|1.040568E7| 203586.0|3830510.0|   0.0|117761.7|  90.0| 91.0|300.32|  2.0| 64.0|6014460.0| 200.0|  0.0| 7.0|1176.0| 40.0| -1.0|  26.0| 250.0|   0.0|  0.0|   0.0|1933712.0| 114695.0| 72.0| 2754.0|  null| 790.0| 3080.0|  98310.0|   0.0|  0.0|2912.0|  56.0| 2.045717|  2.0|   0.0|\n",
      "| 18.0| 842480.0|  32.0| 7608840.0| 210014.0|2577245.0|  32.0|149686.5|  66.0|  0.0|166.56|  2.0| 72.0|5317974.0| 208.0|  0.0| 7.0|1141.0| 10.0| -1.0|  28.0| 260.0|   0.0|  5.0|   0.0|1858088.0| 227892.0|114.0| 6561.0|  null|1065.0|  155.0| 132210.0|  18.0|  6.0| 164.0| 112.0| 6.326853| 24.0|   2.0|\n",
      "|  9.0| -4831.72|  32.0|   28704.0|   7134.0|      0.0|   0.0|2739.141|  12.0|105.0|133.12|  6.0|112.0|   3534.0|  92.0|  0.0| 7.0| 490.0|  0.0| -1.0|   0.0| 115.0|   0.0|  0.0|   0.0|  13048.0|   9128.0|128.0| 5823.0|  58.0| 215.0|   95.0|   8334.0|  18.0|  0.0|2060.0|  56.0| 4.091647|  6.0|   0.0|\n",
      "| 27.0|  67166.4| 120.0|  613124.0|  15166.0| 134105.0|   8.0| 9885.09|  24.0|371.0|133.12|  8.0| 40.0| 446868.0| 236.0|  0.0|14.0| 798.0| 10.0| -1.0|  38.0| 295.0|   0.0|  5.0|   0.0| 147784.0|  50701.0| 76.0|66825.0|  52.0| 295.0|13115.0| 154482.0|   0.0|  0.0|2892.0| 120.0| 4.432173| 28.0|   0.0|\n",
      "|  9.0| -45625.2|   0.0|       0.0|      0.0|      0.0|   0.0| 34917.0|   0.0|  0.0|240.56| null|  0.0|      0.0|   0.0|  0.0| 0.0| 595.0|  0.0| -1.0|   0.0|   0.0|   0.0|  0.0|   0.0|      0.0|     null|148.0|44154.0|  10.0| 205.0|    0.0|      0.0|  27.0|  0.0|2044.0|  null| 1.899594|  0.0|   0.0|\n",
      "|  9.0|-179277.6| 104.0| 5710720.0|1095062.0|2772010.0|   0.0|19991.43|  54.0|  0.0|176.56| 10.0|216.0|  87294.0| 480.0|  0.0| 0.0|2268.0| 25.0| -1.0| 110.0| 600.0|   0.0|  0.0|   0.0| 399032.0| 793065.0| 48.0| 5202.0|  null|1575.0|    0.0|1521276.0|   0.0|  0.0|   0.0| 104.0| 2.524888| 22.0|   0.0|\n",
      "|  9.0| 357684.4|  16.0|1.029736E7|  57596.0|2170355.0|   0.0| 32325.3| 102.0|161.0|236.08|  0.0|  8.0|8521620.0| 148.0|  0.0| 7.0|3633.0| 30.0|  1.0|  22.0| 185.0|   0.0|  0.0|   0.0|1382400.0|      0.0| 34.0|31104.0|   8.0|1325.0|   70.0|      0.0|  18.0|  0.0| 500.0|  32.0| 3.357189|  0.0|   0.0|\n",
      "|  9.0|-2135.144|  16.0|   17016.0|      0.0|      0.0|   0.0|1691.829|   0.0|  0.0|166.56|  0.0|  0.0|      0.0|   8.0|  0.0| 0.0| 259.0|  0.0| -1.0|   0.0|  10.0|   0.0|  0.0|   0.0|  34032.0|      0.0|132.0| 2574.0|  null|  20.0|    0.0|      0.0|   9.0|  0.0| 348.0|  16.0| 1.224525|  0.0|   0.0|\n",
      "|  9.0|  -9852.0| 112.0| 9025240.0| 587392.0|3571845.0|  48.0|25544.49| 186.0|175.0|220.08|  6.0|320.0|3253506.0| 584.0|  0.0| 7.0|5152.0| 75.0| -1.0|  78.0| 730.0|   0.0| 30.0|   0.0|1660392.0|  27881.0|158.0| 8019.0|  null|2470.0| 8495.0|2008362.0|  36.0|  0.0|2024.0|  96.0| 2.873745| 14.0|   0.0|\n",
      "|  9.0|-35706.72| 128.0|  982036.0|  31436.0| 500295.0|   8.0| 30019.2|  30.0|  0.0|166.56|  2.0| 32.0| 548748.0| 168.0|  0.0| 7.0|1449.0|  0.0| -1.0|  34.0| 210.0|   0.0|  5.0|   0.0| 306192.0| 173894.0|140.0| 5319.0|  null| 690.0|  120.0|      0.0|  18.0|  0.0|  28.0|  16.0| 3.308268|  0.0|   0.0|\n",
      "| 18.0|-514956.0|   8.0| 2898024.0|      0.0| 918350.0|   0.0|232930.5|   0.0|  0.0|253.52|  2.0|  0.0|      0.0|  12.0|  0.0| 7.0| 574.0|  0.0| -1.0|   2.0|  15.0|   0.0|  0.0|   0.0|4132128.0| 170240.0| 66.0|13788.0|   4.0| 165.0|    5.0| 145920.0|   9.0|  0.0|1752.0|  16.0| 5.430891|  2.0|   0.0|\n",
      "|  9.0|  64737.2|  24.0| 2807668.0| 392138.0|2055150.0|   0.0| 46263.9|   0.0|  0.0|166.56|  2.0| 96.0|      0.0| 156.0|  0.0| 7.0| 658.0|  0.0| -1.0|  34.0| 195.0|   0.0|  0.0|   0.0| 153672.0|  32249.0| 38.0| 7110.0|  null| 845.0|   45.0| 453654.0|   0.0|  0.0|   0.0|  56.0| 5.402905| 14.0|   0.0|\n",
      "+-----+---------+------+----------+---------+---------+------+--------+------+-----+------+-----+-----+---------+------+-----+----+------+-----+-----+------+------+------+-----+------+---------+---------+-----+-------+------+------+-------+---------+------+-----+------+------+---------+-----+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_subset.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subset.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29675"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subset.dropna().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+----------------+------------------+-----------------+-----------------+------------------+-----------------+-----------------+-------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+--------------------+------------------+------------------+------------------+-----------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+--------------------+\n",
      "|summary|             Var65|             Var113|            Var112|           Var153|           Var134|            Var133|            Var132|             Var81|          Var123|             Var74|            Var28|            Var24|             Var25|            Var38|            Var21|              Var44|             Var7|             Var6|             Var83|             label|           Var160|            Var22|              Var143|             Var35|            Var181|             Var76|           Var149|            Var73|           Var125|            Var126|            Var119|           Var140|            Var163|            Var144|             Var78|             Var13|            Var109|             Var57|             Var85|              Var173|\n",
      "+-------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+----------------+------------------+-----------------+-----------------+------------------+-----------------+-----------------+-------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+--------------------+------------------+------------------+------------------+-----------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+--------------------+\n",
      "|  count|             44461|              50000|             44991|            44991|            44991|             44991|             44991|             44471|           44991|             44461|            44989|            42770|             44991|            44991|            44471|              44991|            44461|            44471|             44991|             50000|            44991|            44991|               44991|             44991|             44991|             44991|            42770|            50000|            44461|             36080|             44471|            44461|             44991|             44471|             44991|             44461|             42770|             50000|             44991|               44991|\n",
      "|   mean|14.868896336114798|-153278.60848674004| 66.22106643550933|6181967.170056233|437340.3848769754|2273571.9194950103|3.5246160343179747|103084.05269283189|60.1880376075215|103.65812734756304|224.5076691635697|4.507926116436755| 96.82700984641373|2579106.927518837|234.5182253603472|0.16683336667333468|6.809495962753874|1326.437116322997|20.023560267609078|          -0.85272|38.80300504545354|290.2453824098153|0.058011602320464095|0.7168100286724012| 0.611455624458225|1490153.8366339935|294920.8042553192|         66.64108|27887.62517712152|-0.553880266075388| 916.1121854691821|1381.259643282877|486077.95665799826|11.727665220031032|0.5347069413882777|1249.6884010706012| 60.88866027589432|3.5123114701969853| 8.461025538441021|0.006845813607165...|\n",
      "| stddev|10.134327331736102|  761365.3695940942|157.63591913155358|4348877.763364088| 604334.753868816| 2438572.485511682|  9.99476701529489|106270.93869067538|221.548840273776|  766.703355566123| 98.5191453387728|9.928702830121349|214.31590153937137|3010043.004255119|565.5537877935196| 1.6293259756923721|6.325981976755302|2685.663472104636| 88.12848255574919|0.5223682624356115|99.49604349450925|704.4821604398131|   0.643053676353341| 2.995973814257233|2.4956536011110058|1853672.8684453683|656886.9466036924|52.85864145467229|90127.36785162108| 22.53178295782185|2165.4088085123512|3990.465644888673| 848854.3752908827|11.720156701794112|2.1353354773088835| 2794.923441824558|141.24755079494338| 2.025861574298318|20.620255501550034| 0.13250153033152776|\n",
      "|    min|               9.0|         -9803600.0|               0.0|              0.0|              0.0|               0.0|               0.0|               0.0|             0.0|               0.0|           -66.88|              0.0|               0.0|              0.0|              0.0|                0.0|              0.0|              0.0|               0.0|              -1.0|              0.0|              0.0|                 0.0|               0.0|               0.0|               0.0|              0.0|              4.0|              0.0|             -32.0|               0.0|              0.0|               0.0|               0.0|               0.0|               0.0|               0.0|       2.136296E-4|               0.0|                 0.0|\n",
      "|    max|             180.0|          9932480.0|           10352.0|        1.39078E7|        5735340.0|         1.50099E7|             184.0|         1814403.0|         13086.0|          142156.0|          5158.56|            494.0|           13168.0|        1.88469E7|          36272.0|              135.0|            140.0|         131761.0|            6335.0|               1.0|           4862.0|          45340.0|                18.0|             110.0|              49.0|         1.93536E7|        1.69344E7|            264.0|        5436045.0|              68.0|          105060.0|         520545.0|         1.45152E7|              81.0|              39.0|          197872.0|            7456.0|               7.0|            1178.0|                 6.0|\n",
      "+-------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+------------------+----------------+------------------+-----------------+-----------------+------------------+-----------------+-----------------+-------------------+-----------------+-----------------+------------------+------------------+-----------------+-----------------+--------------------+------------------+------------------+------------------+-----------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+------------------+------------------+------------------+------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "summary = df_subset.describe()\n",
    "summary.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "means_df = summary.toPandas().set_index('summary').loc['mean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Var65        14.868896336114798\n",
       "Var113      -153278.60848674004\n",
       "Var112        66.22106643550933\n",
       "Var153        6181967.170056233\n",
       "Var134        437340.3848769754\n",
       "Var133       2273571.9194950103\n",
       "Var132       3.5246160343179747\n",
       "Var81        103084.05269283189\n",
       "Var123         60.1880376075215\n",
       "Var74        103.65812734756304\n",
       "Var28         224.5076691635697\n",
       "Var24         4.507926116436755\n",
       "Var25         96.82700984641373\n",
       "Var38         2579106.927518837\n",
       "Var21         234.5182253603472\n",
       "Var44       0.16683336667333468\n",
       "Var7          6.809495962753874\n",
       "Var6          1326.437116322997\n",
       "Var83        20.023560267609078\n",
       "label                  -0.85272\n",
       "Var160        38.80300504545354\n",
       "Var22         290.2453824098153\n",
       "Var143     0.058011602320464095\n",
       "Var35        0.7168100286724012\n",
       "Var181        0.611455624458225\n",
       "Var76        1490153.8366339935\n",
       "Var149        294920.8042553192\n",
       "Var73                  66.64108\n",
       "Var125        27887.62517712152\n",
       "Var126       -0.553880266075388\n",
       "Var119        916.1121854691821\n",
       "Var140        1381.259643282877\n",
       "Var163       486077.95665799826\n",
       "Var144       11.727665220031032\n",
       "Var78        0.5347069413882777\n",
       "Var13        1249.6884010706012\n",
       "Var109        60.88866027589432\n",
       "Var57        3.5123114701969853\n",
       "Var85         8.461025538441021\n",
       "Var173    0.0068458136071658775\n",
       "Name: mean, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "means_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imputed = df_subset.na.fill(dict(means_df.apply(float)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Var65=9.0, Var113=117625.6, Var112=168.0, Var153=1812252.0, Var134=69134.0, Var133=1212385.0, Var132=0.0, Var81=7333.11, Var123=6.0, Var74=35.0, Var28=166.56, Var24=14.0, Var25=128.0, Var38=3570.0, Var21=464.0, Var44=0.0, Var7=7.0, Var6=1526.0, Var83=5.0, label=-1.0, Var160=142.0, Var22=580.0, Var143=0.0, Var35=0.0, Var181=0.0, Var76=1350864.0, Var149=397579.0, Var73=36.0, Var125=720.0, Var126=8.0, Var119=1175.0, Var140=185.0, Var163=38418.0, Var144=9.0, Var78=0.0, Var13=184.0, Var109=104.0, Var57=4.076907, Var85=12.0, Var173=0.0),\n",
       " Row(Var65=9.0, Var113=-356411.6, Var112=40.0, Var153=10439160.0, Var134=357038.0, Var133=4136430.0, Var132=8.0, Var81=151098.9, Var123=72.0, Var74=0.0, Var28=353.52, Var24=2.0, Var25=24.0, Var38=4764966.0, Var21=168.0, Var44=0.0, Var7=0.0, Var6=525.0, Var83=25.0, label=-1.0, Var160=32.0, Var22=210.0, Var143=0.0, Var35=0.0, Var181=0.0, Var76=2872928.0, Var149=278334.0, Var73=26.0, Var125=0.0, Var126=-0.553880266075, Var119=590.0, Var140=0.0, Var163=238572.0, Var144=9.0, Var78=3.0, Var13=0.0, Var109=40.0, Var57=5.408032, Var85=2.0, Var173=0.0),\n",
       " Row(Var65=9.0, Var113=405104.0, Var112=336.0, Var153=9826360.0, Var134=248932.0, Var133=3478905.0, Var132=0.0, Var81=16211.58, Var123=114.0, Var74=518.0, Var28=220.08, Var24=26.0, Var25=816.0, Var38=5883894.0, Var21=1212.0, Var44=0.0, Var7=7.0, Var6=5236.0, Var83=40.0, label=-1.0, Var160=206.0, Var22=1515.0, Var143=0.0, Var35=0.0, Var181=0.0, Var76=1675776.0, Var149=320565.0, Var73=130.0, Var125=5967.0, Var126=-28.0, Var119=3230.0, Var140=800.0, Var163=434946.0, Var144=36.0, Var78=0.0, Var13=904.0, Var109=312.0, Var57=6.599658, Var85=58.0, Var173=0.0),\n",
       " Row(Var65=9.0, Var113=-275703.6, Var112=0.0, Var153=0.0, Var134=0.0, Var133=0.0, Var132=0.0, Var81=103084.052693, Var123=0.0, Var74=0.0, Var28=22.08, Var24=4.50792611644, Var25=0.0, Var38=0.0, Var21=234.51822536, Var44=0.0, Var7=0.0, Var6=1326.43711632, Var83=0.0, label=-1.0, Var160=0.0, Var22=0.0, Var143=0.0, Var35=0.0, Var181=0.0, Var76=0.0, Var149=294920.804255, Var73=12.0, Var125=0.0, Var126=-14.0, Var119=916.112185469, Var140=0.0, Var163=0.0, Var144=11.72766522, Var78=0.0, Var13=0.0, Var109=60.8886602759, Var57=1.98825, Var85=0.0, Var173=0.0),\n",
       " Row(Var65=18.0, Var113=10714.84, Var112=56.0, Var153=644836.0, Var134=66046.0, Var133=150650.0, Var132=0.0, Var81=37423.5, Var123=0.0, Var74=224.0, Var28=200.0, Var24=4.0, Var25=64.0, Var38=0.0, Var21=64.0, Var44=0.0, Var7=7.0, Var6=1029.0, Var83=0.0, label=-1.0, Var160=2.0, Var22=80.0, Var143=0.0, Var35=0.0, Var181=0.0, Var76=784448.0, Var149=267162.0, Var73=82.0, Var125=15111.0, Var126=58.0, Var119=215.0, Var140=3255.0, Var163=0.0, Var144=9.0, Var78=0.0, Var13=3216.0, Var109=32.0, Var57=4.552446, Var85=0.0, Var173=0.0)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputed.take(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert to MLlib Datatypes\n",
    "* MLlib: LabeledPoint\n",
    "* `spark.ml`: Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.regression import LabeledPoint\n",
    "from pyspark.mllib.linalg import Vectors\n",
    "\n",
    "def create_points(row):\n",
    "    label = 0.0 if row.label == -1.0 else 1.0\n",
    "    \n",
    "    f_dict = row.asDict()\n",
    "    f_dict.pop('label')\n",
    "    \n",
    "    # make sure the features are in the same order for each data point\n",
    "    features = [ x[1] for x in sorted(f_dict.items())]\n",
    "    \n",
    "    return LabeledPoint(label, Vectors.dense(features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_rdd = imputed.rdd.map(create_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabeledPoint(0.0, [104.0,168.0,117625.6,1175.0,6.0,720.0,8.0,184.0,0.0,1212385.0,69134.0,185.0,0.0,9.0,397579.0,1812252.0,142.0,38418.0,0.0,0.0,464.0,580.0,14.0,128.0,166.56,0.0,3570.0,0.0,4.076907,1526.0,9.0,7.0,36.0,35.0,1350864.0,0.0,7333.11,5.0,12.0])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_rdd.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building a Model with MLlib\n",
    "\n",
    "* test/train split\n",
    "* preprocessing of features\n",
    "* training\n",
    "* evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(train_set, test_set) = feature_rdd.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.classification import LogisticRegressionWithLBFGS, LogisticRegressionModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lrmodel = LogisticRegressionWithLBFGS.train(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.mllib.classification.LogisticRegressionModel"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(lrmodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions  = lrmodel.predict(test_set.map(lambda x: x.features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scores_and_labels = predictions.zip(test_set.map(lambda x: x.label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_err = scores_and_labels.filter(lambda (p, v): v != p).count() / float(test_set.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0.0773821364631\n",
      "Accuracy: 92.2617863537\n"
     ]
    }
   ],
   "source": [
    "print 'Test Error = ' + str(test_err)\n",
    "print \"Accuracy: \" + str((1.0 - test_err) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0, 37039), (1.0, 2907)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.groupBy(lambda x: x.label).mapValues(lambda x: len(x)).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08004418460543686"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2971 / float(37117)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(weights=[-0.000573440103321,0.00027153527539,4.92166700961e-07,-1.36340753014e-05,-1.39969569402e-05,6.00025392036e-08,-0.0337561753727,1.91601306692e-05,-0.00607055384322,-2.06055116386e-08,-1.17046708854e-07,3.44943501257e-06,0.0649222835728,-0.013043144626,8.95194586702e-09,-1.76131602215e-08,-0.000417265358534,1.75779617913e-08,-0.0170142459708,-0.00724631433367,-0.000120079496378,1.23462912376e-05,-0.00391320810314,0.000401922811545,-0.00756802113985,-0.00447888246675,-2.51604738882e-08,-0.0138202890335,-0.0737179786081,1.26566495678e-05,-0.0123837204647,0.00712275822164,5.52724788092e-05,-1.03466870528e-05,-3.21265041211e-10,-0.0405041875314,-2.09495720522e-06,-0.000291517490204,0.00270145995009], intercept=0.0)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([-0.0006, 0.0003, 0.0, -0.0, -0.0, 0.0, -0.0338, 0.0, -0.0061, -0.0, -0.0, 0.0, 0.0649, -0.013, 0.0, -0.0, -0.0004, 0.0, -0.017, -0.0072, -0.0001, 0.0, -0.0039, 0.0004, -0.0076, -0.0045, -0.0, -0.0138, -0.0737, 0.0, -0.0124, 0.0071, 0.0001, -0.0, -0.0, -0.0405, -0.0, -0.0003, 0.0027])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrmodel.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrmodel.threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrmodel.clearThreshold()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrmodel.threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.10774174679968923,\n",
       " 0.016218991336040673,\n",
       " 0.17232673929508993,\n",
       " 0.03424143184711809,\n",
       " 0.054548622927502545,\n",
       " 0.11734345322199889,\n",
       " 0.17445255382820501,\n",
       " 0.14780086614278903,\n",
       " 0.007416738229005967,\n",
       " 0.10117309228908833,\n",
       " 0.11759976552394834,\n",
       " 0.02582705714409853,\n",
       " 0.06619832419496444,\n",
       " 0.046096400771087384,\n",
       " 0.07083667481349189,\n",
       " 0.025904529026136262,\n",
       " 0.0012388046748188842,\n",
       " 0.031840136253872976,\n",
       " 0.07576045988901611,\n",
       " 0.03228555447848915,\n",
       " 0.020023342514966836,\n",
       " 0.4480777460713189,\n",
       " 0.033565510362292425,\n",
       " 0.23925237838264596,\n",
       " 0.03612354173176688,\n",
       " 0.04291898113781928,\n",
       " 0.0867459874610687,\n",
       " 0.1401031995735476,\n",
       " 0.07178901777913832,\n",
       " 0.03993625642555073,\n",
       " 0.043310912708890544,\n",
       " 0.01477579841467171,\n",
       " 0.10609560299163331,\n",
       " 0.01471823969371571,\n",
       " 0.06806511637169647,\n",
       " 0.022489020043764435,\n",
       " 0.06467074000133816,\n",
       " 0.04327131879761095,\n",
       " 0.07413271362708809,\n",
       " 0.30545182748087224,\n",
       " 0.0703230942278116,\n",
       " 0.060607788564793416,\n",
       " 0.2946017697800426,\n",
       " 0.0062540803388741115,\n",
       " 0.03121353660102567,\n",
       " 0.010813288324984878,\n",
       " 0.01954209368701615,\n",
       " 0.18576550076944465,\n",
       " 0.019003259138153292,\n",
       " 0.10415626148258955,\n",
       " 0.04566735611585993,\n",
       " 0.027320247728918765,\n",
       " 0.02088949998347399,\n",
       " 0.02036359643179458,\n",
       " 0.18163081253714752,\n",
       " 0.0952831943313234,\n",
       " 0.20904856894023185,\n",
       " 0.044707700588490884,\n",
       " 0.1284900735141519,\n",
       " 0.05765534796387389,\n",
       " 0.06948004177068597,\n",
       " 0.09047295622854822,\n",
       " 0.0889802753342931,\n",
       " 0.08439470854828748,\n",
       " 0.16476666134938756,\n",
       " 0.06577423721338799,\n",
       " 0.07942156153813143,\n",
       " 0.09578223451335635,\n",
       " 0.03253285281616956,\n",
       " 0.05604544644388006,\n",
       " 0.06462427170382523,\n",
       " 0.11549553540967233,\n",
       " 0.012391833012158536,\n",
       " 0.012618959533142302,\n",
       " 0.022954039731026124,\n",
       " 0.10154334740479973,\n",
       " 0.06564035264487983,\n",
       " 0.019899354180014373,\n",
       " 0.043230061535330454,\n",
       " 0.10241068714921814,\n",
       " 0.021677192352408847,\n",
       " 0.01174599016335525,\n",
       " 0.12182635864606267,\n",
       " 0.09317043249506474,\n",
       " 0.042257070635299765,\n",
       " 0.13766818053491306,\n",
       " 0.01991104111992192,\n",
       " 0.043011485090751964,\n",
       " 0.005666326239133916,\n",
       " 0.07127835177490052,\n",
       " 0.04966034713868294,\n",
       " 0.03221776552669212,\n",
       " 0.015192994194698963,\n",
       " 0.030550903798437008,\n",
       " 0.08851192699019948,\n",
       " 0.06110355007654552,\n",
       " 0.17969051193806485,\n",
       " 0.058552536474064935,\n",
       " 0.12725449647570264,\n",
       " 0.05917677060733096,\n",
       " 0.034665590974195165,\n",
       " 0.04485790776665412,\n",
       " 0.06897322245407488,\n",
       " 0.05248879145396179,\n",
       " 0.0680167132950926,\n",
       " 0.036187416610146766,\n",
       " 0.25158024600347756,\n",
       " 0.04203284886481958,\n",
       " 0.43334642241877064,\n",
       " 0.0312436523227141,\n",
       " 0.0003291820133538752,\n",
       " 0.10474315420669993,\n",
       " 0.011948689443455764,\n",
       " 0.0057631864646685215,\n",
       " 0.15300443000492014,\n",
       " 0.06656886108921398,\n",
       " 0.06969241674330637,\n",
       " 0.09632900397679908,\n",
       " 0.11686769287729985,\n",
       " 0.018465931117468443,\n",
       " 0.016147375027450585,\n",
       " 0.12358059568036857,\n",
       " 0.05934733178293837,\n",
       " 0.052041296964705754,\n",
       " 0.11981269564196001,\n",
       " 0.05677129950203057,\n",
       " 0.14943836871665975,\n",
       " 0.15318759738489637,\n",
       " 0.10614089925618335,\n",
       " 0.14987328836118105,\n",
       " 0.04778538395345786,\n",
       " 0.08707337136389684,\n",
       " 0.04134881725349381,\n",
       " 0.11483819761413945,\n",
       " 0.06268089783967921,\n",
       " 0.03693372427995603,\n",
       " 0.12187385605432663,\n",
       " 0.19512741685014723,\n",
       " 0.02549614891885339,\n",
       " 0.01636682683360797,\n",
       " 0.11302281010578691,\n",
       " 0.007423823899844775,\n",
       " 0.19757464777741504,\n",
       " 0.10245769980783268,\n",
       " 0.03711129547649885,\n",
       " 0.03282835468266463,\n",
       " 0.04326830863417754,\n",
       " 0.03854830304885268,\n",
       " 0.10181139069511683,\n",
       " 0.014480173742973133,\n",
       " 0.04941140024079482,\n",
       " 0.079857545560848,\n",
       " 0.061222293712107,\n",
       " 0.07627470076266578,\n",
       " 0.05400240676235792,\n",
       " 0.06866072238602847,\n",
       " 0.1064612006273841,\n",
       " 0.1034003586804362,\n",
       " 0.013524072280671883,\n",
       " 0.08439513761861177,\n",
       " 0.07808780408396222,\n",
       " 0.01833082457147249,\n",
       " 0.04326325885556276,\n",
       " 0.03237886662069371,\n",
       " 0.16140726614181322,\n",
       " 0.18684964348379637,\n",
       " 0.04233087855180256,\n",
       " 0.03685383283534628,\n",
       " 0.09259483660460871,\n",
       " 0.03512310164038483,\n",
       " 0.030245319731751846,\n",
       " 0.03730080555390815,\n",
       " 0.06371591334132584,\n",
       " 0.08216284186233863,\n",
       " 0.08535343024974781,\n",
       " 0.03691815063574965,\n",
       " 0.15661263297856956,\n",
       " 0.07760258175390704,\n",
       " 0.04997442574822834,\n",
       " 0.055639969066954266,\n",
       " 0.09382450300294687,\n",
       " 0.04534044171618019,\n",
       " 0.06392478155132042,\n",
       " 0.05008322387779222,\n",
       " 0.11008909073085808,\n",
       " 0.02368816518890868,\n",
       " 0.1417306278451952,\n",
       " 0.07569062195800692,\n",
       " 0.02654647024519502,\n",
       " 0.10801423731209649,\n",
       " 0.011455308031295132,\n",
       " 0.012684311920570627,\n",
       " 0.28697907830582914,\n",
       " 0.04961887242908668,\n",
       " 0.04167580004725822,\n",
       " 0.10049350333910437,\n",
       " 0.02677504078941047,\n",
       " 0.056240388956473285,\n",
       " 0.04608365254310088,\n",
       " 0.029257495343244094,\n",
       " 0.08264364580990284,\n",
       " 0.11261074639869348,\n",
       " 0.0826285362189191,\n",
       " 0.05744131296265975,\n",
       " 0.02112592829702271,\n",
       " 0.021322423133371862,\n",
       " 0.02518974223954956,\n",
       " 0.11442571253027883,\n",
       " 0.04443432997410871,\n",
       " 0.055892798934995984,\n",
       " 0.09210821108198461,\n",
       " 0.0594673485044949,\n",
       " 0.026345974248835273,\n",
       " 0.08777154200588878,\n",
       " 0.07941967858516427,\n",
       " 0.13519025908962984,\n",
       " 0.024100669970192327,\n",
       " 0.028176499677601025,\n",
       " 0.052177005192331774,\n",
       " 0.23934253696297525,\n",
       " 0.05234528464758732,\n",
       " 0.03469988240690718,\n",
       " 0.14552541816300424,\n",
       " 0.04514406067074793,\n",
       " 0.07793160482580223,\n",
       " 0.0901732064024441,\n",
       " 0.05171824320145276,\n",
       " 0.10447178853188024,\n",
       " 0.10485843998207742,\n",
       " 0.09550719418109559,\n",
       " 0.16404677806780396,\n",
       " 0.08713868154960276,\n",
       " 0.05588127491672539,\n",
       " 0.07059641804423764,\n",
       " 0.06657039931643398,\n",
       " 0.02783090308028681,\n",
       " 0.06670533711360345,\n",
       " 0.07665670396781973,\n",
       " 0.09728821962613085,\n",
       " 0.05856619877325468,\n",
       " 0.03864977389091644,\n",
       " 0.0037610045521943676,\n",
       " 0.020181943851555435,\n",
       " 0.12844629521555256,\n",
       " 0.02474209333384463,\n",
       " 0.10950493408470574,\n",
       " 0.058705920640418366,\n",
       " 0.06979818358648912,\n",
       " 0.05317412012768782,\n",
       " 0.06242849475114786,\n",
       " 0.10196271831061954,\n",
       " 0.149480761588576,\n",
       " 0.16485286958302012,\n",
       " 0.017580933071640974,\n",
       " 0.05759115755235167,\n",
       " 0.02260263550168413,\n",
       " 0.14739281045973412,\n",
       " 0.1020246025289867,\n",
       " 0.09134622669389922,\n",
       " 0.06798315108682275,\n",
       " 0.014575698735946897,\n",
       " 0.11899347110945419,\n",
       " 0.05380688261821893,\n",
       " 0.08046940450354002,\n",
       " 0.20418400707536832,\n",
       " 0.04478400156517699,\n",
       " 0.21391021743152253,\n",
       " 0.08874581582090603,\n",
       " 0.07187907286136465,\n",
       " 0.055226463640131264,\n",
       " 0.05914652666778114,\n",
       " 0.038660697577811744,\n",
       " 0.1283215417339723,\n",
       " 0.04266817842064381,\n",
       " 0.04264956533499402,\n",
       " 0.041795505696903955,\n",
       " 0.08086674035793527,\n",
       " 0.0875561148703423,\n",
       " 0.12439783413530552,\n",
       " 0.09523799241385696,\n",
       " 0.011306596782443749,\n",
       " 0.08929135297776451,\n",
       " 0.042215396902547934,\n",
       " 0.20555705231880436,\n",
       " 0.03681912988561179,\n",
       " 0.11643646953899511,\n",
       " 0.16376357638649422,\n",
       " 0.04693653668835548,\n",
       " 0.06632810008093187,\n",
       " 0.06732056271581054,\n",
       " 0.16786821730083923,\n",
       " 0.07543139626916572,\n",
       " 0.08343583262024702,\n",
       " 0.045767442692986784,\n",
       " 0.06964526614340404,\n",
       " 0.11808342529540194,\n",
       " 0.03528156052892072,\n",
       " 0.1852604632655488,\n",
       " 0.010513930102193194,\n",
       " 0.06820466915880702,\n",
       " 0.04609922200588672,\n",
       " 0.041750639867590714,\n",
       " 0.038960098333843465,\n",
       " 0.08054535234260213,\n",
       " 0.11953377234541886,\n",
       " 0.09178808039066509,\n",
       " 0.1351176718803896,\n",
       " 0.11298039207403772,\n",
       " 0.05427971348942702,\n",
       " 0.058896014338286506,\n",
       " 0.045481146483341416,\n",
       " 0.03185087191254831,\n",
       " 0.14171514607748417,\n",
       " 0.010829035358721357,\n",
       " 0.19422634164716687,\n",
       " 0.03751208796807978,\n",
       " 0.06505917601129521,\n",
       " 0.04931763848480393,\n",
       " 0.0024640805133511363,\n",
       " 0.05995503171845001,\n",
       " 0.04603480402761096,\n",
       " 0.10133361797852047,\n",
       " 0.06854827947717687,\n",
       " 0.13090784734024874,\n",
       " 0.06876546783242368,\n",
       " 0.0882242781674475,\n",
       " 0.03780221947326492,\n",
       " 0.049947646681739066,\n",
       " 0.17709792180117076,\n",
       " 0.0545117079444125,\n",
       " 0.09819712107968301,\n",
       " 0.018241891849865643,\n",
       " 0.020627832950106992,\n",
       " 0.051935404656118754,\n",
       " 0.1219184848693664,\n",
       " 0.059380020261961326,\n",
       " 0.03710958554867159,\n",
       " 0.354379836546472,\n",
       " 0.1220153251181271,\n",
       " 0.14054720786969424,\n",
       " 0.05874060760862115,\n",
       " 0.1176962230360608,\n",
       " 0.0515806865049033,\n",
       " 0.049820814331611556,\n",
       " 0.10506359371000158,\n",
       " 0.03626650507910029,\n",
       " 0.03476009140540353,\n",
       " 0.021621882262896696,\n",
       " 0.10378860560682573,\n",
       " 0.09704605183776868,\n",
       " 0.1032667697395646,\n",
       " 0.06436147674707585,\n",
       " 0.06469556499653978,\n",
       " 0.025090344635043536,\n",
       " 0.0461188958704596,\n",
       " 0.07089030957668785,\n",
       " 0.0928240876182239,\n",
       " 0.052008818406150646,\n",
       " 0.06632452997087619,\n",
       " 0.07106991258797753,\n",
       " 0.062206178065172146,\n",
       " 0.18885645564710563,\n",
       " 0.04243542016262354,\n",
       " 0.1436927713560711,\n",
       " 0.06885184186250902,\n",
       " 0.07080188714818261,\n",
       " 0.06597491395858958,\n",
       " 0.06774762062711315,\n",
       " 0.01330898538935408,\n",
       " 0.13840037901446553,\n",
       " 0.08278574290413634,\n",
       " 0.07781836099164467,\n",
       " 0.05600814651793026,\n",
       " 0.14199979248279315,\n",
       " 0.20987518475362266,\n",
       " 0.006669257473990879,\n",
       " 0.024251217122006796,\n",
       " 0.05027217297302213,\n",
       " 0.06545629937993856,\n",
       " 0.029782235367698102,\n",
       " 0.06431666587193437,\n",
       " 0.05594271016970452,\n",
       " 0.031400681839072256,\n",
       " 0.12694449575011427,\n",
       " 0.04572408449257294,\n",
       " 0.2323819798601482,\n",
       " 0.00013075236249334655,\n",
       " 0.03447009229544777,\n",
       " 0.08224542606567622,\n",
       " 0.006596127796785262,\n",
       " 0.023255496226128238,\n",
       " 0.30335382653684073,\n",
       " 0.07284319405098513,\n",
       " 0.06906846852633491,\n",
       " 0.17023814466700654,\n",
       " 0.030925762604381662,\n",
       " 0.18406852477008834,\n",
       " 0.019777500130507723,\n",
       " 0.13661081244500659,\n",
       " 0.05516826319307715,\n",
       " 0.05962113917659166,\n",
       " 0.16648516171708333,\n",
       " 0.016188637035418722,\n",
       " 0.16899553043557322,\n",
       " 0.1374596289873913,\n",
       " 0.12334108849288405,\n",
       " 0.03605751759096101,\n",
       " 0.008499417336670319,\n",
       " 0.0102584977012195,\n",
       " 0.1255169010146022,\n",
       " 0.11204299138856504,\n",
       " 0.1656942775879732,\n",
       " 0.19362717799951176,\n",
       " 0.20162318380569552,\n",
       " 0.055298702700950955,\n",
       " 0.08281327808754947,\n",
       " 0.03788667094283396,\n",
       " 0.05540462306568068,\n",
       " 0.060515433854084665,\n",
       " 0.07454491908214865,\n",
       " 0.12621688693028962,\n",
       " 0.13017089876694585,\n",
       " 0.0696688501027885,\n",
       " 0.03179619348727521,\n",
       " 0.01044245021896016,\n",
       " 0.0631069245907608,\n",
       " 0.0985841052412939,\n",
       " 0.00637904164190563,\n",
       " 0.13725920854192958,\n",
       " 0.1180232422418211,\n",
       " 0.2971060149935985,\n",
       " 0.0529563130817631,\n",
       " 0.012332855426281512,\n",
       " 0.02965238102179647,\n",
       " 0.10508385231678985,\n",
       " 0.014817584815017854,\n",
       " 0.07857426539110597,\n",
       " 0.052017146971661855,\n",
       " 0.03383281236356536,\n",
       " 0.019506276745297042,\n",
       " 0.053596205267210185,\n",
       " 0.27990025379561195,\n",
       " 0.040514096460671,\n",
       " 0.12400917987681816,\n",
       " 0.04480003117064839,\n",
       " 0.031467553279896476,\n",
       " 0.033762438777968466,\n",
       " 0.06306217068185595,\n",
       " 0.01749845048265114,\n",
       " 0.07063639344984797,\n",
       " 0.10863192342528244,\n",
       " 0.021091646094841758,\n",
       " 0.11943038189528292,\n",
       " 0.03021432509381264,\n",
       " 0.037427848737774036,\n",
       " 0.040652021164714704,\n",
       " 0.12029877001051692,\n",
       " 0.03861205101677355,\n",
       " 0.08142590519954325,\n",
       " 0.029126397648906937,\n",
       " 0.026763454305444224,\n",
       " 0.012149344301971902,\n",
       " 0.03688618723074799,\n",
       " 0.09194913397638546,\n",
       " 0.02959485199184591,\n",
       " 0.1383748588865879,\n",
       " 0.04037816896654422,\n",
       " 0.04791512728454461,\n",
       " 0.12770025546267397,\n",
       " 0.09518527778585882,\n",
       " 0.12160318722736502,\n",
       " 0.025982890469013104,\n",
       " 0.11428873658365934,\n",
       " 0.025721387975461814,\n",
       " 0.05863087893037008,\n",
       " 1.8773680707378702e-07,\n",
       " 0.06812817937727363,\n",
       " 0.055165785147164234,\n",
       " 0.020396504252763268,\n",
       " 0.03592832174711034,\n",
       " 0.019145735909318246,\n",
       " 0.05206560286865547,\n",
       " 0.06477804880626696,\n",
       " 0.04521634005600504,\n",
       " 0.053686303931323405,\n",
       " 0.09520206838705228,\n",
       " 0.08181162021226358,\n",
       " 0.06408130961022274,\n",
       " 0.08040576685233275,\n",
       " 0.10473670413763644,\n",
       " 0.2691427039573804,\n",
       " 0.04402638360119518,\n",
       " 0.07226915749617446,\n",
       " 0.08470464258922393,\n",
       " 0.09801984199273811,\n",
       " 0.20370961798627238,\n",
       " 0.011281279185668578,\n",
       " 0.021927633721737273,\n",
       " 0.09158332614452068,\n",
       " 0.042548393146506214,\n",
       " 0.17728219037409568,\n",
       " 0.049514725106367535,\n",
       " 0.18361243940276029,\n",
       " 0.2672175551253171,\n",
       " 0.01557550164240275,\n",
       " 0.048138491095745216,\n",
       " 0.14811653585403048,\n",
       " 0.05377057734043587,\n",
       " 0.029134426701354632,\n",
       " 0.12294382346403177,\n",
       " 0.09097680928149449,\n",
       " 0.038760922038828825,\n",
       " 0.05418113445626592,\n",
       " 0.06230892126243547,\n",
       " 0.08173893062033136,\n",
       " 0.1788821192467597,\n",
       " 0.14923245674554989,\n",
       " 0.06807209338064862,\n",
       " 0.03522684884150419,\n",
       " 0.028785378324657343,\n",
       " 0.060089918676918395,\n",
       " 0.00215541040716432,\n",
       " 0.052120668702161733,\n",
       " 0.12400171769282298,\n",
       " 0.0730389462704558,\n",
       " 0.20340537356728136,\n",
       " 0.06658008597994451,\n",
       " 0.03404275153426902,\n",
       " 0.015226182826858921,\n",
       " 0.02157219099013518,\n",
       " 0.10333597231316534,\n",
       " 0.05075301870320621,\n",
       " 0.04584641830958432,\n",
       " 0.06745610271576849,\n",
       " 0.0237633105630907,\n",
       " 0.1883285231808376,\n",
       " 0.11353055205109619,\n",
       " 0.01681433328578664,\n",
       " 0.03505827732030693,\n",
       " 0.06688966891332344,\n",
       " 0.04569304656457459,\n",
       " 0.049086065410402974,\n",
       " 0.0021140575156118257,\n",
       " 0.02152788499589006,\n",
       " 0.04716989474188665,\n",
       " 0.03269643413024525,\n",
       " 0.33951506138748866,\n",
       " 0.1843644300081158,\n",
       " 0.03799068852674619,\n",
       " 0.15584157726979037,\n",
       " 0.047459607613819545,\n",
       " 0.10940333552812895,\n",
       " 0.018689219128275206,\n",
       " 0.05207825965905318,\n",
       " 0.061123603109071556,\n",
       " 0.02230312413836564,\n",
       " 0.01537046305102924,\n",
       " 0.1008628176114525,\n",
       " 0.07720609753731261,\n",
       " 0.0871678357833013,\n",
       " 0.17246832373415402,\n",
       " 0.15626533035011492,\n",
       " 0.12155600899668877,\n",
       " 0.17715106041657092,\n",
       " 0.01915942895299378,\n",
       " 0.025675173912061977,\n",
       " 0.13885240895024378,\n",
       " 0.18539045958935624,\n",
       " 0.0579103816241744,\n",
       " 0.20754457514133207,\n",
       " 0.01833079136276197,\n",
       " 0.0803561014871713,\n",
       " 0.047513330509224364,\n",
       " 0.08464363119124212,\n",
       " 0.10589843161155672,\n",
       " 0.038894606893888635,\n",
       " 0.015467327248703876,\n",
       " 0.11830357164317559,\n",
       " 0.13361324674211605,\n",
       " 0.02226714275745219,\n",
       " 0.11484812151628583,\n",
       " 0.024338210745661122,\n",
       " 0.03648459678512639,\n",
       " 0.03233380634698643,\n",
       " 0.18034693417332445,\n",
       " 0.03437354167881261,\n",
       " 0.04710835817827743,\n",
       " 0.058131094666824944,\n",
       " 0.08935672706591767,\n",
       " 0.09849507889334205,\n",
       " 0.044177886393929194,\n",
       " 0.06747055715259162,\n",
       " 0.007843153503465105,\n",
       " 0.04087801732159959,\n",
       " 0.03498429708412485,\n",
       " 0.03055584580499494,\n",
       " 0.035481209955489396,\n",
       " 0.25961127659727967,\n",
       " 0.32631712084637776,\n",
       " 0.020803476248066846,\n",
       " 0.047045540327652714,\n",
       " 0.2139667461838522,\n",
       " 0.09615698632127483,\n",
       " 0.0011431274337713245,\n",
       " 0.14674839187342717,\n",
       " 0.10772265267562639,\n",
       " 0.045847791589040315,\n",
       " 0.1964835661899081,\n",
       " 0.09430323504244642,\n",
       " 0.030013370573531258,\n",
       " 0.005398622335222398,\n",
       " 0.030381520591295293,\n",
       " 0.009796249185084333,\n",
       " 0.008808977728148494,\n",
       " 0.09219338469844929,\n",
       " 0.05112615108267408,\n",
       " 0.08170419395040071,\n",
       " 0.05667894537693977,\n",
       " 0.03756315298770803,\n",
       " 0.044468624809898785,\n",
       " 0.013130409847193962,\n",
       " 0.07070233467254491,\n",
       " 0.048942677776328464,\n",
       " 0.07100717415134841,\n",
       " 0.016670749386677917,\n",
       " 0.03900919858597911,\n",
       " 0.11714017196802566,\n",
       " 0.07054152713042385,\n",
       " 0.19944888182882184,\n",
       " 0.06414894779778411,\n",
       " 0.021207296961183248,\n",
       " 0.101189717736818,\n",
       " 0.04516972200799712,\n",
       " 0.057434058315837054,\n",
       " 0.01391776132527945,\n",
       " 0.09342095702238144,\n",
       " 0.10959358298846131,\n",
       " 0.06136374790663974,\n",
       " 0.047967489493136097,\n",
       " 0.06229602816217417,\n",
       " 0.019250233521512967,\n",
       " 0.09807750456642829,\n",
       " 0.02417340861682181,\n",
       " 0.07249059246559941,\n",
       " 0.16575680260637013,\n",
       " 0.038777394553862206,\n",
       " 0.05368828679836972,\n",
       " 0.016665449671807817,\n",
       " 0.047442503482698725,\n",
       " 0.1724064700856935,\n",
       " 0.04005124192386385,\n",
       " 0.17947506800908417,\n",
       " 0.06083216034410374,\n",
       " 0.030990253688959316,\n",
       " 0.038836201114994445,\n",
       " 0.022965488949471403,\n",
       " 0.041871710718833936,\n",
       " 0.04206694993901114,\n",
       " 0.09231133690047033,\n",
       " 0.051646240528707324,\n",
       " 0.08562623253219337,\n",
       " 0.04679437409144144,\n",
       " 0.004340828827349629,\n",
       " 0.049423644627071835,\n",
       " 0.025816939324911577,\n",
       " 0.014137757493977915,\n",
       " 0.04497520393835546,\n",
       " 0.03416447750228881,\n",
       " 0.5111187483433824,\n",
       " 0.06848058872542534,\n",
       " 0.14848927591866037,\n",
       " 0.07573475386030466,\n",
       " 0.03917311398099917,\n",
       " 0.026156702801479935,\n",
       " 0.04262445777154587,\n",
       " 0.28958259314530804,\n",
       " 0.12244078035681967,\n",
       " 0.022084756148949607,\n",
       " 0.015021754452334125,\n",
       " 0.19884789159849336,\n",
       " 0.07468922802862353,\n",
       " 0.046484186019727855,\n",
       " 0.0199759301020299,\n",
       " 0.0868209002941518,\n",
       " 0.04401819409225444,\n",
       " 0.03969699780345892,\n",
       " 0.04183452243215846,\n",
       " 0.050772320606285594,\n",
       " 0.023369054421272257,\n",
       " 0.03976517445151053,\n",
       " 0.12924015037122757,\n",
       " 0.05080847071926857,\n",
       " 0.1313857315535587,\n",
       " 0.1376229530478801,\n",
       " 0.08217489342949238,\n",
       " 0.052496710636772886,\n",
       " 0.07529319539192737,\n",
       " 0.04232668463692176,\n",
       " 0.033027060998640966,\n",
       " 0.0297473188088524,\n",
       " 0.16120977891867344,\n",
       " 0.03865827115962916,\n",
       " 0.10700136205105044,\n",
       " 0.0628085164237875,\n",
       " 0.030820903415935755,\n",
       " 0.011673261815316197,\n",
       " 0.12551931260637303,\n",
       " 0.01922844532332952,\n",
       " 0.21454139058598967,\n",
       " 0.01971196532292925,\n",
       " 0.02679898206681635,\n",
       " 0.01155346168103329,\n",
       " 0.1302786949937344,\n",
       " 0.04617582342829073,\n",
       " 0.05144446103495415,\n",
       " 0.031090812389223406,\n",
       " 0.29850455617732596,\n",
       " 0.0922351126078002,\n",
       " 0.045243785075502245,\n",
       " 0.04879571259330408,\n",
       " 0.018636495573344368,\n",
       " 0.07455079324238148,\n",
       " 0.16980220897421477,\n",
       " 0.02254457049954421,\n",
       " 0.0027537551582995554,\n",
       " 0.24197637262788801,\n",
       " 0.10599956576251264,\n",
       " 0.0761188779873313,\n",
       " 0.07907997329819987,\n",
       " 0.1535493563847929,\n",
       " 0.015333395428578155,\n",
       " 0.01604007220781353,\n",
       " 0.11309264647628682,\n",
       " 0.02775268973400405,\n",
       " 0.1077093645098977,\n",
       " 0.02478662267129504,\n",
       " 0.0426342316423595,\n",
       " 0.11463497160192268,\n",
       " 0.04858008718230895,\n",
       " 0.16608648268995868,\n",
       " 0.031770315330455025,\n",
       " 0.05948152194698687,\n",
       " 0.32662549519471407,\n",
       " 0.08846320867152806,\n",
       " 0.331690111678013,\n",
       " 0.024967327868392616,\n",
       " 0.01917995568047419,\n",
       " 0.16847042970906398,\n",
       " 0.05428433921928461,\n",
       " 0.0076231250924436635,\n",
       " 0.004260502388239342,\n",
       " 0.040610645228403644,\n",
       " 0.07446832854611829,\n",
       " 0.06577259465846995,\n",
       " 0.04131639686334159,\n",
       " 0.0496421752168733,\n",
       " 0.06017431290763472,\n",
       " 0.06398216870331891,\n",
       " 0.03688725247950794,\n",
       " 0.021933856460234023,\n",
       " 0.021292680625893807,\n",
       " 0.05699556500485551,\n",
       " 0.02281049321385649,\n",
       " 0.03068011967268028,\n",
       " 0.014494359763430227,\n",
       " 0.0312697193086119,\n",
       " 0.044229846339976404,\n",
       " 0.028224099334583346,\n",
       " 0.037945977627111174,\n",
       " 0.0406619116671755,\n",
       " 0.055548490654034846,\n",
       " 0.03580054168762627,\n",
       " 0.015222393250438938,\n",
       " 0.02030347843999661,\n",
       " 0.03709837250680535,\n",
       " 0.058153072694528074,\n",
       " 0.06324592047658355,\n",
       " 0.029687496911755467,\n",
       " 0.0006946201047485843,\n",
       " 0.19179516132082097,\n",
       " 0.06274218992050104,\n",
       " 0.2832640524850018,\n",
       " 0.03249148237054889,\n",
       " 0.016751138063373208,\n",
       " 0.0268724808574022,\n",
       " 0.03908959786499411,\n",
       " 0.10913954014275593,\n",
       " 0.12454461750573252,\n",
       " 0.02791923061609173,\n",
       " 0.05456645294901273,\n",
       " 0.025233585752071755,\n",
       " 0.06496381543875857,\n",
       " 0.07395159877360116,\n",
       " 0.0507529937630301,\n",
       " 0.08445405917994396,\n",
       " 0.029070470720859973,\n",
       " 0.14044996513814045,\n",
       " 0.3033645025982875,\n",
       " 0.0521025905289028,\n",
       " 0.06208236537957225,\n",
       " 0.03660369790362627,\n",
       " 0.03728413021664692,\n",
       " 0.06508203426605642,\n",
       " 0.020283328922062208,\n",
       " 0.08363878180844277,\n",
       " 0.03312120241979076,\n",
       " 0.13930288415109804,\n",
       " 0.03684113765875668,\n",
       " 0.06778693071967923,\n",
       " 0.04811401909183672,\n",
       " 0.008900661904951125,\n",
       " 0.12019750465338365,\n",
       " 0.03866728953286576,\n",
       " 0.045206962562353195,\n",
       " 0.09496874827215283,\n",
       " 0.17618249553563836,\n",
       " 0.07076001086441407,\n",
       " 0.1723156336049605,\n",
       " 0.04034576966165721,\n",
       " 0.043475944867310934,\n",
       " 0.043781903849281885,\n",
       " 0.12601082325943044,\n",
       " 0.11366997152028845,\n",
       " 0.11284237683128182,\n",
       " 0.07576309587222878,\n",
       " 0.17122665813649984,\n",
       " 0.10945603341069235,\n",
       " 0.19075628464624242,\n",
       " 0.046190990817730676,\n",
       " 0.027827436174663274,\n",
       " 0.04490656112383939,\n",
       " 0.005569829133982454,\n",
       " 0.02816517680095596,\n",
       " 0.013774905030942471,\n",
       " 0.0522387920578276,\n",
       " 0.03527530100492566,\n",
       " 0.011974362805394228,\n",
       " 0.07473232699845025,\n",
       " 0.054295079243805576,\n",
       " 0.12125647195287828,\n",
       " 0.024425817951698753,\n",
       " 0.0567532766192717,\n",
       " 0.14310745776284547,\n",
       " 0.08309927597191187,\n",
       " 0.14230014174264252,\n",
       " 0.044457415249531985,\n",
       " 0.17042369366843943,\n",
       " 0.049259101102330616,\n",
       " 0.1269570005677744,\n",
       " 0.045826891404634,\n",
       " 0.03044978424350864,\n",
       " 0.21318874217944922,\n",
       " 0.22279880593179419,\n",
       " 0.06439026728249982,\n",
       " 0.030303916420788297,\n",
       " 0.10704757060867379,\n",
       " 0.020729723061808595,\n",
       " 0.02565704651632342,\n",
       " 0.04091990766439897,\n",
       " 0.01618274669190153,\n",
       " 0.012205109414802701,\n",
       " 0.0524991265242983,\n",
       " 0.01042753177231234,\n",
       " 0.07000417813982844,\n",
       " 0.03303655686930559,\n",
       " 0.33022708486708585,\n",
       " 0.07532681040911603,\n",
       " 0.009078673668567329,\n",
       " 0.02269869702747828,\n",
       " 0.12949178345158222,\n",
       " 0.2496011227182911,\n",
       " 0.07279690766816908,\n",
       " 0.06109088670883436,\n",
       " 0.08727162638550519,\n",
       " 0.0738834239063817,\n",
       " 0.14839738604324357,\n",
       " 0.031907357523748646,\n",
       " 0.0017624187994181913,\n",
       " 0.06370163957964373,\n",
       " 0.052997166244711506,\n",
       " 0.06124381898424883,\n",
       " 0.07781175784229234,\n",
       " 0.00867548062143805,\n",
       " 0.02158276705690799,\n",
       " 0.05151334908784168,\n",
       " 0.019589633930704616,\n",
       " 0.054936476704135755,\n",
       " 0.09192403719112639,\n",
       " 0.09138879621820889,\n",
       " 0.0223367392506835,\n",
       " 0.03346937895600999,\n",
       " 0.08190702943189358,\n",
       " 0.0805428373776087,\n",
       " 0.11017273471559147,\n",
       " 0.060836713703945944,\n",
       " 0.21946298358977082,\n",
       " 0.04817912487657683,\n",
       " 0.13906421709443864,\n",
       " 0.2949516372345125,\n",
       " 0.17539370522368708,\n",
       " 0.058516088065357616,\n",
       " 0.1296694765119584,\n",
       " 0.057051201764461455,\n",
       " 0.047391315546135425,\n",
       " 0.023611122552791013,\n",
       " 0.058512595336041855,\n",
       " 0.08607333254984752,\n",
       " 0.1438667956566397,\n",
       " 0.04267986517434315,\n",
       " 0.18149711239775695,\n",
       " 0.08109819015420124,\n",
       " 0.04127789880983361,\n",
       " 0.07088434715941587,\n",
       " 0.044160763806075316,\n",
       " 0.36001068390838503,\n",
       " 0.09365682921458467,\n",
       " 0.07891792163517793,\n",
       " 0.1713869129303154,\n",
       " 0.005863128673551513,\n",
       " 0.097622706213451,\n",
       " 0.09943145832736548,\n",
       " 0.02052584698686147,\n",
       " 0.22182288901161126,\n",
       " 0.048809249365818666,\n",
       " 0.03524592790951608,\n",
       " 0.04892021113571148,\n",
       " 0.008991561852396421,\n",
       " 0.06245042561583049,\n",
       " 0.03544394019111126,\n",
       " 0.05719670095667936,\n",
       " 0.006292845137176821,\n",
       " 0.2929896428375712,\n",
       " 0.010663410416475593,\n",
       " 0.06215577120019448,\n",
       " 0.1656781852688869,\n",
       " 0.011303502618998278,\n",
       " 0.09169575321716886,\n",
       " 0.019122161010266578,\n",
       " 0.1361030018434609,\n",
       " 0.12372803687449362,\n",
       " 0.04822109334020985,\n",
       " 0.0729181705147605,\n",
       " 0.017836532518881044,\n",
       " 0.05620099048704851,\n",
       " 0.00827514507949957,\n",
       " 0.11073656877510385,\n",
       " 0.08641036804724417,\n",
       " 0.061789425617267954,\n",
       " 0.05205376517414245,\n",
       " 0.04216148571908571,\n",
       " 0.21274864314315597,\n",
       " 0.037842388034871215,\n",
       " 0.302596592043492,\n",
       " 0.1457359462595776,\n",
       " 0.06787383154648478,\n",
       " 0.05437103716083565,\n",
       " 0.05165179337774693,\n",
       " 0.01920473761230926,\n",
       " 0.06802219942456184,\n",
       " 0.04680984212383751,\n",
       " 0.022117012336599958,\n",
       " 0.06409830045654168,\n",
       " 0.049143775248769415,\n",
       " 0.09374553556946039,\n",
       " 0.1441696749713501,\n",
       " 0.045987594023136326,\n",
       " 0.023107231733153075,\n",
       " 0.019900959813943974,\n",
       " 0.04818598396475466,\n",
       " 0.3512210261486663,\n",
       " 0.03461219345184888,\n",
       " 0.05783504860021291,\n",
       " 0.04480694024818388,\n",
       " 0.03658065791102416,\n",
       " 0.10024453470266409,\n",
       " 0.033435111115829766,\n",
       " 0.1448817022543585,\n",
       " 0.10377930083910347,\n",
       " 0.06687171638805785,\n",
       " 0.12064457546853538,\n",
       " 0.00732350719423629,\n",
       " 0.1580994985094973,\n",
       " 0.013789449262554425,\n",
       " 0.11722294866361696,\n",
       " 0.2323037128628111,\n",
       " 0.01370134333307698,\n",
       " 0.037527441342384164,\n",
       " 0.0458887352505444,\n",
       " 0.12970024204636738,\n",
       " 0.0679047524470946,\n",
       " 0.01617186057705989,\n",
       " 0.1291866566717041,\n",
       " 0.1280840591832086,\n",
       " 0.2586396397301993,\n",
       " 0.03822231198725349,\n",
       " 0.009069009320788565,\n",
       " 0.014841131071347765,\n",
       " 0.07856408224476388,\n",
       " 0.09321041057308009,\n",
       " 0.04973809362714788,\n",
       " ...]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrmodel.predict(test_set.map(lambda x: x.features)).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_true = np.array(scores_and_labels.map(lambda lp: lp[1]).collect())\n",
    "y_pred = np.array(scores_and_labels.map(lambda lp: lp[0]).collect())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0.0773821364631\n",
      "Accuracy: 92.2617863537\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_err = (y_true != y_pred).sum() / float(len(y_pred))\n",
    "    \n",
    "print 'Test Error = ' + str(test_err)\n",
    "print \"Accuracy: \" + str((1.0 - test_err) * 100)\n",
    "print \"\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(2)\n",
    "    plt.xticks(tick_marks, ['no upsell', 'upsell'], rotation=45)\n",
    "    plt.yticks(tick_marks, ['no upsell', 'upsell'])\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, Logistic Regression\n",
      "[[9274    5]\n",
      " [ 773    2]]\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEnCAYAAADl8/suAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8bWPZ//HPvZZtsx3KIUoOpcejX3npsPETtr21SxH1\n",
       "FK704BFFTpFQT1IokthJj0MhlA664oXIsWUf2PLYUT9FQs7kTE5t7cP4/THuufbY05xjzrnmaYy5\n",
       "vu9e89Wa9xxrzntOe13rWte4x32FJEkQEZHuG+r3BERExgsFXBGRHlHAFRHpEQVcEZEeUcAVEekR\n",
       "BVwRkR5RwB0gIYRdQwh/CCE8GUK4P4RwfQjh37r0WtvF13g4vuZwl15nYnwf63Xj+XsphPDpEMIN\n",
       "HXiegflMxptl+j0B6YwQwg7A9wBLkmROHHsP8HCXXvJA4KwkSU7o0vMDkCTJq8D7u/kaZaPPpLyU\n",
       "4Q6OrwPfqARbgCRJ/hB/OAEIIbwxhPDLEMKDMTs9I4SwYubxaSGER0IIHw8hzAshPBVC+HkIIWSO\n",
       "OSGE8DDwAeDwmOHOzE4khLA4hLB+5v4xIYQLqo7ZO4TwpxDCQ/F2WNXjb4vP/XD182WO2TSEMCce\n",
       "c1cI4QtVjx8T539UCOHO+H4ObeEzJYTwQAjhkBDCfSGE80IIO4UQHg0hfCNzzCohhO+FEO4IIfw9\n",
       "hHB7CGGrquf5NTAD2DTzvv5v1THnx8/3tPiZPBpC2LjZzyT+93sshLBmvD8UQpgZQtinlfcsXZQk\n",
       "iW4lvwHDwL+Ad+UcMwTMA74FBGA54OfALzPHTAMWxGMmAqsBzwLTazzfTGDvOq+1GFg/c/9o4ILM\n",
       "/Q2AfwLvyIytkDP3pZ4vjq0V57ZjvP9m4HZg/8wxxwDPA7vG++8HXgUmtvDZ3g+cDfx7nMcRwIeB\n",
       "xzPHTAS2BYbi/WOBO2o8157ADTmvdT7wJLBvvL9SK59JHP8acHXmc/9Zv/996rbkpgx3MKxJWh56\n",
       "MeeYTYH1gKOS1HzgIGDnEMLqmeMeT5LkyCRJXk2S5BngTmCdOs8Z6ow3Ou4fpIFv2xDCmwGSJHm5\n",
       "yeeq2AOYlSTJ5fH7HyUNMAdVHffrJEkujF/fAEwA3tTia/0CeCB+/UvgIWDVyoPxs7o2SZLFcegi\n",
       "YMMaz9Po80oAT5LkrPi8ef896zkOWCaE8BNgF2DfMTyHdIkC7mCo/GCukXPMesCDmaBAkiTPAs/F\n",
       "x+pZQP1/J81uxLHUcUmSPAlsBqwLXB9CuC2EML3J56pYF/hb1di9vPa9jAa5JEkWxC/H8u9+cdXX\n",
       "2TJLCCEcGssbNwKnA0MhhLG8znNj+J5RSZraHg7sDpw4hl9k0kUKuAMgZkJ3kn8i5SHgrSGECZWB\n",
       "EMIawCrAg52eEkv/23pNZpckyd1JknwxSZINSf/0vzyEMLGF13iI12aRb2dJJlqZRy98HtgZ2CVJ\n",
       "kq1ISwd9EYP8ycCPgGNDCK/v11zktRRwB8cJwGEhhOlhiSmZEyv/SxqMTgghDIcQJpFmYhcnSfJ0\n",
       "g+eu96dwvfFHgI0A4rK0T1EV/KpO+CxLWmJY1GAeWRcAU0MIn4jPty5p7fSMJubXaWsDjwNPxvLM\n",
       "SXF8QtVxzwEbhBBWiv993lz1eCfmexTwbJIk+5CWQs7pwHNKhyjgDogkSX5GmmnNID3x8iDwJWDF\n",
       "+HgC7Ej6p/j9pBnxM8De1U9V6+nrvWyd8cOBk0IIV8Wvr80+GEJYGfC4IuJh0iVm2yVJsrDZ10mS\n",
       "5DHSE1WHhhAeAa4jPbl1ZtX3VX9vO1lvUufrGcDqwGPAb4CzgIWkJ/KyrgZuA+4D7uK19dVa821m\n",
       "LgCEEKaSlhI+G4eOBtYJIezX5HNKl4X051BERLpNGa6ISI8o4IqI9IgCrohIjyjgioj0iDav6YCR\n",
       "kRGdeRRpwvTp0zu2VK+Vn7tOvm47FHA7ZIfDL+n3FNq26MVHGV6peiVTOT0377R+T6Ej5syexdZT\n",
       "p/V7Gh0xd/ZIx59zhyMua3jMFSd9rOOvO1YKuCJSXqG9xNXMPgd8GngJOMDd7zGzD5CuYQY42t2v\n",
       "j8e2NF6LargyalCy20EyKNlt14Shxrc6zGwSsJe7v4/0ashvmVkgvWJx23g7Jh471Ox4fI6alOGK\n",
       "SHm1l+EGYIKZTSTdxvONpFuH3u3u/wQws7+Z2QakyWlT48C/AffUekEFXBk1SDXcQTFINdyuGBp7\n",
       "Zyd3f9nMvgVcRbrj3iqkQfd5MzslHvYP0n2hQ4vjCrgiMmDGtAPmEu5+MXAxgJndRroJ0euBA0iD\n",
       "6RnA06SZbCvjNamGK6OU3RaPstsGQmh8a4KZbQ/8kXRP5X/PPLSBu99LuvdyK+M1KeCKSHkNDTe+\n",
       "5TCzH5nZjcDBwJfcfTHpSbDrSHe5OwbA3Re1Ml6PSgoySjXc4lENt4H2SwqfqTF2LVVbio5lvBYF\n",
       "XBEprzbX4faaAq6MUnZbPMpuG2gzw+01BVwRKa/hsS8L64dy/XqQrlr04qP9noJUmTN7Vr+nUGxt\n",
       "XGnWD8pwRaS8VMOVslINt3hUw22gYBlsIwq4IlJebVza2w/l+vUgXaUabvGohttAh6406xVluCJS\n",
       "XiopSFmphls8quE2ULAMthEFXBEpL9VwpaxUwy0e1XAb0DpcEZEeKVhAbUQBV0aphls8quE20GZJ\n",
       "wcz+CzgQWAgc5e4zu9lEUgFXRMqr/ZNmhwPvAVYArjGzLUj3t/1AfPwa4PpMs8iG42Y2092TWi9W\n",
       "rnxcuko13OJRDbeB9mu4dwJTgR2Am8k0kYyNISvNIpseJ20iWZMyXBEpr/Yz3GuBLwATSPuRrYaa\n",
       "SEovqIZbPKrh5hsaGvsf6Wa2PrCDu3803p8DHISaSIqI1BCauNU3TEw6zSwAy6MmktIrquEWj2q4\n",
       "+UIIDW/1uPs9wM1mdiVwFXC6u7+CmkiKiLxWXkBthrt/q8aYmkhK96mGWzyq4eZrp4bbDwq4IlJe\n",
       "5dq7RjVcWUI13OJRDTdfOzXcflCGKyKlVbSA2ogCroxSDbd4VMPNpxquiEivlCvBVQ1XllANt3hU\n",
       "w82nGq6ISI+opCClpRpu8aiG20CxEtiGFHBFpLSKVjJopFz5uHSVarjFoxpuPtVwRUR6RDVcKS3V\n",
       "cItHNdwGipXANqSAKyKlVbSSQSMKuDJq0YuPKsstmDmzZynLzdFOwDWzlYHLMkPvdffXqWtvl5nZ\n",
       "ecA04CB3/02jcREphjA09oDr7i8A2wCY2cbA52PnB3Xt7SZ33ws4v9nxQaXstniU3ebr4CqFg4H/\n",
       "IW2XU86uvWZ2O2nriS2B6939yDi+J/A50pL3ue5+doPnmefum9b4+i5gLrARcIW7fzOOHwDsDiwC\n",
       "znP3c+P4h4GvAwnwXXe/uMNvWUR6qBM1XDNbDVjH3W83s/fRxa693c5wVwG+QxpwdwQws9WB/YCt\n",
       "ga2A3cxsrKnVcsAhwPuA7cxszTi+C7C9u0/JBNsh4NvAB0n70B9iZhPH+LoDSetwi0frcPN1KMPd\n",
       "Fzgrfv0MaRfeI4Gvxq+fHsN4Td0OuI+7+xOx0dr8OLY+8Ht3XxjHbyZNy8fiKXd/yd0XA7cC68Xx\n",
       "PYDPmdlJZrZlHFsdWBu4gjTrfj2w1hhfV0QKIAyFhrc8ZrYMsANwSRxqtTtvS117+3HS7D5gEzOb\n",
       "QPqn/RbAqQ2+ZwjAzCYBkzLja5nZqqRp/HtJywW4+yPAiWa2HGnJYTLwFPAX4GOxWN5R2TP8lUxR\n",
       "9/t3P3t2v5IllvH+1lOnFWo+7dwfpvM6UFL4D+DymLTh7ovMrNKFFzLdeVsZrzvfJKl5Mq0jzOwW\n",
       "d9+sxtfZGu6P3P2cBs9zJvAy8BKws7tvFMfvAOYB7wQucPfvx/FTgHcDK5O2Pq6UFbYEjicN9I+5\n",
       "+26Z1zga2An4sbvPaDSeNTIykuxw+CW1HpI+eW7eaf2eglSZO3uE6dOnd2zh7MjISLLvlf9qeNxZ\n",
       "2y/b0ddtR1cDbrdlT6D106AE3EFahzsoAXeQ1uF2I+Dud/WChsf94MMTChNwy74Ot7y/LUSkbSW7\n",
       "0KzcAbdSopDOGJTsdpAMSnbbLbq0V0SkR0oWb3WlmSyhdbjFo3W4+YaHQ8NbkSjDFZHSUklBSks1\n",
       "3OJRDTdfyeKtAq6IlFfZMlzVcGWUarjFoxpuvqGh0PBWJMpwRaS0mstwi7NcXwFXRqmGWzyq4eYr\n",
       "WUVBAVdEyqtsGa5quDJKNdziUQ03n2q4IiI9Mi5KCmZ2prvv3+nJSH+phls8quHma3dZmJmtDVxA\n",
       "GgtvcffD+tK118wmU7v4EYDNm39LIiLd0YGSwcnAV939JhhtxdW1rr15Ge4I8Ic6jz3f5JuREhmk\n",
       "/XAHxSDth9sN7SS4ZjYMvK0SbKPRLrzxmEp33qFmx0m79tZsIpkXcP/k7tuM/e2IiHRXmyWFNwDL\n",
       "mdmlpN1h/gd4nC527c0LuEfnPCYDSNlt8Si7zddmCfcZ0gC5E2nLtbnAZ0gbzB5AGkzPIO3CO9Ti\n",
       "eE11l4XlFX5FRIpgaGio4a0ed18APAy80d3/BbwK3EsXu/Y2vQ7XzFZp9lgpJ63DLR6tw80XQuNb\n",
       "A18GzjazucCv3P0V0pNg1wHXkunO28p4PQ2XhcXVCucCE4G3x7Ny57j73g3fiohIF7W7LMzdHwK2\n",
       "rxq7ljR4Vh/b0ngtzWS43wU+Cvw9Pvli0jN5MmBUwy0e1XDzdSDD7almAm7i7g9WjU3sxmRERFox\n",
       "iJf2Pm1m2wHBzFYCvkH99blSYlqHWzxah5tvqGgpbAPNZLj7A3sCG5GuLZsEHNbNSYmINKNsJYWG\n",
       "Ga67PwXs2oO5SJ8puy0eZbf5ytZip+nNa8xsArCw3jXCIiK9NlywGm0jDUsKZraJmd0KPAY8aWbX\n",
       "mtnbuj816TWtwy0ercPNV7aSQjM13HOAI939De7+BuBM0u3MRET6KjTxvyJpJuC+4u7XVO64+yXA\n",
       "ou5NSfpFNdziUQ033/BQaHgrkmYC7jVmtnXljpl9BPht96YkItKcspUU8jYgf4l0A/IAHGVm8+ND\n",
       "k4DK9cYyQLQOt3i0Djdf2dbh1g247r5iLyciItKqksVbNZGUJZTdFo+y23xFu3S3kWZ2C1sO+Djw\n",
       "JtLyQgDWdPcjujw3EZFcA1NSyLgYWACsQHpp73uAWV2ck/SJarjFoxpuvnbCrZmdD2wIzAfOc/ef\n",
       "dLNjLzQXcNdy9/eY2b6ku5sfCfy0+bclItIdbV7amwCfjHvidr1jLzS3LOzJ+P93A1u4+/PAOs2/\n",
       "JykLZbfFo+w2XwfW4WYPGO3YG7vwVjrzNj1O2rG3rmYy3JvN7A3AjcDJcavGO5v4PhGRrmqzhPsi\n",
       "8HMzexY4FFiVLnbsheZ2Cxvt3mtmU0lrHn9s4U1JSaiGWzyq4eZrp6Tg7gcDmNm7gZNI+5t1rWMv\n",
       "tLgszN1fBm5r5XtERLqlQ6vC5pMuDKjZsdfMhlsZz51vR6YrA0HZbfEou803FELDWz1mdqGZzQZm\n",
       "AEfEfo1d69gLY7zwwczOdPf9x/K9IiKd0s46XHd/TWOFbnbshfy9FCaTLpuoFoDNm30BKQ/VcItH\n",
       "Ndx8JbvuITfDHaF+s8jnuzAXEZGWDNKlvX9y9216NhPpO2W3xaPsNt8gXdp7dM5jIiJ9V7J4m7s9\n",
       "Y+41wbK0+2Z9t99TaNtNN8xmiylT+z0NyVANN9/Adu0VESmaYQVcKStlt8Wj7DZfyc6ZKeCKSHmV\n",
       "LeDqSjMZddMNs/s9BakyZ/asfk+h0EIIDW9FogxXREpruGQpowKujFINt3hUw803SOtwRUQKrWQJ\n",
       "bunmK12kGm7xqIabL4TGtyJppmvv6sCJwNru/iEzC8CB7n5a12cnIpKjiRY6hdJMSeFs4GfA5wHc\n",
       "PTGzXQEF3AGjGm7xqIabrxPx1swmkvZs/I67n97Nzr3NlBRWdfeLgMUtfp+ISFe1swF5xn7ArUAS\n",
       "/4I/Ftg23o6BpTr6NhyPz1F7vk1MZpGZvalyx8w+jrZnHEiq4RaParj5hoca3/KY2STgg8BlpHt9\n",
       "d7VzbzMlhSOAq4F1zOw2YCLwsSa+T0SkqwJt1xQOJi2Prhnvr0YXO/c2zHDd/VZgU2AasDuwcaNG\n",
       "aVJOquEWj2q4+YZC41s9ZvY6YCt3vxpGI/czpJ14jwS+Gr9+egzjNTW1Dtfd/wXc3syxIiK90tRJ\n",
       "s1qNwlJbAsuZ2S+At5LGwxvoYufeZpaFHVbrLbh7+TeAlaVoP9zi0X64+ZpaFra49rC7XwlcCWBm\n",
       "ewIruPvtZlbpxAuZDr2tjNfTTIa7Ekv/jtgUnTQTkQLo1IUN7v7jzNdd69zbMOC6+zHZ+2Y2ATi5\n",
       "mSeXclF2WzzKbvOVbS+FltfTuvsCYI0uzEVEpCXtnDTrh2ZquJdXDa0BPNqd6Ug/qYZbPKrh5hvE\n",
       "Fjszqu4/6+5asSAifVeyeNtUDXdWD+YhBaDstniU3eYrWsmgkYY1XDNbrxcTERFpVYf2UuiZZk6a\n",
       "/brrs5BC0F4KxaO9FPIND4WGtyJppob7z67PQkRkDAqWwDbUTMA9x8xmAMdnB9392e5MSfpFNdzi\n",
       "UQ03X9n2iW0m4B5FeqXZJzJjCbB+V2YkItKkotVoG2lmlcJbejAPKQCtwy0ercPNN3ABV0SkqMoV\n",
       "bnNKIGb2lV5ORPpP2W3xKLvNN0hde7cHTujVREREWtXOpb1mdhywBekGjvu6+33dbCAJ+QF3GTNb\n",
       "td6DWqUweFTDLR7VcPOFNgKuux8FYGZbAl82s/1IG0J+IB5yDXB9plFkw3Ezm+nudbc8zwu47yHt\n",
       "ZFmLVimISN91qGKwOfAXMg0hAcys0ihyqNlx0gaSNfuZQX7AnefuUzrxbqQclN0Wj7LbfO1kuABm\n",
       "NgdYHZhC2iqnaw0koXzrhkVERg2H0PCWx923Bj4N/IQuN5CE/IB7aoP3KgNGeykUj/ZSyBeauDXh\n",
       "cdK/9u+ldkPIv7U4XlfdgOvuFzU3VxGR/mhnWZiZ/dLMRoAfAge5+2LSk2DXkfYoOwbSRpGtjOfO\n",
       "N0nq9xCW5oyMjCTv2ETl7iJZZYVl+z0FqTJ39gjTp0/v2MrYkZGR5JU1Nmp43KQn/9zR122HrjQT\n",
       "kdIq26W9Omkmo1TDLR7VcPMN0pVmIiKFNlSy3RQUcGWU1uEWj9bh5hsq2d/oCrgiUlqhZBluyX4/\n",
       "SDephls8quHmGwqNb0WiDFdESquZDLdIC18VcAEzOw+YRrr4+TeNxgeVarjFoxpuvmaWhS3qwTya\n",
       "pZIC4O57Aec3Oy4ixaCSQheY2Tx337TG13cBc4GNgCvc/Ztx/ABgd9Jfbue5+7lx/MPA10n/yviu\n",
       "u1/c8zdTYNoPt3i0H24+nTTrreWAQ4D3AduZ2ZpxfBdge3efkgm2Q8C3gQ8CU4FDzGxiH+YsIh2i\n",
       "Cx966yl3fwnAzG4F1gOeAPYAPmdmqwOXuvtc0j0v1wauiN/7emAt4P6ez7qglN0Wj7LbfO202OmH\n",
       "sgTcIQAzmwRMyoyvFdsA/QN4L2m5AHd/BDjRzJYjLTlMBp4i3dX9Y+7+QqcnmP1zvLK8Svf7d3+l\n",
       "5SeMBqvK0ird7+/9YTqvXOG2JLuFmdmZwMvAS8DO7r5RHL8DmAe8E7jA3b8fx08B3g2sDJyeKSts\n",
       "CRxPWsN9zN13y7zG0cBOwI/dfUaj8axB2S1skGq4g7Jb2CDVcLuxW9gKb5nc8LiXH7hVu4W1wt33\n",
       "z9w9JvP1K+7+6RrHH1rneeaSLvOq9dixpHtbNjUuIv3XzkkzM/sBsCHpX9B79aJrb9lPmhU/PS+R\n",
       "QcluB8mgZLfd0s6yMHffz923IU2ojjCzEL/eNt6OgdET7k2Nx+eoqxQZbj3uvlm/5yAifdSZQsGL\n",
       "wL/oc9deGWcGqYY7KAaphtsNHVqHuzdpD8fVUNdeEZHa2r3SzMx2BP7q7nfR5669Ms4ouy0eZbcN\n",
       "1GvVm73VYWaTganu/r041Gp33s517RURKbrQxP9y/ArY1MxmmtmprXbnVdfePtE63OLROtzi6cY6\n",
       "3NU2aHze/Jl7btE6XBGRdpXsyl4FXFliULLbQTIo2W23lG23MAVcESmtsmW4Omkmo9TTrHjU0yxf\n",
       "G4sU+kIZroiUVihZiquAK6NUwy0e1XDzlSzeKuCKSHmVLN6qhitLqIZbPKrhNlCyIq4yXBEprWba\n",
       "pBeJAq6MUg23eFTDzVeucKuAKyJlVrKIqxqujFINt3hUw803FELDW5EowxWR0ipWOG1MAVdGqYZb\n",
       "PKrhNlCyiKuAKyKl1WbX3inADGC2ux8Rx9S1V3pDNdziUQ03X5stdiYCJ1TutNKdt954o669Crgi\n",
       "Ul5tXPjg7r8Fns0MjXbtjZ14K915mx4n7dpbl0oKMko13OJRDTdfh/fDXZUud+1VwBWR0urwqq9K\n",
       "F94DSIPpGaRdeIdaHK9LJQUZpRpu8aiGm6/dNuksXXRQ114RkfrGXsQ1sy+TngDb0cx+qK69JTEo\n",
       "XXsHyaB07R0k3eja+/bJjX/u7rr1BnXtFRFpVyGiaAtUUpBRquEWj2q4+bSXgohIrxQrnjakgCuj\n",
       "tA63eLQON1/J4q0CroiUV9FKBo2ohiujVMMtHtVwG1BPMxGR3ihYPG1IAVdGqYZbPKrh5itZRUEB\n",
       "V0TKSzVcKS3VcItHNdzBogxXREqrZAmuAq4soRpu8aiGm6/D++F2nQKuiJRWE9svFopquDJKNdzi\n",
       "UQ23Aa3DFRHpjXZLCq123W2XMlwZpRpu8aiGmy+Exrd6xtJ1t10KuCJSWu0EXMbQdbddCrgySjXc\n",
       "4lENN19o4n85Rrv0xs67la67XaMabofc+fsb+j2Ftr1++aGBeB+DZJi0NY3UdtOctj6bel16u0Y9\n",
       "zURkXDKzYWAO8AHSgHudu2/ZzddUSUFExqWxdN1tlzJcEZEeUYYrItIjCrgiIj2igCtSYmamlUYl\n",
       "ooArbTOzt5nZhH7PY7wxs68C25vZiv2eizRHAVfaYmY/Ar4HbGFmE/s9n/HCzPYHPglMBrZU0C0H\n",
       "BVwZMzNbC3gEuAjYAtjczJaLjxVsn6aB8wt33xj4X2ArYIqZrdznOUkDqv/ImLn7Y8Sdlszss8A0\n",
       "0l/iM4FlgVf7NrnB9yKAu19pZpNIf+E9bWYJENx9Xl9nJzUpw5WWmdlKma+XAXD3c4CHgE3M7CDg\n",
       "eDN7XZ+mOJDMbCcz2wDSRfvxSinc/SLgduDTwHkUbhdYqdCFD9ISM/s+8Djwc3d/II4Nx6t2MLPP\n",
       "ADsBp7r7NX2b6IAxs5OAjwOnACPuflccz372HwKecfff92+mkkcBV5pmZtsCBwO/Bx4EZmaC7jLu\n",
       "vjB+vaa7P2Fmwd31D6wDzGxL4AHgU8B80qD7lzrHBgB99sWjgCstM7P1gM+SBt3r3f2+Oscp4LbJ\n",
       "zIbcfXHlszSztwK7kW4l+AtgIvCSu/+jrxOVpijgSkO1MiYzeyfpsqTbSU+OrUx65nxxXyY5gLLl\n",
       "gqrxdYHdSfdz3QTY193v7vX8pHUKuJKrqlSwkru/mHnsncCepO1JvuDus/ozy8GTyWyHgJ2Bf7n7\n",
       "pZnH1yH97Oe6+8x+zVNao4ArDcWz4ecDTwIvACe5+yvxsbcCy7r7X1VC6Kz4l8VM4ApgG+AJ4GB3\n",
       "fyk+PsHdF6hmWx5ahyvNOI70RNnlwDzSCx3uAHD3+/s4r0G3DnCLu59sZlsDl2UfdPcF8f8VaEtC\n",
       "63DlNbJXicXsdiFpZvs10pNlL5jZDtXfpx/89lTW1Wa8DHzEzO4FTgX+CFxsZl1tdCjdo4ArS4kn\n",
       "apL49cR40mY28B3gOeB3pOUFXUXWQZUTZGY2ZGb/YWYruvszwGdIf+FtCPwKOMXd7+3rZGXMVMOV\n",
       "UVUnak4k/QGfF5cjfYR0VcJC4BJ3v1w1286Kn/uVwGXufmblhGU8QfYGYJG7/z/VbMtLAVeWEn+Y\n",
       "fwXc6u4nxA1qFrj7U2a2PLDY3V/VD33nmdkewDvc/Stmtg+wGXCmu99WdZx+0ZWUAq4sxczWB04m\n",
       "vS7/v0lbSX8Y2NHd5/dxagOn8hdF5v4epJ/174DrgQ8Bz7n7+f2ZoXSaarjjXPWJmnjV2B3AFOAC\n",
       "4ELS2q02GO+g7BVkcT0z7n4B6bKv04CE9JfePX2cpnSYMtxxLHuiBvg88KK7n5t5fGPSs+OnZhfd\n",
       "S3uqPncHVgIeBU4C7gLWIP0r4wJ3v1YlhMGhDHccy/zQfw9Yn7Rdy1Fmtmo85B3A0e5+qTYU75z4\n",
       "uQfgTOBKd/8Q8FfgAHdP3P0J4MuVYNvXyUpHKeCOUzHQApwDrOnuh5BuirIusBeAu1/o7nP0Q98Z\n",
       "VeWbNYAdgFUA3P1E4M1mNjU+/vc4nii7HRwqKYwz1X+emtk04CzgsLjU60OkJ26+AryqH/bOir/o\n",
       "3uHuf46biZ9HWla4FLgY2Mvd/9zPOUr3KMMdfwKAmZ1uZhvGDWf2Ak42s+NIF9pf6+7zFWw7oyqz\n",
       "3QK4wsy2cPd7SD/7A4GfAXvHQFx9xZkMCAXccaJSQsgsQ3oAONvM1nf3ucC+wEeBm9z9Kv3Qd06l\n",
       "ZhuvHrsROAQ4xcy2ikF3W2ABsGn8Fm1xOaBUUhhHYi32WHf/ery/H+kWf3u6+91mNpl0U+tD3P2q\n",
       "Pk514MQVH+cD0939uXjl3unAru5+c1z/fDrwn+7+XB+nKl2kDHcciSWCdc3sl/H+D0gX2f/UzFZw\n",
       "91uBXUmXJkkHufvtpO2JLoxDVwG3ENc3x/XPOyrYDjZluONE9qomMzsFWIt0968dScsIv6u+8kk6\n",
       "L65COJZ0569zYs1W62zHCQXccaSqw+vJpPutnujut2lvhN4xsw2BiTHrlXFEAXecqQq6a8ZF9tIH\n",
       "ymzHHwXcAdPMD7FKByL9oZNmAyTun1rZPLzusi4FW5H+UIY7IDK7Tw0D3wRWAH4N/DF2DhCRPlOG\n",
       "OyAyWevPSLsyzCa9Vn8LWLpPmYj0hzLcAWJmbwRmuPtu8f77gUOBT1Vaa4tI/yjDLbEam4c/Drxs\n",
       "ZkfGofuBf6LNw0UKQRluSVVtYv0V0lYsZ5jZesARpBc2rAl8x90v6+dcRSSlgFtisS77DdITZBuR\n",
       "dtj9anxiX701AAAEQklEQVRsMmkQvk/rPUWKYZl+T0Dachrp5uE7A5jZpWZ2irsfGvdFEJECUQ23\n",
       "3M4G3mVmu8f7nwTebmYbZlclKLsVKQaVFErOzDYi3dbvfHc/T+UDkeJShltysR3L54EDzWwtYBi0\n",
       "7lakiJThDojYTUBrbUUKTBnu4HgZlNmKFJkyXBGRHlGGKyLSIwq4IiI9ooArItIjCrgiIj2iS3sF\n",
       "MzsG+C/gMWBZ4OvufnWHX2MT4CR336ZqfCcguPtFLT7fF4Afuvs/25jTS+6+YhPHrQJcTtp08/vu\n",
       "PqOF15gFHNbqpdbtvKYUlzJcAUiA09x9K8CAc81s+V68sLtf3GqwjQ4BJrX58k0t0XH35+Jnc263\n",
       "XqPDrykFpQxXKipt0h8ws4dJM6u7AczsAeA44LPA8sBH3f3B+NjuwP6kgeUWd/9i5QnNbE/gS6SZ\n",
       "823ZFzOz9wHfAdalRgZnZvsBe5J2r0iA6e6+wMyWA34LvBG4wswWAv/p7g/H75sMnER6xd2zwD7u\n",
       "/nR87L3AOcALwI2V99wuM9sF2BtYifSXwKfc/a+ZQz5hZjOANYCT3f3c+H3DwLeBzUl/Fs9w9ws6\n",
       "MScpJmW4shQz2xB4HXBfZjgB/o+7b+7u78oE23eSBuGpMRubYGZ7xMfeDBwPTHP3DwLPZ1/H3X/n\n",
       "7lOokcHFALYT8H53n+LuW7v7gvh98+NrPQ58JD5eCbbLxufbzd2nkrYbOjHz1D8h/fN+GnA1MHHs\n",
       "n9RSrnf37eK8fgJ8serxl+NrTgOON7PV4/g+wOL4OUwDDjCzt3ZoTlJAynAF0kxvfzPbEfgHsIu7\n",
       "L6w65vga3zedNEO9zswgze6ejY9tBoy4+1Px/rXAtk3OZ2fSem+r9dm3k2bmP4/zGQLmw2hNdGV3\n",
       "nwng7jea2fwWn78md3/GzN4NbAxsCLyp6pCr43FPmtlNwLtJs/RtgbeY2WbxuOXje7i/E/OS4lHA\n",
       "FUgz2DPc/bstft8C4NJsGSFjIUv/yd7qn+9j+be5EHig+sRctGgMz9cUMzsvfvkr4FZg7apDsu99\n",
       "GHg1fr0AONrdL+/W3KRYVFKQirHUM68GdjGzt1UGMns53AxMMbNV4tguLTzvxcB/m9lKVc+ZNZ+0\n",
       "hVD28b8CE83s49XzcfcXgCfMbMs4vgPtn3Sr+BhpHftqYDKv/UVj8TXXAd7Lknr2pcCXzGzFqvch\n",
       "A0oBVyryzqbXfMzd7yet4f7UzG40s7nEtuyxlPA1YA4wq8FrLDXu7g5cCIyY2Q3A7ErwzTgT+LWZ\n",
       "/Rb4TPy+RaTBbx8zu8nMbgQOynzPZ4FT4zw3J27406IDzWyOmWUbcx4H3A5cB/yF+Isg895eNbPr\n",
       "gSuBA9z95TjfXwCXALPi+5xTCb5NvKaUkDavERHpEWW4IiI9ooArItIjCrgiIj2igCsi0iMKuCIi\n",
       "PaKAKyLSIwq4IiI9ooArItIj/x8Rqjmvsf5niAAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113f71590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_true,y_pred)\n",
    "print('Confusion matrix, Logistic Regression')\n",
    "print(cm)\n",
    "plt.figure()\n",
    "plot_confusion_matrix(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0, 46318), (1.0, 3682)]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_rdd.groupBy(lambda x: x.label).mapValues(lambda x: len(x)).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6992"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downsample = feature_rdd.filter(lambda x: x.label != 1.0).sample(False, 0.15)\n",
    "downsample.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "balanced_classes = downsample.union(feature_rdd.filter(lambda x: x.label == 1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.0, 6992), (1.0, 3682)]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "balanced_classes.groupBy(lambda x: x.label).mapValues(lambda x: len(x)).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(train_set, test_set) = balanced_classes.randomSplit([0.8, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrmodel = LogisticRegressionWithLBFGS.train(train_set)\n",
    "predictions = lrmodel.predict(test_set.map(lambda x: x.features))\n",
    "scores_and_labels = predictions.zip(test_set.map(lambda x: x.label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(y_true, y_pred, model):\n",
    "    test_err = (y_true != y_pred).sum() / float(len(y_pred))\n",
    "    \n",
    "    print 'Test Error = ' + str(test_err)\n",
    "    print \"Accuracy: \" + str((1.0 - test_err) * 100)\n",
    "    print \"\\n\"\n",
    "    \n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    print('Confusion matrix ' + model)\n",
    "    print(cm)\n",
    "    plt.figure()\n",
    "    plot_confusion_matrix(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0.287729196051\n",
      "Accuracy: 71.2270803949\n",
      "\n",
      "\n",
      "Confusion matrix Logistic Regression\n",
      "[[1242  134]\n",
      " [ 478  273]]\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEnCAYAAADl8/suAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3X24HFWV7/HvPgGDCMibgMiLIgGvMIwj4kUgEEwGBWEc\n",
       "B1joiBdBRUSEYZBxRDSJI8NEwIgXYVSEKCPiUh5RGITgSQgSdYygFwfFkPAOw1tCIGCCIan7R+0+\n",
       "VJrTVd2n36r6/D48/dC9a3fVPg29zjqrdu0KSZIgIiLdN9TvAYiIjBcKuCIiPaKAKyLSIwq4IiI9\n",
       "ooArItIjCrgiIj2igDtAQgjvDSH8JoTweAjh3hDCvBDCrl061qHxGA/GY07o0nEmxp9j527sv5dC\n",
       "CB8MIfysA/sZmM9kvNmg3wOQzgghHA58GbAkSW6JbX8FPNilQ34c+HqSJOd2af8AJEnyPPD2bh6j\n",
       "avSZVJcy3MHxOeDztWALkCTJb+KXE4AQwnYhhO+FEO6P2enFIYRNMtunhBAeCiG8J4SwKITwRAjh\n",
       "yhBCyPQ5N4TwIDAN+GTMcOdnBxJCWBdC2CXzekYI4Yq6PieEEH4XQnggPs6o2/76uO8H6/eX6bNP\n",
       "COGW2OeuEMI/1G2fEcd/dgjh9/HnOb2Fz5QQwn0hhNNCCPeEEC4PIRwZQng4hPD5TJ8tQghfDiHc\n",
       "GUL4nxDCHSGEA+r282PgAmCfzM/1v+v6zImf70XxM3k4hLBXs59J/O/3SAhh2/h6KIQwP4TwkVZ+\n",
       "ZumiJEn0qPgDmAD8GfjLnD5DwCLgX4EAbARcCXwv02cKsCb2mQhsBSwHpo6yv/nACQ2OtQ7YJfN6\n",
       "OnBF5vUkYBXwxkzbK3LGvt7+Ytv2cWxHxNevAe4APpbpMwNYAbw3vn478DwwsYXP9l7gG8BucRxn\n",
       "Au8EHs30mQgcAgzF1zOBO0fZ13HAz3KONQd4HDgxvt60lc8ktn8WuCHzuX+n3/9/6vHiQxnuYNiW\n",
       "tDy0MqfPPsDOwNlJajVwCnBUCGHrTL9HkyQ5K0mS55MkWQb8HtixwT5Dg/aifk+TBr5DQgivAUiS\n",
       "5Lkm91XzAeDmJEmuje9/mDTAnFLX78dJklwVn/8M2BB4dYvH+i5wX3z+PeABYMvaxvhZzU2SZF1s\n",
       "+gGw+yj7Kfq8EsCTJPl63G/ef89GvgBsEEL4NnA0cOIY9iFdooA7GGpfzG1y+uwM3J8JCiRJshx4\n",
       "Km5rZA2N/z9pdiGO9folSfI48FZgJ2BeCOH2EMLUJvdVsxOwtK5tCS/9WUaCXJIka+LTsfx/v67u\n",
       "ebbMEkIIp8fyxq3AV4GhEMJYjvPUGN4zIklT208CxwKzxvCLTLpIAXcAxEzo9+SfSHkAeF0IYcNa\n",
       "QwhhG2AL4P5OD4n1/996SWaXJMniJEn+MUmS3Un/9L82hDCxhWM8wEuzyDfwYiZaG0cvfAI4Cjg6\n",
       "SZIDSEsHfRGD/PnAN4GZIYTN+zUWeSkF3MFxLnBGCGFqeNHkzImV/yINRueGECaEEDYmzcSuTpLk\n",
       "yYJ9N/pTuFH7Q8CeAHFa2vuoC351J3xeRlpiWFswjqwrgINCCH8X97cTae304ibG12k7AI8Cj8fy\n",
       "zHmxfcO6fk8Bk0IIm8b/Pq+p296J8Z4NLE+S5COkpZBLO7BP6RAF3AGRJMl3SDOtC0hPvNwP/BOw\n",
       "SdyeAEeQ/il+L2lGvAw4oX5Xo+2+0WEbtH8SOC+E8JP4fG52YwhhM8DjjIgHSaeYHZokyQvNHidJ\n",
       "kkdIT1SdHkJ4CLiJ9OTWJXXvq39vO1lv0uD5BcDWwCPAfwJfB14gPZGXdQNwO3APcBcvra+ONt5m\n",
       "xgJACOEg0lLCh2PTdGDHEMJJTe5Tuiyk30MREek2ZbgiIj2igCsi0iMKuCIiPaKAKyLSI1q8pgOG\n",
       "h4d15lGkCVOnTu3YVL1WvnedPG47FHA75PBP/rDfQ2jb2pUPM2HT+plM1fTUoov6PYSOuGXBzRx4\n",
       "0JR+D6MjFi4Y7vg+Dz/zR4V9rjvv3R0/7lgp4IpIdYVSJK5NU8CVEYOS3Q6SQcluu2ZMy1WkzGwy\n",
       "6UUrC9z9zNj276SXjA8Bx7v7PbF9GumFJADT3X1eXnsjOmkmItUVQvGjsYmkl8SPcPeT3P1g0svE\n",
       "a0F4KL4+JD5mNGo3s9wDKuDKiLUrH+73EKTOLQtu7vcQym1oQvGjAXf/KemayqNZSbrGNKTrNy92\n",
       "91XuvgpYamaTRmsHcm9ppZKCiFRXGyWFAicAF8bnWwIrzGx2fP006eL8oUH73Y12qoArI1TDLR/V\n",
       "cAt04aSZmR0B/NHd74pNy4DNgZNJg+zFwJOkFYLR2htSSUFEqquNkkK0XsQ2s72Bg9z9y5nmpaS3\n",
       "WKqZ5O5LctobD7fwB5JxQzXc8lENt0AYKn40YGafIj0BdoSZfS02fx/Yx8zmm9lXANx9LenJsZtI\n",
       "lxqdkdeeRyUFEamuNkoK7j4LmFXX9pK7Q8f2udSt65zX3ogCroxQDbd8VMMt0L2TZl2hgCsi1TWh\n",
       "sEZbKtX69SBdpRpu+aiGW6CNGm4/KMMVkerSWgpSVarhlo9quAVKlsEWUcAVkeoqnmdbKtX69SBd\n",
       "pRpu+aiGW6C9xWt6ThmuiFSXSgpSVarhlo9quAVKlsEWUcAVkepSDVeqSjXc8lENt4Dm4YqI9EjJ\n",
       "AmoRBVwZoRpu+aiGW6BiJQUFXBGproqdNKtWPi5dpRpu+aiGW0A1XBGRHqlYhquAKyNUwy0f1XDz\n",
       "DQ2VK4MtooArItVVrQRXNVx5kWq45aMabr4QQuGjTJThikhllS2gFlHAlRGq4ZaParj52qnhmtlk\n",
       "4AJggbufGdumAdNjl+nuPi+2zwF2B1YDc9z9W3n9G1HAFZHqai/BnQicC+wHYGZDpLc9nxa33wjU\n",
       "AmgCHOPuD9TePFp/M5vv7kmjA6qGKyNUwy0f1XDztVPDdfefAsszTZOAxe6+yt1XAUvNbFL2cHW7\n",
       "eEl/YNe88SrDFZHK6nANd0tghZnNjq+fBrYC7gZWAlea2XLgdHdfUtB/VAq4MkI13PJRDTdfh+fh\n",
       "LgM2B04mzWYvBp4EcPdTAczsTcB5wHvy+jeigCsi1dV+gpvdw1Jgt8zrSTGTzVoNrGmh/3pUw5UR\n",
       "quGWj2q4+dqp4ZrZp4AZwBFm9jV3X0t6EuwmYG7cVut7lZktAM4HzgTI69+IMlwRqax2SgruPguY\n",
       "Vdc2lzR41vd9b4N9jNq/EQVcGaEabvmohlugWtc9KOCKSHVV7Uoz1XBlhGq45aMabj6tpSAi0iNa\n",
       "nlEqSzXc8lENt0C5EthCCrgiUlllKxkUqVY+Ll2lGm75qIabTzXcCjKzy4EpwCnu/p9F7SJSDmGo\n",
       "XAG1iDJcwN2PB+Y02z6oVMMtH9Vw8ynDzTCzO0gve9sfmOfuZ8X244CPkpa8L3P3bxTsZ5G77zPK\n",
       "87uAhcCewHXu/i+x/WTgWGAtcLm7Xxbb3wl8jnRtyy+5+9Ud/pFFpIfKFlCLdDvD3QL4ImnAPQLA\n",
       "zLYGTgIOBA4A3m9mY02tNgJOA94GHGpm28b2o4HD3H1yJtgOAf8G/DVwEHCamU0c43EHkmq45aMa\n",
       "bj5luOt71N0fAzCz1bFtF+DX7v5CbP8l6UK+Y/m2P+Huz8b93AbsDDwGfAD4aAzu17j7QmBrYAfg\n",
       "uvjezYHtgXvH8oOJSP9VrYbbj5Nm9wBvMbMNSf+03w+4sOA9QwBmtjGwcaZ9ezPbknTh3zeTlgtw\n",
       "94eAWWa2EWnJYW/gCeAPwLvd/ZnO/TiptSsfHqmB1jJFve7f61sW3DxS/6xliVV8feBBU0o1nnZe\n",
       "T6DzypbBFglJ0vD2O20zs1+5+1tHeZ6t4X7T3S8t2M8lwHPAs8BR7r5nbL8TWATsAVzh7l+J7bOB\n",
       "NwGbAV/NlBX2B84hDfSPuPv7M8eYDhwJfMvdLyhqzxoeHk4O/+QPW/pspLueWnRRv4cgdRYuGGbq\n",
       "1Kkdi5DDw8PJidf/ubDf1w97WUeP246uBtxuy55A66dBCbjZLL3qBiXgZjP1qutGwD3phjWF/f79\n",
       "nRuWJuBWfR5udX9biEjbKlZRqHbArZUopDMGJbsdJIOS3XZL1Wq4lQ64IjK+VSze6kozeZHm4ZaP\n",
       "5uHmmzAhFD7KRBmuiFRWuyUFM/so8EHSGVAnu/vdZjYNmB67THf3ebHvqO2tUIYrI1TDLR/VcPOF\n",
       "UPxoJM7rP97d3wa8D/hXMwukd+I9JD5mxL5D9e2xb0uU4YpIZbWZ4QZgw3iJ/wpgO9KrXhe7+yoA\n",
       "M1tqZpNIk9P12oFdgbtbOaACrowYpHm4g2KQ5uF2w1Abl/a6+3Nm9q/AT4CVpGu/bAesiBdPQXoV\n",
       "61akwXm0dgVcERkfmstwG0/XjysGXg1gZrcDj5Kus3IyaZC9GHiSNMMdrb0lquHKCGW35aPsNl87\n",
       "NdwsMzsM+C2wBNgts2mSuy8BljZob4kyXBGprHYzXDP7JrA76SyFY919nZnNJF3HG+JJM3dfO1p7\n",
       "qxRwZYRquOWjGm6+dmq4AO7+oVHa5gJzm21vhQKuiFTWuLjSLC6XKANG2W35KLvNNzB3fDCzvRm9\n",
       "+BGAfbs2IhGRJrVbUui1vJLCMPCbBttWdGEs0meq4ZaParj5SpbAFsoLuL9z94N7NhIRkRaVrWRQ\n",
       "JC/gTs/ZJgNI2W35KLvNV7F42zjgjmUlHBGRXhoaqta1W02P1sy26OZApP+0Hm75aD3cfJ260qxX\n",
       "CufhxtkKlwETgTfEZcoudfcTuj04EZE8VavhNpPhfgn4G+B/ANx9HekSZjJgVMMtH9Vw81Utw20m\n",
       "4Cbufn9d28RuDEZEpBVDQ6HwUSbNXNr7pJkdCgQz2xT4PI3n50qFaR5u+Wgebr6hsqWwBZrJcD8G\n",
       "HAfsSbrY7sbAGd0clIhIM6pWUijMcN39CeC9PRiL9Jmy2/JRdpuvaifNml4tzMw2BF5w98aLS4qI\n",
       "9NCEktVoixSWFMzsLWZ2G/AI8LiZzTWz13d/aNJrmodbPpqHm69qJYVmariXAme5+6vc/VXAJcAV\n",
       "3R2WiEix0MQ/ZdJMwP2Tu99Ye+HuPwTWdm9I0i+q4ZaParj5JgyFwkeZNBNwbzSzA2svzOxdwE+7\n",
       "NyQRkeZUraSQtwD5s6QLkAfgbDNbHTdtDPwJmNn94UkvaR5u+Wgebr525uGa2WbAjzJNb3b3V5rZ\n",
       "HNIbS64G5rj7t2L/aby4iuL0sSzwlbda2Cat7kxEpJfayWDd/RngYAAz2wv4RNyUAMe4+wO1vnEN\n",
       "mZnAtNh0o5nNb3XWlm4iKSOU3ZaPstt8Hbx091TgK5nX9TueBCx291UAZrYU2JX0YrCmNbNa2EbA\n",
       "e4BXx0EEYFt3P7OVA4mIdFonLu01s62AHd39jti0ErjSzJYDp7v7EmBLYIWZzY59nga2osWA28xJ\n",
       "s6uBY4BDgdcDRwIvtHIQqQbNwy0fzcPNF5p4NOFE4Ou1F+5+qrvvD3wWOC82LwM2B84CPhOfP9nq\n",
       "eJsJuNu7+98C3ycNvocCf9HqgUREOq3d26Sb2QbA4cAPR9m8GlgTny8FdstsmxQz35Y0E3Afj/9e\n",
       "DOzn7iuAHVs9kJSfarjloxpuvg7Mw/1b4Nq4zjcAZnaVmS0AzgfOBHD3taQnzW4C5gIzxjLeZk6a\n",
       "/dLMXgXcCpwfl2r8/VgOJiLSSe2WcN39B6O0jbpYl7vPJQ22Y9bMamEjd+81s4NI56f9tp2DSjlp\n",
       "Hm75aB5uvoFdLQzA3Z8Dbu/SWEREWlKyK3cLaR6ujFB2Wz7KbvMN4h0fXsLMLun0QEREWjUUQuGj\n",
       "TPLWUtib9BK3egHYt2sjkr5RDbd8VMPNV7J4WiivpDBM45tFrujCWEREWlK2u/IWyQu4v3P3g3s2\n",
       "Euk7Zbflo+w2X9lKBkXyAu70nG0iIn1XsXibuzxjy2s9jmf+7c/2ewht+92ihfzFPvv3exgd8chT\n",
       "q/o9hI745cJb2Hf/A4s7jlMDPQ9XRKRMJijgSlUNSnY7SJTd5qvYOTMFXBGprqoF3DFd+CCD6XeL\n",
       "FvZ7CFLnlwtv6fcQSq3d5Rl7TRmuiFTWhIqljAq4MkI13PJRDTffIM3DFREptYoluJUbr3SRarjl\n",
       "oxpuvhCKH2XSzF17twZmATu4+zvMLAAfd/eLuj46EZEcTdxCp1SayXC/AfwE2AjA3RNg1FtQSLWp\n",
       "hls+quHmGwrFjzJpJuBuGe/7sy7TplKEiPTdwKyHm7HWzF5de2Fm70HLMw6kQVpLYVBoLYV87cZT\n",
       "M9sBuII0Fv7K3c8ws2m8uHjX9Nq6Mo3aW9FMwD0TuAHY0cxuByYC7271QCIindaBtRTOBz7j7j8H\n",
       "MLMh0tuhT4vbbwTmjdZuZvNjibVphaUBd78N2AeYAhwL7OXuS1o5iFSDstvyUXabr50arplNAF5f\n",
       "C7bRJGCxu69y91XAUjObNFo7sGur421qHq67/xm4o9Wdi4h0U1MnxRrnoK8CNjKza4DNgP8LPAqs\n",
       "MLPZsc/TwFaktxYbrf3uVsbbzLSwM0ZpTtz9S60cSMpPNdzyUQ03X1PTwtY13LKMNHAeCUwAFgIf\n",
       "AjYHTiYNshcDT5JWA0Zrb0kzsw02BTbJPA4G3tzqgUREOq2dCx/cfQ3wILBd/Cv+eWAJsFum26RY\n",
       "Ql3aoL0lhRmuu8/IvjazDUkLzTJglN2Wj7LbfB2Y9vUp4Btm9krA3f1PZjYTuClun0G6Ye1o7a1q\n",
       "eS0Fd19jZtuM5WAiIp3U7oUN7v4AcFhd21xg7ih9R21vRTM13GvrmrYBHm7noFJOquGWj2q4+Qbx\n",
       "FjsX1L1e7u6asSAifVexeNtUDffmHoxDSkDZbfkou81XtrUSihTOUjCznXsxEBGRVlVtLYVmpoX9\n",
       "uOujkFLQerjlo/Vw800YCoWPMmmmhruq66MQERmDkiWwhZoJuJea2QXAOdlGd1/enSFJv6iGWz6q\n",
       "4ear2jqxzQTcs0mvRv67TFsC7NKVEYmINKlst0Ev0swshdf2YBxSApqHWz6ah5tvEOfhioiUUrXC\n",
       "bU4JxMw+3cuBSP8puy0fZbf5qnbX3rya82E520RE+m5CCIWPMskrKWxgZls22qhZCoNHNdzyUQ03\n",
       "3yCdNPsr4LYG2zRLQUT6rlrhNj/gLnL3yT0bifSdstvyUXabb5AyXBGRUitbjbZI3kmzC3s2CikF\n",
       "raVQPlpLIV9o4lEmDTNcd/9BLwciItKqiiW4KinIi1TDLR/VcPMNlS6HzaeAKyKV1Yn1bs1sIrAY\n",
       "mOXuF5vZHGB3YDUwx92/FftNA6bHt01393ktj7ft0crAUA23fFTDzdehK81OYv0psAlwjLsfnAm2\n",
       "Q8BM4JD4mGFmLUd7BVwRqawhQuEjj5ltDPw18CPWP8dW/8ZJwGJ3X+Xuq4ClwK6tjlclBRmhGm75\n",
       "qIabrwMVhVOBi4BtM20rgSvNbDlwursvAbYEVpjZ7NjnaWAr4O5WDqYMV0Qqq517mpnZK4ED3P0G\n",
       "Mhmtu5/q7vsDnwXOi83LgM2Bs4DPxOdPtjzeVt8gg0s13PJRDTffUCh+5Ngf2MjMvktaxz3ezPbI\n",
       "bF8NrInPlwK7ZbZNiplvS1RSEJHKCk1MC0satLv79cD1AGZ2HPAKd7/TzK4CXk1aWvh47LvWzGYC\n",
       "N8W3zxjTeJOk0XDGDzO7HJgCnOLu/1nUXm94eDj50zZ7dnuY0oI9ttus30OQOvfe8XOmTp3asYmz\n",
       "w8PDyYQd/qqw39qHftPR47ZDJQXA3Y8H5jTbLiLl0GZJoecqUVIws0Xuvs8oz+8CFgJ7Ate5+7/E\n",
       "9pOBY4G1wOXufllsfyfwOdK/Mr7k7lf3/IcpMa2HWz5aDzdfMyWFMql6hrsRcBrwNuBQM6tN7Tga\n",
       "OMzdJ2eC7RDwb6Rz7g4CTotXmIhIRVXtFjuVyHBzPOHuzwKY2W3AzsBjwAeAj5rZ1sA17r4Q2BrY\n",
       "AbguvndzYHvg3p6PuqSU3ZaPstt8VVuesSoBdwhGrgrZONO+fbwN0NPAm0nLBbj7Q8AsM9uItOSw\n",
       "N/AE8Afg3e7+TKcHmP1zvDa9Sq/793rllq8YCVa1qVV63d/X227a+XBTrXBbkVkKZnYJ8BzwLHCU\n",
       "u+8Z2+8EFgF7AFe4+1di+2zgTcBmwFczZYX9gXNIa7iPuPv7M8eYDhwJfMvdLyhqzxqUWQqDVMMd\n",
       "lFkKg1TD7cYshVe8du/Cfs/dd1tpZilUIuA2kj2B1k8KuOWjgFs+3Qi4m7z2LYX9nr3v16UJuFUp\n",
       "KTRS3d8WJTQowXaQDEqw7ZayTfsqUumA6+5v7fcYRKSPKhZwqz4tTDpIaymUj9ZSyBea+KdMKp3h\n",
       "isj4VrFZYQq48iLVcMtHNdx8CrgiIj1StpJBEdVwZYRquOWjGm4+XdorItIjZQuoRRRwZYRquOWj\n",
       "Gm6+qpUUFHBFpLKqluGqhisjVMMtH9Vw84UmHmWiDFdEKitULMVVwJURquGWj2q4+SoWbxVwRaS6\n",
       "2om3ZvYFYD9gHXCiu99jZtOA6bHLdHefF/uO2t4q1XBlhGq45aMaboE2irjufra7v500kH7KzAIw\n",
       "EzgkPmbAyO251muPfVumDFdEKmuoMzWFfUnvBjMJWOzuqwDMbKmZTSJNTNdrB3YF7m71QAq4MkI1\n",
       "3PJRDTdfu+HWzG4hvd/hZGA3YEW8Ywykt+7aKh5mtPaWA65KCiJSXW3OC3P3A4EPAt8GlpHeXPYs\n",
       "4DPx+ZM57S1TwJURquGWj2q4+Tq0Hu6jpH/tLyHNcmsmufsSYGmD9pappCAildXOLXbM7Huk5YQ/\n",
       "A6e4+zozmwncFLvMAHD3taO1j0WlbyJZFoNyE8lBMig3kRwk3biJ5Gv32q+w330dPm47lOGKSGVV\n",
       "bfEa1XBlhGq45aMabr6hUPwoE2W4IlJdJQuoRRRwZYTm4ZaP5uHmq1pJQQFXRCqraovXqIYrI1TD\n",
       "LR/VcPOphisi0jMli6gFFHBlhGq45aMabr6qlRQUcEWksioWb1XDlRephls+quHmGwqh8FEmynBF\n",
       "pLrKFU8LKeDKCNVwy0c13HwVi7cKuCJSXSWrGBRSDVdGqIZbPqrh5gshFD7KRBmuiFRWucJpMQVc\n",
       "GaEabvmohpuvZAlsIQVcEamssk37KqIaroxQDbd8VMMdLMpwRaSyKpbgKuDKi1TDLR/VcPO1sx6u\n",
       "mU0GLgAWuPuZsW0OsDuwGpjj7t+K7dOA6fGt09193liOqYArIpXV5vKLE4FzgeydKBPgGHd/oNZg\n",
       "ZkPATGBabLrRzOa7e8t34FUNV0aohls+quEWCE08GnD3nwLLG+w1axKw2N1XufsqYCmw61iGqwxX\n",
       "RCqrC7fYWQlcaWbLgdPdfQmwJbDCzGbHPk8DWwF3t7pzBVwZoRpu+aiGm6/TJ83c/VQAM3sTcB7w\n",
       "HmAZsDlwMmn2ezHw5Fj2r4ArIpXVgYDbaA+rgTXx+VJgt8y2STHzbZlquDJCNdzyUQ03X2jin0bM\n",
       "7FPADOAIM/tabPuemS0AzgfOBHD3taQnzW4C5sb3jG28SdLyiTapMzw8rA9RpAlTp07tWBGgle9d\n",
       "J4/bDgVcEZEeUUlBRKRHFHBFRHpEAVdEpEcUcEVEekQBV6TCzExz6StEAVfaZmavN7MN+z2O8cbM\n",
       "PgMcZmab9Hss0hwFXGmLmX0T+DKwn5lN7Pd4xgsz+xhwDLA3sL+CbjUo4MqYmdn2wEPAD0iXuNvX\n",
       "zDaK20ox0XyAfdfd9wL+CzgAmGxmm/V5TFJA9R8ZM3d/hLgos5l9GJhC+kt8PvAy4Pm+DW7wrQRw\n",
       "9+vNbGPSX3hPmlkCBHdf1NfRyaiU4UrLzGzTzPMNANz9UuAB4C1mdgpwjpm9sk9DHEhmdqSZTYL0\n",
       "+n4zmxCf/wC4A/ggcDnVu3v4uKFLe6UlZvYV4FHgSne/L7ZNiAt8YGYfAo4ELnT3G/s20AFjZrWl\n",
       "AmcDw+5+V2zPfvbvAJa5+6/7N1LJo4ArTTOzQ4BTgV8D9wPzM0F3A3d/IT7f1t0fM7MwltuQyEuZ\n",
       "2f7AfcD7SJcOHHb3PzToGwD02ZePAq60zMx2Bj5MGnTnufs9Dfop4LbJzIbcfV3tszSz1wHvJ73r\n",
       "wHdJ78v1rLs/3deBSlMUcKXQaBmTme1BOi3pDtKTY5uRnjlf15dBDqBsuaCufSfgWNJbv7wFONHd\n",
       "F/d6fNI6BVzJVVcq2NTdV2a27QEcBxwC/IO739yfUQ6eTGY7BBwF/Nndr8ls35H0s1/o7vP7NU5p\n",
       "jQKuFIpnw+cAjwPPAOe5+5/ittcBL3P3P6qE0FnxL4v5wHXAwcBjwKnu/mzcvqG7r1HNtjo0D1ea\n",
       "8QXSE2XXAotIL3S4E8Dd7+3juAbdjsCv3P18MzsQ+FF2o7uvif9WoK0IzcOVl8heJRaz2xdIM9vP\n",
       "kp4se8bMDq9/n7747anNq814DniXmS0BLgR+C1xtZrv2fHDSEQq4sp54oiaJzyfGkzYLgC8CTwG/\n",
       "IC0v6CqyDqqdIDOzITP7WzPbxN2XAR8i/YW3O/B9YPZY7xgr/acaroyoO1Ezi/QLvihOR3oX6ayE\n",
       "F4Afuvu1qtl2Vvzcrwd+5O6X1E5YxhNkrwLWuvv/U822uhRwZT3xy/x94DZ3PzcuULPG3Z8ws5cD\n",
       "69z9eX3pO8/MPgC80d0/bWYfAd4KXOLut9f10y+6ilLAlfWY2S7A+aTX5f8zsAx4J3CEu6/u49AG\n",
       "Tu0viszrD5B+1r8A5gHvAJ5y9zn9GaF0mmq441z9iZp41didwGTgCuAq0tqtFhjvoOwVZHE+M+5+\n",
       "Bem0r4uAhPSX3t19HKZ0mDLccSx7ogb4BLDS3S/LbN+L9Oz4hdlJ99Keus/dgU2Bh4HzgLuAbUj/\n",
       "yrjC3eeqhDA4lOGOY5kv/ZeBXUhv13K2mW0Zu7wRmO7u12hB8c6Jn3sALgGud/d3AH8ETnb3xN0f\n",
       "Az5VC7Z9Hax0lALuOBUDLcClwLbufhrpoig7AccDuPtV7n6LvvSdUVe+2QY4HNgCwN1nAa8xs4Pi\n",
       "9v+J7Ymy28GhksI4U//nqZlNAb4OnBGner2D9MTNp4Hn9WXvrPiL7o3u/t9xMfHLScsK1wBXA8e7\n",
       "+3/3c4wE7ROLAAAGOElEQVTSPcpwx58AYGZfNbPd44IzxwPnm9kXSCfaz3X31Qq2nVGX2e4HXGdm\n",
       "+7n73aSf/ceB7wAnxEBcf8WZDAgF3HGiVkLITEO6D/iGme3i7guBE4G/AX7u7j/Rl75zajXbePXY\n",
       "rcBpwGwzOyAG3UOANcA+8S1a4nJAqaQwjsRa7Ex3/1x8fRLpEn/HuftiM9ubdFHr09z9J30c6sCJ\n",
       "Mz7mAFPd/al45d5Xgfe6+y/j/OevAn/v7k/1cajSRcpwx5FYItjJzL4XX/876ST7/zCzV7j7bcB7\n",
       "SacmSQe5+x2ktye6Kjb9BPgVcX5znP98hILtYFOGO05kr2oys9nA9qSrfx1BWkb4Rf2VT9J5cRbC\n",
       "TNKVvy6NNVvNsx0nFHDHkbo7vJ5Put7qLHe/XWsj9I6Z7Q5MjFmvjCMKuONMXdDdNk6ylz5QZjv+\n",
       "KOAOmGa+xCodiPSHTpoNkLh+am3x8IbTuhRsRfpDGe6AyKw+NQH4F+AVwI+B38Y7B4hInynDHRCZ\n",
       "rPU7pHdlWEB6rf5+sP59ykSkP5ThDhAz2w64wN3fH1+/HTgdeF/t1toi0j/KcCtslMXDHwWeM7Oz\n",
       "YtO9wCq0eLhIKSjDrai6Raw/TXorlovNbGfgTNILG7YFvujuP+rnWEUkpYBbYbEu+3nSE2R7kt5h\n",
       "9zNx296kQfgezfcUKYcN+j0AactFpIuHHwVgZteY2Wx3Pz2uiyAiJaIabrV9A/hLMzs2vj4GeIOZ\n",
       "7Z6dlaDsVqQcVFKoODPbk3RZvznufrnKByLlpQy34uLtWD4BfNzMtgcmgObdipSRMtwBEe8moLm2\n",
       "IiWmDHdwPAfKbEXKTBmuiEiPKMMVEekRBVwRkR5RwBUR6REFXBGRHtGlvYKZzQD+D/AI8DLgc+5+\n",
       "Q4eP8RbgPHc/uK79SCC4+w9a3N8/AF9z91VtjOlZd9+kiX5bANeS3nTzK+5+QQvHuBk4o9VLrds5\n",
       "ppSXMlwBSICL3P0AwIDLzOzlvTiwu1/darCNTgM2bvPwTU3Rcfen4mdzWbeO0eFjSkkpw5Wa2m3S\n",
       "7zOzB0kzq8UAZnYf8AXgw8DLgb9x9/vjtmOBj5EGll+5+z/WdmhmxwH/RJo53549mJm9DfgisBOj\n",
       "ZHBmdhJwHOndKxJgqruvMbONgJ8C2wHXmdkLwN+7+4PxfXsD55Fecbcc+Ii7Pxm3vRm4FHgGuLX2\n",
       "M7fLzI4GTgA2Jf0l8D53/2Omy9+Z2QXANsD57n5ZfN8E4N+AfUm/ixe7+xWdGJOUkzJcWY+Z7Q68\n",
       "Ergn05wA/8vd93X3v8wE2z1Ig/BBMRvb0Mw+ELe9BjgHmOLufw2syB7H3X/h7pMZJYOLAexI4O3u\n",
       "PtndD3T3NfF9q+OxHgXeFbfXgu3L4v7e7+4Hkd5uaFZm198m/fN+CnADMHHsn9R65rn7oXFc3wb+\n",
       "sW77c/GYU4BzzGzr2P4RYF38HKYAJ5vZ6zo0JikhZbgCaab3MTM7AngaONrdX6jrc84o75tKmqHe\n",
       "ZGaQZnfL47a3AsPu/kR8PRc4pMnxHEVa7221PvsG0sz8yjieIWA1jNREN3P3+QDufquZrW5x/6Ny\n",
       "92Vm9iZgL2B34NV1XW6I/R43s58DbyLN0g8BXmtmb439Xh5/hns7MS4pHwVcgTSDvdjdv9Ti+9YA\n",
       "12TLCBkvsP6f7K3++T6W/zdfAO6rPzEXrR3D/ppiZpfHp98HbgN2qOuS/dknAM/H52uA6e5+bbfG\n",
       "JuWikoLUjKWeeQNwtJm9vtaQWcvhl8BkM9sith3dwn6vBv7ZzDat22fWatJbCGW3/xGYaGbvqR+P\n",
       "uz8DPGZm+8f2w2n/pFvNu0nr2DcAe/PSXzQWj7kj8GZerGdfA/yTmW1S93PIgFLAlZq8s+mjbnP3\n",
       "e0lruP9hZrea2ULibdljKeGzwC3AzQXHWK/d3R24Chg2s58BC2rBN+MS4Mdm9lPgQ/F9a0mD30fM\n",
       "7OdmditwSuY9HwYujOPcl7jgT4s+bma3mFn2xpxfAO4AbgL+QPxFkPnZnjezecD1wMnu/lwc73eB\n",
       "HwI3x5/zllrwbeKYUkFavEZEpEeU4YqI9IgCrohIjyjgioj0iAKuiEiPKOCKiPSIAq6ISI8o4IqI\n",
       "9IgCrohIj/x/tMyZlfaM86UAAAAASUVORK5CYII=\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1142379d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_true = np.array(scores_and_labels.map(lambda lp: lp[1]).collect())\n",
    "y_pred = np.array(scores_and_labels.map(lambda lp: lp[0]).collect())\n",
    "    \n",
    "evaluate(y_true, y_pred, \"Logistic Regression\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `spark.ml`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator, Evaluator\n",
    "from pyspark.ml.classification import LogisticRegression, RandomForestClassifier, GBTClassifier\n",
    "from pyspark.ml.feature import StandardScaler, VectorAssembler, StringIndexer, VectorIndexer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# lrmodel = LogisticRegressionWithLBFGS.train(train_set)\n",
    "rf = RandomForestClassifier(featuresCol=\"features\", labelCol=\"indexed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pyspark.ml.classification.RandomForestClassifier"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stringIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexed\")\n",
    "si_model = stringIndexer.fit(train_set.toDF())\n",
    "index_df = si_model.transform(train_set.toDF())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+-------+\n",
      "|            features|label|indexed|\n",
      "+--------------------+-----+-------+\n",
      "|[56.0,32.0,-4831....|  0.0|    0.0|\n",
      "|[60.8886602759,0....|  0.0|    0.0|\n",
      "|[56.0,72.0,132687...|  0.0|    0.0|\n",
      "|[136.0,208.0,-157...|  0.0|    0.0|\n",
      "|[32.0,64.0,73289....|  0.0|    0.0|\n",
      "|[48.0,32.0,145049...|  0.0|    0.0|\n",
      "|[32.0,72.0,28480....|  0.0|    0.0|\n",
      "|[40.0,16.0,51529....|  0.0|    0.0|\n",
      "|[32.0,8.0,90504.0...|  0.0|    0.0|\n",
      "|[60.8886602759,66...|  0.0|    0.0|\n",
      "|[40.0,40.0,68819....|  0.0|    0.0|\n",
      "|[32.0,40.0,233151...|  0.0|    0.0|\n",
      "|[96.0,48.0,-12529...|  0.0|    0.0|\n",
      "|[120.0,120.0,1065...|  0.0|    0.0|\n",
      "|[48.0,96.0,-96118...|  0.0|    0.0|\n",
      "|[32.0,56.0,109776...|  0.0|    0.0|\n",
      "|[24.0,40.0,-55352...|  0.0|    0.0|\n",
      "|[72.0,56.0,296858...|  0.0|    0.0|\n",
      "|[48.0,56.0,-50004...|  0.0|    0.0|\n",
      "|[60.8886602759,66...|  0.0|    0.0|\n",
      "+--------------------+-----+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "index_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rf_model = rf.fit(index_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = rf_model.transform(test_set.toDF())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+----------+\n",
      "|            features|label|prediction|\n",
      "+--------------------+-----+----------+\n",
      "|[24.0,24.0,-35327...|  0.0|       0.0|\n",
      "|[0.0,0.0,-2526.71...|  0.0|       0.0|\n",
      "|[64.0,56.0,350274...|  0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|       0.0|\n",
      "|[32.0,32.0,113571...|  0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|       0.0|\n",
      "|[32.0,16.0,83453....|  0.0|       0.0|\n",
      "|[16.0,0.0,75306.4...|  0.0|       0.0|\n",
      "|[40.0,32.0,318276...|  0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|       0.0|\n",
      "|[8.0,0.0,153616.0...|  0.0|       0.0|\n",
      "|[40.0,56.0,36570....|  0.0|       0.0|\n",
      "|[32.0,48.0,941976...|  0.0|       0.0|\n",
      "|[60.8886602759,66...|  0.0|       0.0|\n",
      "|[60.8886602759,66...|  0.0|       0.0|\n",
      "|[40.0,32.0,-15022...|  0.0|       0.0|\n",
      "|[80.0,104.0,-2040...|  0.0|       0.0|\n",
      "|[24.0,104.0,15944...|  0.0|       0.0|\n",
      "|[40.0,24.0,294188...|  0.0|       0.0|\n",
      "+--------------------+-----+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.76      0.99      0.86      1376\n",
      "        1.0       0.94      0.42      0.59       751\n",
      "\n",
      "avg / total       0.82      0.79      0.76      2127\n",
      "\n",
      "Test Error = 0.212505876822\n",
      "Accuracy: 78.7494123178\n",
      "\n",
      "\n",
      "Confusion matrix Random Forest\n",
      "[[1356   20]\n",
      " [ 432  319]]\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEnCAYAAADl8/suAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XvcZXPd//HX9xrMOBa3nHIocuuOh9Tgp2EYjVsRuYUP\n",
       "3fET5RAiye1OylSkCUm3QyFGSnziQZFTXcZp5G6iflNKkjM55DhkmMP6/bG++5o1295r733t01r7\n",
       "ej977Ed7f9e61vruba7P/l6f9V3fT0iSBBER6b6hfndARGSsUMAVEekRBVwRkR5RwBUR6REFXBGR\n",
       "HlHAFRHpEQXcARJC2CeE8LsQwjMhhIdCCDeHEN7VpXPtFM/xWDznuC6dZ3x8H+t14/i9FEL4ZAjh\n",
       "9g4cZ2A+k7FmqX53QDojhLAL8B3AkiS5Lba9D3isS6c8HDgvSZJTunR8AJIkeR34YDfPUTb6TMpL\n",
       "I9zB8RXga5VgC5Akye/iLycAIYQ1QgiXhxAeiaPTc0IIK2S2TwkhPB5C2D2EMDuE8GwI4dIQQsjs\n",
       "c0oI4TFgB+ALcYQ7M9uREMKiEML6mdfTQgiXVO1zYAjhDyGER+PjmKrtG8RjP1Z9vMw+W4QQbov7\n",
       "3BdC+FzV9mmx/yeEEP4U38/RLXymhBAeDiEcFUJ4MIRwUQhhjxDCEyGEr2X2WTmE8J0Qwr0hhL+H\n",
       "EOaEELapOs7PgdOBLTLv6/9U7TMjfr5nxc/kiRDCps1+JvG/35MhhNXj66EQwswQwkGtvGfpoiRJ\n",
       "9Cj5AxgHvAG8N2efIWA28A0gABOAS4HLM/tMAebHfcYD/wI8D0ytcbyZwIF1zrUIWD/z+kTgkszr\n",
       "DYHXgPdk2pbP6fsSx4tta8W+7Rpfvx2YA3wms8804EVgn/j6g8DrwPgWPtuHgPOBf439OBb4MPBU\n",
       "Zp/xwI7AUHz9VeDeGsfaH7g951wzgGeAg+PrFVv5TGL7l4EbMp/7j/v971OPxQ+NcAfD6qTpobk5\n",
       "+2wBrAeckKTmAUcAe4YQVs3s91SSJMcnSfJ6kiTPAX8C1qlzzFCnvdF+L5EGvh1DCG8HSJLk1SaP\n",
       "VbEfcEuSJNfEn3+CNMAcUbXfz5MkuSw+vx1YGlizxXP9BHg4Pr8ceBRYpbIxflY3JUmyKDZdAWxU\n",
       "4ziNPq8E8CRJzovHzfvvWc9JwFIhhB8CewEHj+IY0iUKuIOh8ou5Ws4+6wGPZIICSZI8D7wQt9Uz\n",
       "n/r/TppdiGOJ/ZIkeQbYElgXuDmEcE8IYWqTx6pYF/hbVdsDvPm9jAS5JEnmx6ej+Xe/qOp5Ns0S\n",
       "QghHx/TGHcDZwFAIYTTneWEUPzMiSYe2XwD2BaaP4otMukgBdwDEkdCfyL+Q8ijwzhDC0pWGEMJq\n",
       "wMrAI53uEkv+23rTyC5JkvuTJPl8kiQbkf7pf00IYXwL53iUN48i383ikWilH73wWWBPYK8kSbYh\n",
       "TR30RQzypwE/AL4aQnhrv/oib6aAOzhOAY4JIUwNi03OXFj5X9JgdEoIYVwIYTnSkdiVSZL8o8Gx\n",
       "6/0pXK/9cWATgDgt7eNUBb+qCz7LkKYYFjboR9YlwHYhhI/F461Lmjs9p4n+ddrawFPAMzE9c2ps\n",
       "X7pqvxeADUMIK8b/Pm+v2t6J/p4APJ8kyUGkqZALOnBM6RAF3AGRJMmPSUdap5NeeHkE+C9ghbg9\n",
       "AXYl/VP8IdIR8XPAgdWHqnX4eqet0/4F4NQQwvXx+U3ZjSGElQCPMyIeI51itlOSJAuaPU+SJE+S\n",
       "Xqg6OoTwOPBL0otb51b9XPXPtjPqTeo8Px1YFXgS+AVwHrCA9EJe1g3APcCDwH28Ob9aq7/N9AWA\n",
       "EMJ2pKmET8emE4F1QgiHNnlM6bKQ/h6KiEi3aYQrItIjutNMRMYkM5tMmg661d2PjW0nAZNIZ6Ic\n",
       "7O4PxvYZpBdp5wEz3P3i2L4DaeoG4ER3vznvnAq4IjJWjSe92Dyp0uDuJwCY2dbAccAhcVMC7O3u\n",
       "j1b2NbMh0gu1O8SmG81sprvXzdMqpSAiY5K7/4r0bsVatgL+XNVWPYtkQ+B+d3/N3V8jnReeu1iU\n",
       "RrgdMDw8rCuPIk2YOnVqx6bqtfJ718p5zew20lknkzPNc4FLzex54Gh3f4D0bsMXzeyMuM9LpLfD\n",
       "/7XesRVwO2SXL1zV7y60beHcJxi3YvVMpnJ6YfZZ/e5CR9x26y1su92UfnejI2bdOtzxY+5y7M8a\n",
       "7nPtqbu1dEx339bMtgR+CHwkth0JYGabkc6z3p10WuVbgcNIR7/nALlz2hVwRaS8QtsD5noHeIra\n",
       "8XEe6e3ukKYQ/jWzbcM48q1LAVdGDMrodpAMyui2a0a1XEXKzI4DdgLWMLOV3P0QM7ucNJ3wBpmF\n",
       "kMzsMtJFj+aS3qiDuy80s6+S3nQD6S3quRRwRaS82hjhuvt0YHpV29519t2nTvtNVN1JmUezFGTE\n",
       "wrlP9LsLUuW2W2/pdxeKbWhc40eBaIQrIuXVRkqhHxRwZYRyuMWjHG4D7V806ykFXBEpr4KlDBop\n",
       "13hcuko53OJRDreBMNT4USAa4YpIeSmlIGWlHG7xKIfbQMFGsI0o4IpIeY1TDldKSjnc4lEOtwHl\n",
       "cEVEekQ5XCkr5XCLRzncBgo2gm1EAVdEykvzcKWslMMtHuVwGwih8aNANMIVkfJSSkHKSjnc4lEO\n",
       "t4E2RrB1qvZ+j7Q67xBwQKZqb83qvK1W7S3X14OISFZ7yzNWqvaOcPdD3X170mq8lSBcqc67Y3xM\n",
       "q9duZrnfAAq4MkI53OJRDreBNubhNqjaO5e06gPUqM5rZhvWakdVe0VkYHUvh3sgcGZ8Xq86b6jT\n",
       "rqq90phyuMWjHG4DXZgWZma7An9x9/tiU73qvEN12ut3t+O9FRHplfanhS2xg5lNBLZz9+9kmutV\n",
       "5225aq8CroxQDrd4lMNtoI0cbqzaOw3Y1cy+H5t/CmxhZjPN7LuQVuclvTj2S9KCkdPy2vMopSAi\n",
       "5dX5qr3r19m3ZnXeVqv2KuDKCOVwi0c53HxDQ+X6I10BV0TKq1h37jZUrq8H6SrlcItHOdx8IYSG\n",
       "jyLRCFdESqtoAbURBVwZoRxu8SiHm085XBGRXinXAFc5XFlMOdziUQ43n3K4IiI9UrSA2ogCroxQ\n",
       "Drd4lMPNpxyuiEivlGuAqxyuLKYcbvEoh5tPOVwRkR5RSkFKSznc4lEOt4FiDWAbUsAVkdIqWsqg\n",
       "kXKNx6WrlMMtHuVw8ymHKyLSI+3kcOuUSa9XDn0Gafn0ecAMd784b/96FHBlhHK4xaMcbgPtDWAr\n",
       "ZdInwRJlz3eI228EKgE0AfZ290crP1xrfzOb6e5JvRMqpSAipdVOSqFGmfR65dBHTld1CJVJl9Fb\n",
       "OPcJjXIL5rZbb9EoN0eHc7T1yqH/FZgLXGpmzwNHx2KRefvXpIALmNlFwBTgCHf/RaN2ESmGMNTR\n",
       "gFuvHDrufiSAmW0GnArsnrd/PUopAO5+ADCj2fZBpdFt8Wh0m68DsxSyOzRT9nweML+F/ZfQ1RGu\n",
       "mc0hLSG8NXCzux8f2/cHDiF9sxe6+/kNjjPb3beo8fw+YBawCXCtu389th8G7AssBC5y9wtj+4eB\n",
       "r5AmwL/t7ld2+C2LSA+1k1KIZdJ3AtYws5Xc/RAzq5Q9h0zZczO7DFiTNLVwOKRl0uvtX0+3Uwor\n",
       "A98iHWb/HjjezFYFDgUmkwa+YTO7zt1HMwl0AnAU8E/gDjM7z92fBvYCdnb3Fys7xiuK3yQN/q8D\n",
       "N5vZte7++ujf3mBRDrd4lMPN107ArVMmvV459H3qHKNQZdKfigEQM5sX29YHfuvuC2L7XaRX+0YT\n",
       "cJ9191fice4G1gOeBvYDDonB/Wp3nwWsCqwNXBt/9q3AWsBDo3ljItJ/Hc7hdl0/Lpo9CGxuZkuT\n",
       "jnAnAWc2+JkhADNbDlgu076Wma1CenXw/aTpAtz9cWC6mU0gTTlMBJ4F/gzs5u4vd+7tpLKjw8od\n",
       "W3rdv9fZkWHlbq0yvt52uymF6k87r8fReUW7k6yRkCR15+i2zcx+4+5b1niezeH+wN0vaHCcc4FX\n",
       "gVeAPd19k9h+LzAb2Bi4xN2/G9vPADYDVgLOzuRwtwZOJg30T7r7JzLnOBHYA7jY3U9v1J41PDyc\n",
       "7PKFq1r6bKS7Xph9Vr+7IFVm3TrM1KlTOxYhh4eHk4Ove6PhfuftvExHz9uOrgbcbsteQOunQQm4\n",
       "g5TDHZSAO0g53G4E3ENvmN9wv+99eOnCBNyyz8Mt77eFiLStZBmFcgfcSopCOmNQRreDZFBGt91S\n",
       "thxuqQOuiIxtJYu3utNMFtN6uMWj9XDzjRsXGj6KRCNcESktpRSktJTDLR7lcPOVLN4q4IpIeZVt\n",
       "hKscroxQDrd4lMPNNzQUGj6KRCNcESmt5ka4xZmur4ArI5TDLR7lcPOVLKOggCsi5aURrpTWIK2l\n",
       "MCgGaS2Fbmg3R2tmhwCfJF0Y6zB3/2tOqfSWSqLX7G9bvRUR6aMQGj/qicu9HuDuHwA+DnzDzAJp\n",
       "6fMd42Na3Heouj3u25JRBdy4XKIMGI1ui0ej23xt1jQLwNJmNh54EViD+qXSWy6JXkvdlIKZTaR2\n",
       "8iMAW7V6IhGRTmsnpeDur5rZN4DrSWuVrUwadGuVPg912uuWRK8lL4c7DPyuzrYX67RLiSmHWzzK\n",
       "4eZrd5ZCLCR7JYCZ3QM8Re3S50N12luSF3D/4O7bt3pAEZFe6dSdZma2M2mh2weoUfrczMbVam/1\n",
       "PHkB98ScbTKANLotHo1u87Ubb83sB8BGpLMU9nX3RbVKn4+mJHotdQPuaKY8iIj00tBQexOt3P1T\n",
       "NdrqlUpvqSR6LU331sxWbudEUnxaS6F4tJZCvnamhfVDwxsf4myFC4HxwLvjfLQL3P3AbndORCTP\n",
       "IK4W9m3go8DfAdx9EemcNBkwyuEWj3K4+co2wm0m4Cbu/khV2/hudEZEpBWDuDzjP8xsJyCY2YrA\n",
       "16g/P1dKTPNwi0fzcPMNFW0I20AzI9zPAPsDm5DeVbEccEw3OyUi0oyypRQajnDd/Vlgnx70RfpM\n",
       "o9vi0eg2X9kumjW9PKOZLQ0scPfiLC4pImPauILlaBtpmFIws83N7G7gSeAZM7vJzDboftek1zQP\n",
       "t3g0Dzdf2VIKzeRwLwCOd/e3ufvbgHOBS7rbLRGRxkIT/yuSZgLuP939xsoLd78KWNi9Lkm/KIdb\n",
       "PMrh5hs3FBo+iqSZgHujmW1beWFmHwF+1b0uiYg0p2wphbwFyF8hXYA8ACeY2by4aTngn6TlJmSA\n",
       "aB5u8Wgebr6yzcPNWy1shV52RESkVSWLt6raK4tpdFs8Gt3ma+fWXTNbCfhZpun97v4WM5tBukbu\n",
       "PGCGu18c92+7am8zq4VNAHYH1iRNLwRgdXc/ttWTiYh0UjspBXd/GdgewMw2BT4bNyXA3u7+aGXf\n",
       "TNXeHWLTjWY2s9X7Epq5aHYlsDewE7ABsAewoJWTSDloHm7xaB5uvtDEo0lHAv9Tdeis7lbtzVjL\n",
       "3d9nZgfHkxwP/KjVE4mIdFonbu01s38B1nH3ObFpLnCpmT0PHB1rl61CB6r2NjPCfSb+//3AJHd/\n",
       "EVinlZNIOSiHWzzK4ebr0Dzcg4HzKi/c/Uh33xr4MnBqbH6OtGrv8cCX4vOWq/Y2E3DvMrO3AXcA\n",
       "u5nZncCfWj2RiEintTsP18yWAnYBrqqxeR4wPz7/G12u2guAu49U7zWz7Uiv3v2+1RNJ8WkebvFo\n",
       "Hm6+DqQU/gO4JlayAcDMLiOdJDAXOBx6ULW3Fnd/FbhnNCcSEem0du/cdfcrarTVXI62E1V7NQ9X\n",
       "Rmh0Wzwa3eYr251moyrqbmbndrojIiKtGgqh4aNI8tZSmEg6AbhaALbqWo+kb5TDLR7lcPMVLJ42\n",
       "lJdSGKZ+scgXu9AXEZGWFK0qbyN5AfcP7r59z3oifafRbfFodJuvaCmDRvIC7ok520RE+q5k8TZ3\n",
       "ecaWV8IZy6645Mv97kLb5syexaZbbN3vbnTEHx97qd9d6Ii777qdiVtN7nc3Cmtgq/aKiBTNOAVc\n",
       "KatBGd0OEo1u85XsmpkCroiUV9kC7qhufJDBNGf2rH53Qarcfdft/e5CoYUQGj6KRCNcESmtcSUb\n",
       "MirgygjlcItHOdx8gzQPV0Sk0Eo2wC1df6WLlMMtHuVw87W7AHmvNVO1d1VgOrC2u3/IzAJwuLuf\n",
       "1fXeiYjkaLKETl1mtjZwCWks/I27H1OvHHonyqQ3M8I9H7gemAAQywLXXKBXyk053OJRDjffUGj8\n",
       "aOA04EvuPjkG20o59B3jYxosUSZ9pD0OPlvrbxP7rBJXRV+UaVMqQkT6rp31cM1sHLCBu9+ZaX5T\n",
       "OXQz27BWO10qk77QzNbMdHJ3tDzjQBqktRQGhdZSyNdmjvZtwAQzuxpYCfgf4Clql0MPddpbKpPe\n",
       "TMA9FrgBWMfM7gHGA7u1chIRkW5ocy2F50gD5x7AOGAW8CnSEuiHkQbZc0jLoQ/VaW9Jw9SAu98N\n",
       "bAFMAfYFNh1NeWApPo1ui0ej23zt5HDdfT7wGLCGu78BvA48QO1y6L0pkx479gYwp9WDi4h0U1OT\n",
       "FGoVClvsOOB8M3sL4O7+z1rl0HtWJt3MjqnRnLj7t0dzQiku5XCLRzncfE1NC1tUf5O7PwrsXNVW\n",
       "sxx6r8qkr8iS3xFboItmIlIARbuxoZGGAdfdp2Vfm9nSpHPXZMBodFs8Gt3mK9taCi3Pp42J5tW6\n",
       "0BcRkZZ04MaHnmomh3tNVdNqwBPd6Y70k3K4xaMcbr5BLLFzetXr591dMxZEpO9KFm+byuHe0oN+\n",
       "SAFodFs8Gt3mK1rKoJGGOVwzW68XHRERaVU7ayn0QzMXzX7e9V5IIWg93OLRerj5xg2Fho8iaSaH\n",
       "+1rXeyEiMgoFG8A21EzAvcDMTgdOzja6+/Pd6ZL0i3K4xaMcbr6yrRPbTMA9gfROs49l2hJg/a70\n",
       "SESkSUUrg95IM7MU3tGDfkgBaB5u8Wgebr5BnIcrIlJI5Qq3OSkQM/tiLzsi/afRbfFodJuvbFV7\n",
       "83LOO+dsExHpu3EhNHwUSV5KYSkzW6XeRs1SGDzK4RaPcrj5OnHRzMzGA/cD0939HDObAWwEzANm\n",
       "uPvFcb+2y6TnBdz3AXfX2aZZCiLSdx0avx7KkrEuAfaOi5MDS5RJ3yE23WhmM909v55ElbyAO9vd\n",
       "9dU6hmh0Wzwa3eZrd4RrZssB/w78FFghe+iqXUfKpMefq5RJ73jVXhGRQupAjvZI4Cxg9UzbXOBS\n",
       "M3seODoWi1yFDpRJz7todmYrB5Ly01oKxaO1FPKFJh71xMKR27j7Ddld3f1Id98a+DJwamx+jrRM\n",
       "+vHAl+LzzpVJd/crWj2YiEgvtTktbGtggpn9hDSPe4CZbZzZPg+YH5/3rky6jA3K4RaPcrj5htq4\n",
       "bObu1wHXAZjZ/sDy7n6vmV0GrEmaWjg87tubMukiIkXVqfVuK1O/4vN96uzTdpn0si22I12kHG7x\n",
       "KIebr2x3mmmEKyKl1U5KoR8UcGWEcrjFoxxuvqKNYBtRwBWR0ipazbJGlMOVEcrhFo9yuPmGQuNH\n",
       "kWiEKyKlFZrI4ba02EGXKeACZnYRMAU4wt1/0ah9UCmHWzzK4eZrJqWwsAf9aJZSCoC7HwDMaLZd\n",
       "RIpBKYUuMLPZ7r5Fjef3AbOATYBr3f3rsf0wYF/SL7eL3P3C2P5h4Cukf2V8292v7PmbKTCth1s8\n",
       "Wg83XzMphSIp+wh3AnAU8AFgJzOrrPizF7Czu0/OBNsh4JukS7FtBxwVFx4WkZLSjQ+99ay7vwJg\n",
       "ZncD6wFPA/sBh5jZqsDV7j4LWBVYG7g2/uxbgbWAh3re64LS6LZ4NLrNV7QSOo2UJeAOwchiwctl\n",
       "2teKZYBeAt5Pmi7A3R8HppvZBNKUw0TgWeDPwG7u/nKnO5j9c7wyvUqv+/d6lQnLjASrytQqve7v\n",
       "63ettiydVq5wCyFJijRpojYzOxd4FXgF2NPdN4nt9wKzgY2BS9z9u7H9DGAzYCXg7ExaYWvgZNIc\n",
       "7pPu/onMOU4E9gAudvfTG7VnDQ8PJ/NW36Szb7oPBimHu/ryE/rdhY4YpBzuSw/+lqlTp3YsRg4P\n",
       "DyfLv2Niw/1effjujp63HaUIuPVkL6D1kwJu8SjgFk83Au4K79i84X6vPNzZ87ajLCmFesr7bVFA\n",
       "gxJsB8mgBNtuaWfal5mdBEwCFgEHu/uD9SrzdqJiL5R8loK7b9nvPohIH7VRY8fdT3D3D5IG0uPM\n",
       "LJBW5t0xPqbBEhV7R9rjvi0rdcCVztJaCsWjtRTyhSb+14StSC+oj1TmjdV5/2ZmG9ZqJ63Y27Ky\n",
       "pxREZAxrd1aYmd1GOmV0MmnNslqVeUOd9pYq9oJGuJKhHG7xKIebr90bH9x9W+CTwA+pX5m3IxV7\n",
       "QQFXREqsQymFp0j/2n+A2pV5O1KxFxRwJUM53OJRDjdfOyNcM7vczIaB75OuCLiI9OLYL0mLRU6D\n",
       "tGJvrfbRUA5XREqrnRyuu+9do61mZd5OVOwFBVzJUA63eJTDzVe21cIUcEWktEq2do1yuLKYcrjF\n",
       "oxxuvjbue+gLjXBFpLRCyYa4CrgyQjnc4lEON1/J4q0CroiUV8nirXK4sphyuMWjHG4DJUviaoQr\n",
       "IqXVTJn0IlHAlRHK4RaPcrj5yhVuFXBFpMxKFnGVw5URyuEWj3K4+Tq0eE3PaIQrIqXVTomdflDA\n",
       "lRHK4RaPcrgNKOCKiPRG0VIGjSiHKyOUwy0e5XDzDYXGjyLRCFdEyqu9MumTgdOBW9392Ng2A9gI\n",
       "mAfMcPeLY3tHyqQr4MoI5XCLRzncfG2mFMYDpwCTMm0JsLe7P1ppyJRJ3yE23WhmM909afWESimI\n",
       "SGm1U2LH3X8FPF/rsFWvO1YmXQFXRiiHWzzK4ebrQg53LnCpmV1jZpWgugqxTHoslV4pk94ypRRE\n",
       "pMQ6e1XM3Y8EMLPNgFOB3VlcJv2weMJzGGWZdAVcGaEcbvEoh5uvA2vX1DvCPGB+fN6xMukKuCJS\n",
       "Wu3EWzM7DtgJWMPMVnL3Q8zscmAN0tTC4ZCWSTezSpl0UJl06YQ5s2dplFswd991u0a5OdpZntHd\n",
       "pwPTq9reVDo9tqtMuoiMcQW7saERBVwZodFt8Wh0m69k8VYBV0TKq2QFHzQPVxbTPNzi0TzcfCGE\n",
       "ho8i0QhXREqrWOG0MQVcGaEcbvEoh5uvYAPYhhRwRaS0yla1VzlcGaEcbvEohztYNMIVkdIq2QBX\n",
       "AVcWUw63eJTDzVe2EjsKuCJSWkUrodOIcrgyQjnc4lEOt4HQxKNANMIVkdJSSkFKSznc4lEON58u\n",
       "momI9EjZAq5yuDJCOdziUQ43X2jif0WiEW6HTHj6j/3uQtu2XPctMADvA9Iqf4PgXasty0sP/rbf\n",
       "3SisO28b7ncXWhKSpOXS6iIiMgpKKYiI9IgCrohIjyjgioj0iAKuiEiPKOCKlJiZaaZRiSjgStvM\n",
       "bAMzW7rf/RhrzOxLwM5mtkK/+yLNUcCVtpjZD4DvAJPMbHy/+zNWmNlngL2BicDWCrrloIAro2Zm\n",
       "awGPA1cAk4CtzGxC3FasW3wGz0/cfVPgf4FtgMlmtlKf+yQNKP8jo+buTwInApjZp4EppF/iM4Fl\n",
       "gNf71rnBNxfA3a8zs+VIv/D+YWYJENx9dl97JzVphCstM7MVM8+XAnD3C4BHgc3N7AjgZDN7S5+6\n",
       "OJDMbA8z2xDA3Rea2bj4/ApgDvBJ4CIKtwqsVOjWXmmJmX0XeAq41N0fjm3j3H1hfP4pYA/gTHe/\n",
       "sW8dHTBmdiqwO3AGMOzu98X27Gf/IeA5d9fiCwWlgCtNM7MdgSOB3wKPADMzQXcpd18Qn6/u7k+b\n",
       "WXB3/QPrADPbGngY+DgwjzTo/rnOvgFAn33xKOBKy8xsPeDTpEH3Znd/sM5+CrhtMrMhd19U+SzN\n",
       "7J3AJ0gXRPsJMB54xd0HZYG0gaaAKw3VGjGZ2cak05LmkF4cW4n0yvmivnRyAGXTBVXt6wL7AqsA\n",
       "mwMHu/v9ve6ftE4BV3JVpQpWdPe5mW0bA/sDOwKfc/db+tPLwZMZ2Q4BewJvuPvVme3rkH72s9x9\n",
       "Zr/6Ka1RwJWG4tXwGcAzwMvAqe7+z7jtncAy7v4XpRA6K/5lMRO4FtgeeBo40t1fiduXdvf5ytmW\n",
       "h+bhSjNOIr1Qdg0wm/RGh3sB3P2hPvZr0K0D/MbdTzOzbYGfZTe6+/z4/wq0JaF5uPIm2bvE4uh2\n",
       "AenI9sukF8teNrNdqn9Ov/jtqcyrzXgV+IiZPQCcCfweuNLM3tXzzklHKODKEuKFmiQ+Hx8v2twK\n",
       "fAt4Afg1aXpBd5F1UOUCmZkNmdl/mNkK7v4c8CnSL7yNgJ8CZ7j7A33trIyacrgyoupCzXTSX/DZ\n",
       "cTrSR0hnJSwArnL3a5Sz7az4uV8H/Mzdz61csIwXyN4GLHT3/6ecbXkp4MoS4i/zT4G73f2UuEDN\n",
       "fHd/1syWBRa5++v6pe88M9sPeI+7f9HMDgK2BM5193uq9tMXXUkp4MoSzGx94DTS+/L/G3gO+DCw\n",
       "q7vP62PXBk7lL4rM6/1IP+tfAzcDHwJecPcZ/emhdJpyuGNc9YWaeNfYvcBk4BLgMtLcrRYY76Ds\n",
       "HWRxPjPufgnptK+zgIT0S++vfeymdJhGuGNY9kIN8FlgrrtfmNm+KenV8TOzk+6lPVWfuwMrAk8A\n",
       "pwL3AauR/pVxibvfpBTC4NAIdwzL/NJ/B1iftFzLCWa2StzlPcCJ7n61FhTvnPi5B+Bc4Dp3/xDw\n",
       "F+Awd0/c/WnguEqw7WtnpaMUcMeoGGgBLgBWd/ejSBdFWRc4AMDdL3P32/RL3xlV6ZvVgF2AlQHc\n",
       "fTrwdjPbLm7/e2xPNLodHEopjDHVf56a2RTgPOCYONXrQ6QXbr4IvK5f9s6KX3Tvcfc/xsXELyJN\n",
       "K1wNXAkc4O5/7GcfpXs0wh17AoCZnW1mG8UFZw4ATjOzk0gn2t/k7vMUbDujamQ7CbjWzCa5+19J\n",
       "P/vDgR8DB8ZAXH3HmQwIBdwxopJCyExDehg438zWd/dZwMHAR4E73f16/dJ3TiVnG+8euwM4CjjD\n",
       "zLaJQXfsWEBQAAAFyElEQVRHYD6wRfwRLXE5oJRSGENiLvar7v6V+PpQ0iX+9nf3+81sIumi1ke5\n",
       "+/V97OrAiTM+ZgBT3f2FeOfe2cA+7n5XnP98NvCf7v5CH7sqXaQR7hgSUwTrmtnl8fX3SCfZ/8jM\n",
       "lnf3u4F9SKcmSQe5+xzS8kSXxabrgd8Q5zfH+c+7KtgONo1wx4jsXU1mdgawFunqX7uSphF+XX3n\n",
       "k3RenIXwVdKVvy6IOVvNsx0jFHDHkKoKr6eRrrc63d3v0doIvWNmGwHj46hXxhAF3DGmKuiuHifZ\n",
       "Sx9oZDv2KOAOmGZ+iZU6EOkPXTQbIHH91Mri4XWndSnYivSHRrgDIrP61Djg68DywM+B38fKASLS\n",
       "ZxrhDojMqPXHpFUZbiW9V38SLFmnTET6QyPcAWJmawCnu/sn4usPAkcDH6+U1haR/tEIt8RqLB7+\n",
       "FPCqmR0fmx4CXkOLh4sUgka4JVW1iPUXSUuxnGNm6wHHkt7YsDrwLXf/WT/7KiIpBdwSi3nZr5Fe\n",
       "INuEtMLul+K2iaRB+EHN9xQphqX63QFpy1mki4fvCWBmV5vZGe5+dFwXQUQKRDnccjsfeK+Z7Rtf\n",
       "7w2828w2ys5K0OhWpBiUUig5M9uEdFm/Ge5+kdIHIsWlEW7JxXIsnwUON7O1gHGgebciRaQR7oCI\n",
       "1QQ011akwDTCHRyvgka2IkWmEa6ISI9ohCsi0iMKuCIiPaKAKyLSIwq4IiI9olt7BTObBvxf4Elg\n",
       "GeAr7n5Dh8+xOXCqu29f1b4HENz9ihaP9zng++7+Wht9esXdV2hiv5WBa0iLbn7X3U9v4Ry3AMe0\n",
       "eqt1O+eU4tIIVwAS4Cx33wYw4EIzW7YXJ3b3K1sNttFRwHJtnr6pKTru/kL8bC7s1jk6fE4pKI1w\n",
       "paJSJv1hM3uMdGR1P4CZPQycBHwaWBb4qLs/ErftC3yGNLD8xt0/Xzmgme0P/BfpyPme7MnM7APA\n",
       "t4B1qTGCM7NDgf1Jq1ckwFR3n29mE4BfAWsA15rZAuA/3f2x+HMTgVNJ77h7HjjI3f8Rt70fuAB4\n",
       "Gbij8p7bZWZ7AQcCK5J+CXzc3f+S2eVjZnY6sBpwmrtfGH9uHPBNYCvS38Vz3P2STvRJikkjXFmC\n",
       "mW0EvAV4MNOcAP/m7lu5+3szwXZj0iC8XRyNLW1m+8VtbwdOBqa4+78DL2bP4+6/dvfJ1BjBxQC2\n",
       "B/BBd5/s7tu6+/z4c/PiuZ4CPhK3V4LtMvF4n3D37UjLDU3PHPqHpH/eTwFuAMaP/pNaws3uvlPs\n",
       "1w+Bz1dtfzWecwpwspmtGtsPAhbFz2EKcJiZvbNDfZIC0ghXIB3pfcbMdgVeAvZy9wVV+5xc4+em\n",
       "ko5Qf2lmkI7uno/btgSG3f3Z+PomYMcm+7Mnab631fzsu0lH5pfG/gwB82AkJ7qSu88EcPc7zGxe\n",
       "i8evyd2fM7PNgE2BjYA1q3a5Ie73jJndCWxGOkrfEXiHmW0Z91s2voeHOtEvKR4FXIF0BHuOu3+7\n",
       "xZ+bD1ydTSNkLGDJP9lb/fN9NP82FwAPV1+YixaO4nhNMbOL4tOfAncDa1ftkn3v44DX4/P5wInu\n",
       "fk23+ibFopSCVIwmn3kDsJeZbVBpyKzlcBcw2cxWjm17tXDcK4H/NrMVq46ZNY+0hFB2+1+A8Wa2\n",
       "e3V/3P1l4Gkz2zq270L7F90qdiPNY98ATOTNXzQWz7kO8H4W57OvBv7LzFaoeh8yoBRwpSLvanrN\n",
       "be7+EGkO90dmdoeZzSKWZY+phC8DtwG3NDjHEu3u7sBlwLCZ3Q7cWgm+GecCPzezXwGfij+3kDT4\n",
       "HWRmd5rZHcARmZ/5NHBm7OdWxAV/WnS4md1mZtnCnCcBc4BfAn8mfhFk3tvrZnYzcB1wmLu/Gvv7\n",
       "E+Aq4Jb4Pm+rBN8mziklpMVrRER6RCNcEZEeUcAVEekRBVwRkR5RwBUR6REFXBGRHlHAFRHpEQVc\n",
       "EZEeUcAVEemR/w9lhBV8F9YYeQAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1144aadd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_true = np.array(predictions.map(lambda x: x.label).collect())\n",
    "y_pred = np.array(predictions.map(lambda x: x.prediction).collect())\n",
    "\n",
    "print classification_report(y_true, y_pred)\n",
    "evaluate(y_true, y_pred, \"Random Forest\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scale features\n",
    "scaler = StandardScaler(inputCol=\"features\", outputCol=\"scaledFeatures\",\n",
    "                        withStd=True, withMean=True)\n",
    "\n",
    "# Compute summary statistics by fitting the StandardScaler\n",
    "scalerModel = scaler.fit(train_set.toDF())\n",
    "\n",
    "# Normalize each feature to have unit standard deviation.\n",
    "scaledData = scalerModel.transform(train_set.toDF())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stringIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexed\")\n",
    "si_model = stringIndexer.fit(scaledData)\n",
    "index_df = si_model.transform(scaledData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gbt = GBTClassifier(labelCol=\"indexed\", featuresCol='scaledFeatures', maxIter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = gbt.fit(index_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.transform(scalerModel.transform(test_set.toDF()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipelineGBT = Pipeline(stages=[scaler, stringIndexer, gbt])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Param(parent='Pipeline_4f90ad0fe7376688a9fb', name='stages', doc='pipeline stages')"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipelineGBT.stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gbt_model = pipelineGBT.fit(train_set.toDF())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = gbt_model.transform(test_set.toDF())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+--------------------+-------+----------+\n",
      "|            features|label|      scaledFeatures|indexed|prediction|\n",
      "+--------------------+-----+--------------------+-------+----------+\n",
      "|[24.0,24.0,-35327...|  0.0|[-0.2841872064726...|    0.0|       0.0|\n",
      "|[0.0,0.0,-2526.71...|  0.0|[-0.4557386866640...|    0.0|       0.0|\n",
      "|[64.0,56.0,350274...|  0.0|[0.00173192717971...|    0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[32.0,32.0,113571...|  0.0|[-0.2270033797421...|    0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[32.0,16.0,83453....|  0.0|[-0.2270033797421...|    0.0|       0.0|\n",
      "|[16.0,0.0,75306.4...|  0.0|[-0.3413710332030...|    0.0|       0.0|\n",
      "|[40.0,32.0,318276...|  0.0|[-0.1698195530116...|    0.0|       0.0|\n",
      "|[60.8886602759,0....|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[8.0,0.0,153616.0...|  0.0|[-0.3985548599335...|    0.0|       0.0|\n",
      "|[40.0,56.0,36570....|  0.0|[-0.1698195530116...|    0.0|       0.0|\n",
      "|[32.0,48.0,941976...|  0.0|[-0.2270033797421...|    0.0|       0.0|\n",
      "|[60.8886602759,66...|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[60.8886602759,66...|  0.0|[-0.0205078617806...|    0.0|       0.0|\n",
      "|[40.0,32.0,-15022...|  0.0|[-0.1698195530116...|    0.0|       0.0|\n",
      "|[80.0,104.0,-2040...|  0.0|[0.11609958064064...|    0.0|       0.0|\n",
      "|[24.0,104.0,15944...|  0.0|[-0.2841872064726...|    0.0|       0.0|\n",
      "|[40.0,24.0,294188...|  0.0|[-0.1698195530116...|    0.0|       0.0|\n",
      "+--------------------+-----+--------------------+-------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[StandardScaler_4d6bb47b72f3943d3a86,\n",
       " StringIndexer_4644863e7bea8b848f5c,\n",
       " GBTClassifier_4a28808bf252ef21c40c]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbt_model.stages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.77      0.97      0.86      1376\n",
      "        1.0       0.90      0.46      0.61       751\n",
      "\n",
      "avg / total       0.81      0.79      0.77      2127\n",
      "\n",
      "Test Error = 0.208274565115\n",
      "Accuracy: 79.1725434885\n",
      "\n",
      "\n",
      "Confusion matrix GBT, Pipeline\n",
      "[[1336   40]\n",
      " [ 403  348]]\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEnCAYAAADl8/suAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8ZNO5//HPOgeNIKaYhwSdzi+8EonhJ2hDWiSEuIKH\n",
       "XPyExCxExHWDpEmIdBCRa8hFaJEYnvBCiNDSpmhxSYvIlZhnYp6aaHrYvz/2qtO7y6ldVaemvet8\n",
       "33nVK1Vr79p7VenznOc8e+21QpIkiIhI5w30ugMiIqOFAq6ISJco4IqIdIkCrohIlyjgioh0iQKu\n",
       "iEiXKOD2kRDCbiGEv4QQXgwhPB5CuCmEsFaHzrVNPMfT8ZyDHTrPmPg5Vu/E8bsphPDVEMIf23Cc\n",
       "vvlORpsFet0BaY8QwnbATwFLkuS22PYp4OkOnfJg4JwkSU7q0PEBSJLkXeCznTxH2eg7KS9luP3j\n",
       "e8D3K8EWIEmSv8QfTgBCCCuEEC4LITwZs9OzQgiLZbZvEUJ4JoSwYwjh7hDCSyGEi0MIIbPPSSGE\n",
       "p4GtgG/HDPfmbEdCCHNDCGtkXh8XQrioap99Qgh/CyE8FR9HVG1fMx776erjZfbZIIRwW9zngRDC\n",
       "N6u2Hxf7f2wI4e/x8xzexHdKCOGJEMJhIYTHQggXhBB2CiE8G0L4fmafpUIIPw0h3B9C+GcI4b4Q\n",
       "wqZVx/ktcCqwQeZz/d+qfSbH7/eM+J08G0L4RKPfSfzv91wIYfn4eiCEcHMIYd9mPrN0UJIkepT8\n",
       "AQwC7wGfzNlnALgb+CEQgIWBi4HLMvtsAcyK+4wBlgFeBSYMc7ybgX1qnGsusEbm9UTgoszrscA7\n",
       "wMczbR/I6ft8x4ttK8W+bR9frwzcBxyY2ec44HVgt/j6s8C7wJgmvtvHgXOBj8Z+HAl8AXg+s88Y\n",
       "YGtgIL4+Hrh/mGPtBfwx51yTgReB/eLrxZv5TmL7d4HrM9/7r3v971OPeQ9luP1hedLy0IycfTYA\n",
       "VgeOTVIzgUOAnUMIy2b2ez5JkqOTJHk3SZJXgL8Dq9Y4ZqjRXm+/N0gD39YhhJUBkiR5u8FjVewJ\n",
       "3JIkyTXx/c+SBphDqvb7bZIkl8bnfwQWBFZs8lyXAE/E55cBTwFLVzbG72pKkiRzY9PlwLhhjlPv\n",
       "+0oAT5LknHjcvP+etZwALBBC+CWwC7DfCI4hHaKA2x8qP5jL5eyzOvBkJiiQJMmrwGtxWy2zqP3v\n",
       "pNGJOObbL0mSF4ENgdWAm0II94QQJjR4rIrVgEer2h7h/Z9lKMglSTIrPh3Jv/u5Vc+zZZYQQjg8\n",
       "ljduB84EBkIIIznPayN4z5AkTW2/DewBTBrBLzLpIAXcPhAzob+TfyHlKeAjIYQFKw0hhOWApYAn\n",
       "290l5v+39b7MLkmSh5Ik+VaSJONI//S/JoQwpolzPMX7s8iPMS8TrfSjG74B7AzskiTJpqSlg56I\n",
       "Qf4U4BfA8SGEJXvVF3k/Bdz+cRJwRAhhQphnfObCyv+QBqOTQgiDIYRFSTOxK5IkebnOsWv9KVyr\n",
       "/RlgHYA4LO0rVAW/qgs+C5GWGObU6UfWRcDmIYQvx+OtRlo7PauB/rXbKsDzwIuxPHNybF+war/X\n",
       "gLEhhMXjf5+Vq7a3o7/HAq8mSbIvaSnkvDYcU9pEAbdPJEnya9JM61TSCy9PAv8BLBa3J8D2pH+K\n",
       "P06aEb8C7FN9qOEOX+u0Ndq/DZwcQvh9fD4luzGEsATgcUTE06RDzLZJkmR2o+dJkuQ50gtVh4cQ\n",
       "ngFuJL24dXbV+6rf20rWm9R4fiqwLPAc8DvgHGA26YW8rOuBe4DHgAd4f311uP420hcAQgibk5YS\n",
       "vh6bJgKrhhAOaPCY0mEh/TkUEZFOU4YrItIlCrgiIl2igCsi0iUKuCIiXaLJa9pg6tSpuvIo0oAJ\n",
       "Eya0baheMz937TxvKxRw22S7b1/Z6y60bM6MZxlcvHokUzm9dvcZve5CW9x26y1stvkWve5GW0y7\n",
       "dWrbj7ndkVfX3efak3do+3lHSgFXRMorFCJxbZgCrgzpl+y2n/RLdtsxI5quoncUcEWkvEqW4Zbr\n",
       "14N01JwZz/a6C1Lltltv6XUXim1gsP6jQJThikh5qaQgZaUabvGohltHyUoKCrgiUl4FKxnUU658\n",
       "XDpKNdziUQ23jjBQ/1EgynBFpLxUUpCyUg23eFTDraNgGWw9CrgiUl6DquFKSamGWzyq4dahGq6I\n",
       "SJeohitlpRpu8aiGW0fBMth6FHBFpLxKNg5XAVeG9NN8uP2in+bD7YgWSgpmNp50iftb3f3I2PZz\n",
       "YBzp9a293f2x2L4V6bLzABPd/aa89lrKlY+LiGS1dtFsDHBStsHdD3D3LYHjgUoQHoivt46P42q1\n",
       "m1nubwAFXBmi7LZ4lN3WEUL9Rw3u/gfg1RqbZwDvxedjgYfc/R13fwd41MzGDtcOrJXXXZUURKS8\n",
       "OlfD3Qc4PT5fGnjdzE6Lr98AlgFCjfaHax1UGa4M0Tjc4tE43Do6MA7XzLYHHnT3B2LTK8CSwNHA\n",
       "MfH5yzntNSngikh5tR5w56s5mNl6wObu/tNM86PARzOvx7r7IzntNSngyhDVcItHNdw6WljxwcyO\n",
       "Ir0Atr2Z/Xds/g2wgZndbGY/A3D3OaQXx24EpsT31GzPoxquiJRXC8PC3H0SMKmqbY0a+04hDaoN\n",
       "tdeiDFeGqIZbPKrh1qG5FEREukRzKUhZqYZbPKrh5hsYKFYGW48CroiUV7kSXNVwZR7VcItHNdx8\n",
       "IYS6jyJRhisipVW0gFqPAq4MUQ23eFTDzacarohIt5QrwVUNV+ZRDbd4VMPNpxquiEiXFC2g1qOA\n",
       "K0NUwy0e1XDzqYYrItIt5UpwVcOVeVTDLR7VcPOphisi0iUqKUhpqYZbPKrh1lGsBLYuBVwRKa1W\n",
       "SgY1lkmvtRz6ZNLl02cCk939wrz9aylXPi4dpRpu8aiGm6/FGu58y6TXWg49SoBd3X3LTLDVMuki\n",
       "MnoMDAzUfdQyzDLptZZDr6gOplomXUZONdziUQ23jvbWcGsth/4wMAO42MxeBQ6Pi0Xm7T8sBVwR\n",
       "Ka02D/uqLHt+EGkoP4u47Lm7HwpgZusCJwM75u1fi0oKMkQ13OJRDTdfG8bhZndoZNnzmcCsJvaf\n",
       "jzJcwMwuALYADnH339VrF5FiCAMtjVI4CtgGWMHMlnD3/c2ssuw5ZC6amdmlwIqkpYWDIV0mvdb+\n",
       "tSjgAu6+t5lNbLS9X6mGWzyq4eZrpaRQY5n0Wsuh71bjGE0tk97RgGtm95FG/02Am9z96Ni+F7A/\n",
       "aTp/vrufW+c4d7v7BsM8fwCYBqwDXOvuP4jtBwF7AHOAC9z9/Nj+BeB7pEM8fuLuV7T5I4tIFxXt\n",
       "1t16Ol3DXQr4MWnA3R7AzJYFDgA2AzYFdjezkaZWCwOHAZ8BtjGz5WP7LsC27j4+E2wHgB8BnwM2\n",
       "Bw4zszEjPG9fUg23eFTDzae5FOb3vLu/AGBmM2PbGsCf3X12bL+TdDzbSH7aX3L3t+JxpgOrAy8A\n",
       "ewL7x+B+lbtPA5YFVgGuje9dElgJeHwkH0xEeq+VGm4v9KKG+xiwvpktSPqn/cbA6XXeMwBgZosC\n",
       "i2baVzKzpUnHv32atFyAuz8DTDKzhUlLDusBLwH/AHZw9zfb93FSc2Y8O1QDrWSKet2717fdestQ\n",
       "/bOSJZbx9Wabb1Go/rTyepD2K1oGW09IkqRjBzezu9x9w2GeZ2u4v3D38+oc52zgbeAtYGd3Xye2\n",
       "3w/cDawNXOTuP4vtpwHrAksAZ2bKCpsAJ5IG+ufcfffMOSYCOwEXuvup9dqzpk6dmmz37Sub+m6k\n",
       "s167+4xed0GqTLt1KhMmTGhbhJw6dWqy33Xv1d3vnG0Xaut5W9HRgNtp2QtovdQvATebpZddvwTc\n",
       "bKZedp0IuAdcP6vufj//woKFCbhlHxZW3t8WItKyklUUyh1wKyUKaY9+yW77Sb9kt51SthpuqQOu\n",
       "iIxuJYu3mktB5tE43OLRONx8g4Oh7qNIlOGKSGmppCClpRpu8aiGm69k8VYBV0TKq2wZrmq4MkQ1\n",
       "3OJRDTffwECo+ygSZbgiUlqNZbjFGa6vgCtDVMMtHtVw85WsoqCAKyLl1WqGa2b7A18lnaflIHd/\n",
       "2My2AioLD0x095vivsO2N0M1XBmiGm7xqIabr5Uabpx9cG93/wzwFeCHZhaA44Gt4+O4uO9AdXvc\n",
       "tynKcEWktFosKQRgwbgQwevACqRzcz/k7u8AmNmjZjaWNDmdrx1Yi5wl0YczooBrZme7+4Ejea8U\n",
       "l2q4xaMabr4W1zR728x+CPyedHHIpUiD7utxildI59pehjQ4D9fenoBrZusxfPEjABs1cxIRkU5o\n",
       "ddhXXNfwCgAzuwd4nnQ1mINIY91ZwMukGe5w7U3Jy3CnAn+pse31Zk8kxddP8+H2i36aD7cT2jVK\n",
       "wcy2Be4FHgE+mtk01t0fMbPB4dqbPU9ewP2bu2/Z7AFFRLql1TvNzOwXwDjSUQp7uPtcMzuedLVx\n",
       "iBfN3H3OcO3Nygu4E3O2SR9Sdls8ym7ztZrhuvvXhmmbAkxptL0ZNQPuSMaYiYh008BAuUa2Ntxb\n",
       "M1uqkx2R3tM43OLRONx8IdR/FEndYWFxtML5wBjgY3EA8Hnuvk+nOycikqcfZwv7CfAl4J8A7j6X\n",
       "dHCw9BnVcItHNdx8ZctwGwm4ibs/WdU2phOdERFpRj9Oz/iymW0DBDNbHPg+tcfnSolpHG7xaBxu\n",
       "voGipbB1NJLhHgjsBaxDehvbosARneyUiEgjylZSqJvhuvtLwG5d6Iv0mLLb4lF2m69sF80anrzG\n",
       "zBYEZrt7caZPF5FRbbBgNdp66pYUzGx9M5sOPAe8aGZTzGzNzndNuk3jcItH43Dzla2k0EgN9zzg\n",
       "aHf/kLt/CDgbuKiz3RIRqS808L8iaSTg/svdb6i8cPcrgTmd65L0imq4xaMabr7BgVD3USSNBNwb\n",
       "zGyzygsz+yLwh851SUSkMWUrKeRNQP4W6QTkATjWzGbGTYsC/yJd30f6iMbhFo/G4eYr2zjcvNnC\n",
       "FutmR0REmlWyeKtFJGUeZbfFo+w2Xyu37prZEsDVmaZPu/sHzWwy6aTkM4HJ7n5h3L/lZdIbmS1s\n",
       "YWBHYEXS8kIAlnf3I5s9mYhIO7VSUnD3N4EtAczsE8A34qYE2NXdn6rsm1kmfavYdIOZ3dzsfQmN\n",
       "XDS7AtgV2AZYE9gJmN3MSaQcNA63eDQON19o4NGgQ4H/qjp01tDy6XGp9Moy6U1ppKSwkrt/ysz2\n",
       "iyc5GvhVsycSEWm3dtzaa2bLAKu6+32xaQZwsZm9ChweF4tcmjYsk95Ihvti/P+HgI3d/XVg1WZO\n",
       "IuWgGm7xqIabr03jcPcDzqm8cPdD3X0T4LvAybH5FdJl0o8GjonPm14mvZGAe6eZfQi4HdjBzO4A\n",
       "/t7siURE2q3VcbhmtgCwHXDlMJtnArPi80fp8DLpALj70Oq9ZrY56dW7e5s9kRSfxuEWj8bh5mtD\n",
       "SeHfgGviSjYAmNmlpIMEZgAHQ3eWSX8fd38buGckJxIRabdW79x198uHaRt2OtqOLpMuo4+y2+JR\n",
       "dpuvbHeajWhRdzM7u90dERFp1kAIdR9FkjeXwnqkA4CrBWCjjvVIekY13OJRDTdfweJpXXklhanU\n",
       "Xizy9Q70RUSkKUVblbeevID7N3ffsms9kZ5Tdls8ym7zFa1kUE9ewJ2Ys01EpOdKFm9zp2dseiac\n",
       "0ezaS47rdRdadu9d01h3w0163Y22+OuT/VH1mn7n7ay30aa97kZh9e2qvSIiRTOogCtl1S/ZbT9R\n",
       "dpuvZNfMFHBFpLzKFnBHdOOD9Kd775rW6y5Ilel33t7rLhRaCKHuo0iU4YpIaQ2WLGVUwJUhquEW\n",
       "j2q4+fppHK6ISKGVLMEtXX+lg1TDLR7VcPO1OgF5tzWyau+ywCRgFXf/vJkF4GB3P6PjvRMRydHg\n",
       "EjqF0UhJ4Vzg18QlhN09MbPdAAXcPqMabvGohpuv1XhrZqsAF5HGwrvc/Qgz24p5UxtMrNx1W6u9\n",
       "qf42sM/ScVb0uZk2lSJEpOfaMB/uKcAx7j4+BtsB4Hhg6/g4DmC49vjXfnP9bWCfOWa2YuWFme2I\n",
       "pmfsS6rhFo9quPlaqeGa2SCwprvfkWkeCzzk7u+4+zvAo2Y2drh2YK1m+9tISeFI4HpgVTO7BxgD\n",
       "7NDsiURE2q3FuRQ+BCxsZlcBSwD/BTwPvG5mp8V93gCWIV14Ybj2h5s5YSOr9k43sw2AjwGzgQfd\n",
       "fU4zJ5FyUA23eFTDzddQDXe4dWtSr5AGzp2AQWAa8DVgSeAg0iB7FvAyaTVguPamNDQO193fA+5r\n",
       "9uAiIp3USsB191lm9jSwgrs/a2bvAo8AH83sNtbdH4nlh/e1N9vfRoaFHTFMc+LuP2n2ZFJs/TQf\n",
       "br/QfLj5GhoWNjd361HAuWb2QcDd/V9mdjxwY9x+HOmGOcO1N6uRDHdx5v8dsQG6aCYiBdDqjQ3u\n",
       "/hSwbVXbFGDKMPsO296MRmq4x2Vfm9mCpEMppM8ouy0eZbf5yjaXQtPjad19FrBcB/oiItKUgVD/\n",
       "USSN1HCvqWpaDni2M92RXlINt3hUw83Xj0vsnFr1+lV314gFEem5ksXbhmq4t3ShH1IAym6LR9lt\n",
       "vqKVDOqpW8M1s9W70RERkWa1YS6FrmrkotlvO94LKQTNpVA8mksh3+BAqPsokkZquO90vBciIiNQ\n",
       "sAS2rkYC7nlmdipwYrbR3V/tTJekV1TDLR7VcPOVbZ7YRgLusaR3mn0505YAa3SkRyIiDSraMuj1\n",
       "NDJK4cNd6IcUgMbhFo/G4ebrx3G4IiKFVK5wm1MCMbPvdLMj0nvKbotH2W2+sq3am1dz3jZnm4hI\n",
       "zw2GUPdRJHklhQXMbOlaGzVKof+ohls8quHm66eLZp8CptfYplEKItJz7Qi3ZjYGeAiY5O5nmdlk\n",
       "YBwwE5js7hfG/VpeJj0v4N7t7uObPaCUl7Lb4lF2m69NGe4BzJ9cJsCucXJyYL5l0reKTTeY2c3u\n",
       "XnvFtGGUbdywiMiQVmu4ZrYo8DngauZPmKvf2JZl0vMC7unNHkzKTXMpFI/mUsgXGnjUcShwRlXb\n",
       "DOBiM7vGzCpBdWniMulxqfTKMulNqRlw3f3yZg8mItJNrQwLiwtHburu15OJze5+qLtvAnwXODk2\n",
       "v0K6TPrRwDHxedPLpKukIENUwy0e1XDzDRDqPnJsAixsZpeQ1nH3NrO1M9tnArPi80fpxjLpIiJF\n",
       "1cp8t+5+HXAdgJntBXzA3e83s0uBFUlLCwfHfbu2TLqMEhqHWzwah5uvXcNwK0O/4vPdauzT+WXS\n",
       "RUSKqk7JoHAUcGWIstviUXabr2Q3mingikh5FW3Nsno0SkGGaBxu8Wgcbr6BUP9RJMpwRaS0QgM1\n",
       "3Kbuve0wBVzAzC4AtgAOcfff1WvvV6rhFo9quPkaKSnM6UI/GqWSAuDuewOTG20XkWJQSaEDzOxu\n",
       "d99gmOcPANOAdYBr3f0Hsf0gYA/SX24XuPv5sf0LwPdI/8r4ibtf0fUPU2Aah1s8Goebr5GSQpGU\n",
       "PcNdGDgM+AywjZktH9t3AbZ19/GZYDsA/Ih0ZqDNgcPiPJgiUlJlW2KnFBlujpfc/S0AM5sOrA68\n",
       "AOwJ7G9mywJXufs0YFlgFeDa+N4lgZWAx7ve64JSdls8ym7zFW0JnXrKEnAHYGjuykUz7SvFZYDe\n",
       "AD5NWi7A3Z8BJpnZwqQlh/WAl4B/ADu4+5vt7mD2z/HK8Cq97t3rRQYHh4JVZWiVXvf29bgVFqHd\n",
       "yhVuISRJkQZNDM/MzgbeBt4Cdnb3dWL7/cDdwNrARe7+s9h+GrAusARwZqassAlwImkN9zl33z1z\n",
       "jonATsCF7n5qvfasqVOnJqz8yfZ+6B7opxruYguWJZfI10813LeemM6ECRPaFiOnTp2afODD69Xd\n",
       "7+02n7cVpQi4tWQvoPWSAm7xKOAWTycC7mIfXr+B8/65MAG37P8qy/vbooD6Jdj2k34Jtp1StGFf\n",
       "9ZQ64Lr7hr3ug4j0kAKulFU/lRT6RT+VFDqhlXG4ZnYCsDEwF9jP3R+rtRR6O5ZIh/KPwxWRUayV\n",
       "cbjufqy7f5Y0kB5lZoF0KfSt4+M4mG+J9KH2uG/TlOHKEGW3xaPsNl+bhuFuRDpkdGgpdAAze9TM\n",
       "xpImpvO1ky6R/nCzJ1LAFZHSavXWXjO7jfSmqPGki0S+HoeVwryl0EON9qYDrkoKMkTz4RaP5sPN\n",
       "1+qtve6+GfBV4JfUXgq9LUukgwKuiJRYm+ZSeJ70r/1HGH4p9LYskQ4qKUiGarjFoxpuvhZHKVxG\n",
       "Wk54j3TO67nDLYXeriXSQQFXREqslYtm7r7rMG3DLoXejiXSQSUFyVANt3hUw80XGngUiTJcESmt\n",
       "oOkZpaxUwy0e1XDzlSzeKuCKSHmVLN6qhivzqIZbPKrh1lGyIq4yXBEprUaWSS8SBVwZohpu8aiG\n",
       "m69c4VYBV0TKrGQRVzVcGaIabvGohpsvNPC/IlGGKyKlpSV2pLRUwy0e1XDrUMAVEemOopUM6lEN\n",
       "V4aohls8quHmGwj1H0WiDFdEyqtgAbUeBVwZohpu8aiGm69sJQUFXBEprZLdaKaAK/Pce9c0ZbkF\n",
       "M/3O25Xl5milRmtm44FTgVvd/cjYNhkYB8wEJrv7hbF9K9Ll1AEmuvtNIzmnAq6IlFhLKe4Y4CRg\n",
       "40xbAuzq7k9VGsxsADge2Co23WBmN7t70uwJNUpBhii7LR5lt/laWUTS3f8AvDrcYatejwUecvd3\n",
       "3P0d0kUl1xpJf5XhikhpdaCEOwO42MxeBQ6Pq/MuDbxuZqfFfd4AlgEebvbgynBliMbhFo/G4eYb\n",
       "CKHuoxnufqi7bwJ8Fzg5Nr8CLAkcDRwTn788kv4qwxWR8mo9xa11hJnArPj8UeCjmW1jY+bbNAVc\n",
       "GaIabvGohpuvlXhrZkcB2wArmNkS7r6/mV0GrEBaWjgYwN3nmNnxwI3xrceN9JwKuCJSWq2Mw3X3\n",
       "ScCkqrZda+w7BZgy8rOlVMOVIarhFo9quPlCCHUfRaIMV0RKq1jhtD4FXBmiGm7xqIabr2AJbF0K\n",
       "uCJSWmVbtVc1XBmiGm7xqIbbX5ThikhplSzBVcCVeVTDLR7VcPNpPlwRkS4p2hI69aiGK0NUwy0e\n",
       "1XDrCA08CkQZroiUlkoKUlqq4RaParj5dNFMRKRLyhZwVcOVIarhFo9quPlCA/8rEmW47fLsX3vd\n",
       "g5atu/JiffE5AN7qdQfaZNwKi/DWE9N73Y3CuuO2qb3uQlNCkjS9DpqIiIyASgoiIl2igCsi0iUK\n",
       "uCIiXaKAKyLSJQq4IiVmZhppVCIKuNIyM1vTzBbsdT9GGzM7BtjWzBbrdV+kMQq40hIz+wXwU2Bj\n",
       "MxvT6/6MFmZ2ILArsB6wiYJuOSjgyoiZ2UrAM8DlwMbARma2cNxWrFt8+s8l7v4J4H+ATYHxZrZE\n",
       "j/skdaj+IyPm7s8BEwHM7OvAFqS/xG8GFgLe7Vnn+t8MAHe/zswWJf2F97KZJUBw97t72jsZljJc\n",
       "aZqZLZ55vgCAu58HPAWsb2aHACea2Qd71MW+ZGY7mdlYAHefY2aD8fnlwH3AV4ELKNwssFKhW3ul\n",
       "KWb2M+B54GJ3fyK2Dbr7nPj8a8BOwOnufkPPOtpnzOxkYEfgNGCquz8Q27Pf/eeBV9z9z73rqeRR\n",
       "wJWGmdnWwKHAn4EngZszQXcBd58dny/v7i+YWXB3/QNrAzPbBHgC+AowkzTo/qPGvgFA333xKOBK\n",
       "08xsdeDrpEH3Jnd/rMZ+CrgtMrMBd59b+S7N7CPA7sAbwCXAGOAtd3+jpx2VhijgSl3DZUxmtjbp\n",
       "sKT7SC+OLUF65XxuTzrZh7Llgqr21YA9gKWB9YH93P2hbvdPmqeAK7mqSgWLu/uMzLa1gb2ArYFv\n",
       "uvstvell/8lktgPAzsB77n5VZvuqpN/9NHe/uVf9lOYo4Epd8Wr4ZOBF4E3gZHf/V9z2EWAhd39Q\n",
       "JYT2in9Z3AxcC2wJvAAc6u5vxe0Luvss1WzLQ+NwpREnkF4ouwa4m/RGh/sB3P3xHvar360K3OXu\n",
       "p5jZZsDV2Y3uPiv+vwJtSWgcrrxP9i6xmN3OJs1sv0t6sexNM9uu+n36wW9NZVxtxtvAF83sEeB0\n",
       "4F7gCjNbq+udk7ZQwJX5xAs1SXw+Jl60uRX4MfAa8CfS8oLuImujygUyMxsws38zs8Xc/RXga6S/\n",
       "8MYBvwFOc/dHetpZGTHVcGVI1YWaSaQ/4HfH4UhfJB2VMBu40t2vUc22veL3fh1wtbufXblgGS+Q\n",
       "fQiY4+5/Vc22vBRwZT7xh/k3wHR3PylOUDPL3V8ys0WAue7+rn7o28/M9gQ+7u7fMbN9gQ2Bs939\n",
       "nqr99IuupBRwZT5mtgZwCul9+f8JvAJ8Adje3Wf2sGt9p/IXReb1nqTf9Z+Am4DPA6+5++Te9FDa\n",
       "TTXcUa76Qk28a+x+YDxwEXApae1WE4y3UfYOsjieGXe/iHTY1xlAQvpL7+EedlPaTBnuKJa9UAN8\n",
       "A5jh7udntn+C9Or46dlB99Kaqu/dgcWBZ4GTgQeA5Uj/yrjI3aeohNA/lOGOYpkf+p8Ca5Au13Ks\n",
       "mS0dd/k4MNHdr9KE4u0Tv/cAnA1c5+6fBx4EDnL3xN1fAI6qBNuedlbaSgF3lIqBFuA8YHl3P4x0\n",
       "UpTVgL0B3P1Sd79NP/TtUVW+WQ7YDlgKwN0nASub2eZx+z9je6Lstn+opDDKVP95amZbAOcAR8Sh\n",
       "Xp8nvXDzHeBd/bC3V/xF93F3/984mfgFpGWFq4ArgL3d/X972UfpHGW4o08AMLMzzWxcnHBmb+AU\n",
       "MzuBdKD9FHefqWDbHlWZ7cbAtWa2sbs/TPrdHwz8GtgnBuLqO86kTyjgjhKVEkJmGNITwLlmtoa7\n",
       "TwP2A74E3OHuv9cPfftUarbx7rHbgcOA08xs0xh0twZmARvEt2iKyz6lksIoEmuxx7v79+LrA0in\n",
       "+NvL3R8ys/VIJ7U+zN1/38Ou9p044mMyMMHdX4t37p0J7Obud8bxz2cC/+7ur/Wwq9JBynBHkVgi\n",
       "WM3MLovXQMG7AAAFbUlEQVSvf046yP5XZvYBd58O7EY6NEnayN3vI12e6NLY9HvgLuL45jj+eXsF\n",
       "2/6mDHeUyN7VZGanASuRzv61PWkZ4U/Vdz5J+8VRCMeTzvx1XqzZapztKKGAO4pUrfB6Cul8q5Pc\n",
       "/R7NjdA9ZjYOGBOzXhlFFHBHmaqgu3wcZC89oMx29FHA7TON/BCrdCDSG7po1kfi/KmVycNrDutS\n",
       "sBXpDWW4fSIz+9Qg8APgA8BvgXvjygEi0mPKcPtEJmv9NemqDLeS3qu/Mcy/TpmI9IYy3D5iZisA\n",
       "p7r77vH1Z4HDga9UltYWkd5Rhltiw0we/jzwtpkdHZseB95Bk4eLFIIy3JKqmsT6O6RLsZxlZqsD\n",
       "R5Le2LA88GN3v7qXfRWRlAJuicW67PdJL5CtQ7rC7jFx23qkQfgxjfcUKYYFet0BackZpJOH7wxg\n",
       "ZleZ2WnufnicF0FECkQ13HI7F/ikme0RX+8KfMzMxmVHJSi7FSkGlRRKzszWIZ3Wb7K7X6DygUhx\n",
       "KcMtubgcyzeAg81sJWAQNO5WpIiU4faJuJqAxtqKFJgy3P7xNiizFSkyZbgiIl2iDFdEpEsUcEVE\n",
       "ukQBV0SkSxRwRUS6RLf2CmZ2HPD/gOeAhYDvufv1bT7H+sDJ7r5lVftOQHD3y5s83jeB/3b3d1ro\n",
       "01vuvlgD+y0FXEO66ObP3P3UJs5xC3BEs7dat3JOKS5luAKQAGe4+6aAAeeb2SLdOLG7X9FssI0O\n",
       "AxZt8fQNDdFx99fid3N+p87R5nNKQSnDlYrKMulPmNnTpJnVQwBm9gRwAvB1YBHgS+7+ZNy2B3Ag\n",
       "aWC5y92/VTmgme0F/Adp5nxP9mRm9hngx8BqDJPBmdkBwF6kq1ckwAR3n2VmCwN/AFYArjWz2cC/\n",
       "u/vT8X3rASeT3nH3KrCvu78ct30aOA94E7i98plbZWa7APsAi5P+EviKuz+Y2eXLZnYqsBxwiruf\n",
       "H983CPwI2Ij0Z/Esd7+oHX2SYlKGK/Mxs3HAB4HHMs0J8H/cfSN3/2Qm2K5NGoQ3j9nYgma2Z9y2\n",
       "MnAisIW7fw54PXsed/+Tu49nmAwuBrCdgM+6+3h338zdZ8X3zYzneh74YtxeCbYLxePt7u6bky43\n",
       "NClz6F+S/nm/BXA9MGbk39R8bnL3bWK/fgl8q2r72/GcWwAnmtmysX1fYG78HrYADjKzj7SpT1JA\n",
       "ynAF0kzvQDPbHngD2MXdZ1ftc+Iw75tAmqHeaGaQZnevxm0bAlPd/aX4egqwdYP92Zm03ttsffZj\n",
       "pJn5xbE/A8BMGKqJLuHuNwO4++1mNrPJ4w/L3V8xs3WBTwDjgBWrdrk+7veimd0BrEuapW8NfNjM\n",
       "Noz7LRI/w+Pt6JcUjwKuQJrBnuXuP2nyfbOAq7JlhIzZzP8ne7N/vo/k3+Zs4InqC3PRnBEcryFm\n",
       "dkF8+htgOrBK1S7Zzz4IvBufzwImuvs1neqbFItKClIxknrm9cAuZrZmpSEzl8OdwHgzWyq27dLE\n",
       "ca8A/tPMFq86ZtZM0iWEstsfBMaY2Y7V/XH3N4EXzGyT2L4drV90q9iBtI59PbAe7/9FY/GcqwKf\n",
       "Zl49+yrgP8xssarPIX1KAVcq8q6mD7vN3R8nreH+ysxuN7NpxGXZYynhu8BtwC11zjFfu7s7cCkw\n",
       "1cz+CNxaCb4ZZwO/NbM/AF+L75tDGvz2NbM7zOx24JDMe74OnB77uRFxwp8mHWxmt5lZdmHOE4D7\n",
       "gBuBfxB/EWQ+27tmdhNwHXCQu78d+3sJcCVwS/yct1WCbwPnlBLS5DUiIl2iDFdEpEsUcEVEukQB\n",
       "V0SkSxRwRUS6RAFXRKRLFHBFRLpEAVdEpEsUcEVEuuT/A9K0moIEE7LRAAAAAElFTkSuQmCC\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114558890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_true = np.array(predictions.map(lambda x: x.label).collect())\n",
    "y_pred = np.array(predictions.map(lambda x: x.prediction).collect())\n",
    "\n",
    "print classification_report(y_true, y_pred)\n",
    "evaluate(y_true, y_pred, \"GBT, Pipeline\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.77      0.95      0.85      1376\n",
      "        1.0       0.84      0.49      0.62       751\n",
      "\n",
      "avg / total       0.80      0.79      0.77      2127\n",
      "\n",
      "Test Error = 0.212505876822\n",
      "Accuracy: 78.7494123178\n",
      "\n",
      "\n",
      "Confusion matrix GBT, Pipeline (100 iterations)\n",
      "[[1305   71]\n",
      " [ 381  370]]\n"
     ]
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAVwAAAEnCAYAAADl8/suAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8HfO9//HXdwdBUbe61aVFpC0Pbd2OIkSjWkqdFh96\n",
       "cJTWpShVdZy6NElLNUVTPS49KFGt8ikPlCohcSl6UdqfHi0h7tQtEUITcpnfH/PdO5Nlr1lr7XWb\n",
       "WXk/PdbDWt+ZNfPda7I++7M/853vhCRJEBGR9uvrdgdERJYUCrgiIh2igCsi0iEKuCIiHaKAKyLS\n",
       "IQq4IiIdooDbQ0II+4cQ/hJCeDmE8GQIYWoIYeM27Wu3uI9n4z6HtWk/w+PPsUE7tt9JIYQvhRB+\n",
       "14Lt9MxnsqRZqtsdkNYIIewB/AiwJEnujm0fB55t0y6PBi5KkuTMNm0fgCRJ3gY+2c59lI0+k/JS\n",
       "hts7vg18pz/YAiRJ8pf45QQghLBWCOHqEMLTMTu9IISwQmb56BDCcyGEz4cQ7g8hvBJCuDKEEDLr\n",
       "nBlCeBbYBfhmzHDvyHYkhLAwhLBh5vW4EMIVFescGkL4Wwjhmfg4oWL5RnHbz1ZuL7PO1iGEu+M6\n",
       "j4QQvl6xfFzs/6khhL/Hn+f4Bj5TQghPhRCOCyE8EUK4LISwdwjh+RDCdzLrrBJC+FEI4eEQwj9D\n",
       "CA+FEHao2M6vgXOArTM/179VrDMpfr7nxc/k+RDC5vV+JvH4vRBCWDO+7gsh3BFCOKyRn1naKEkS\n",
       "PUr+AIYB7wAfzVmnD7gf+B4QgGWBK4GrM+uMBubFdYYDqwEzgTGDbO8O4NAq+1oIbJh5PRa4IvN6\n",
       "BDAH+Eim7T05fV9se7Ftndi3PePr9wMPAV/NrDMOmAXsH19/EngbGN7AZ/skcDGwSezHicBngBcz\n",
       "6wwHdgX64uvxwMODbOtg4Hc5+5oEvAwcHl+v2MhnEttPA27JfO6/6Pa/Tz0WPZTh9oY1SctDs3PW\n",
       "2RrYADg1Sc0FjgH2CSGsnlnvxSRJTk6S5O0kSWYAfwfWq7LNUKW91nqvkwa+XUMI7wdIkuStOrfV\n",
       "7yDgziRJbozvf540wBxTsd6vkyS5Kj7/HbA0sHaD+/ol8FR8fjXwDLBq/8L4WU1OkmRhbLoGGDnI\n",
       "dmp9XgngSZJcFLebdzyrOR1YKoTwM2Bf4PAhbEPaRAG3N/R/MdfIWWcD4OlMUCBJkpnAa3FZNfOo\n",
       "/u+k3ok4FlsvSZKXgW2A9YGpIYQHQwhj6txWv/WB6RVtj/Pun2UgyCVJMi8+Hcq/+4UVz7NllhBC\n",
       "OD6WN+4Bzgf6QghD2c9rQ3jPgCRNbb8JHAhMGMIvMmkjBdweEDOhv5N/IuUZ4IMhhKX7G0IIawCr\n",
       "AE+3ukss/m/rXZldkiTTkiT5RpIkI0n/9L8xhDC8gX08w7uzyA+xKBPt70cnfA3YB9g3SZIdSEsH\n",
       "XRGD/NnAT4HxIYSVu9UXeTcF3N5xJnBCCGFMWGRU5sTKH0mD0ZkhhGEhhOVJM7FrkyR5tca2q/0p\n",
       "XK39OWAzgDgs7YtUBL+KEz7LkJYYFtToR9YVwE4hhC/E7a1PWju9oI7+tdq6wIvAy7E8c1ZsX7pi\n",
       "vdeAESGEFePxeX/F8lb091RgZpIkh5GWQi5pwTalRRRwe0SSJL8gzbTOIT3x8jTwX8AKcXkC7En6\n",
       "p/iTpBnxDODQyk0Ntvlqu63S/k3grBDCb+PzydmFIYSVAI8jIp4lHWK2W5Ik8+vdT5IkL5CeqDo+\n",
       "hPAccBvpya0LK95X+d5mst6kyvNzgNWBF4DfABcB80lP5GXdAjwIPAE8wrvrq4P1t56+ABBC2Im0\n",
       "lPCV2DQWWC+EcGSd25Q2C+n3UERE2k0ZrohIhyjgioh0iAKuiEiHKOCKiHSIJq9pgSlTpujMo0gd\n",
       "xowZ07Kheo1871q532Yo4LbIHt+8rttdaNqC2c8zbMXKkUzl9Nr953W7Cy1x9113suNOo7vdjZa4\n",
       "964pLd/mHifeUHOdm87aq+X7HSoFXBEpr1CIxLVuCrgyoFey217SK9lt2wxpuoruUcAVkfIqWYZb\n",
       "rl8P0lYLZj/f7S5IhbvvurPbXSi2vmG1HwWiDFdEykslBSkr1XCLRzXcGkpWUlDAFZHyKljJoJZy\n",
       "5ePSVqrhFo9quDWEvtqPAlGGKyLlpZKClJVquMWjGm4NBctga1HAFZHyGlauGq4CrgzopbkUekUv\n",
       "zaXQFk1kuGY2ivT2SHe5+4mx7SekNyftAw5x9ydi+y6ktywCGOvuU/PaqylXPi4ikhVC7Ud1w0lv\n",
       "vjrA3Y90951Jb0jaH4T74utd42NctXYzy92hAq4MUHZbPMpua2hilIK73w7MrLJ4NvBOfD4CmObu\n",
       "c9x9DjDdzEYM1g5snNddlRREpLzaNw73UODc+HxVYJaZTYyvXwdWI72t/WDtj1XbqDJcGaBxuMWj\n",
       "cbg1NFdSGJSZ7Qk86u6PxKYZwMrAycAp8fmrOe1VKeCKSHk1f+HDYhHZzLYEdnL3H2WapwObZF6P\n",
       "cPfHc9qrUsCVAarhFo9quDU0keGa2UmkJ8D2NLP/jc2/ArY2szvM7McA7r6A9OTYbcDk+J6q7XlU\n",
       "wxWR8mqihuvuE4AJFW0bVll3MmlQrau9GmW4MkA13OJRDbcGzaUgItIhBQuotSjgygDVcItHNdwa\n",
       "SjY9owKuiJRXyWYLK1c+Lm2lGm7xqIZbg2q4IiIdUrIMVwFXBqiGWzyq4ebr6ytWBluLAq6IlFe5\n",
       "ElzVcGUR1XCLRzXcfCGEmo8iUYYrIqVVtIBaiwKuDFANt3hUw82nGq6ISKeUK8FVDVcWUQ23eFTD\n",
       "zacarohIhxQtoNaigCsDVMMtHtVw86mGKyLSKeVKcBVwZZEFs59Xllswd991p7LcHM2UFMxsFHAO\n",
       "cJe7998SfRdgbFxlrLtPje2TgJHAXGCSu1+et341CrgiUlpNlhSGA2cC2wGYWR/pLXN2ictvBfoD\n",
       "aALs5+7P9L95sPXN7A53T6r2t5neSm9Rdls8ym5rCHU8qnD324GZmaYRwDR3n+Puc4DpZjaiYm/k\n",
       "rQ9snNddZbgiUlotHqWwKjDLzCbG168DqwGPAbOBK81sJnB8vDtv3vqDUoYrAzQOt3g0Djdfi8fh\n",
       "zgBWBk4GTonPXwVw92PdfXvgNOCsWutXowxXREqrBcPCshF5OrBJ5vWImMlmzQXmNbD+YhRwZYBq\n",
       "uMWjGm4NTVQUzOwkYDdgLTNbyd2PMLPxwG1xlXGZda8C1iYtLRwN4O4Lqq1fjQKuiJRWMzVcd58A\n",
       "TKhomwxMHmTd/atsY9D1q1ENVwaohls8quHm01wKJWRmlwGjgWPc/Te12kWkGEJfsQJqLcpwAXc/\n",
       "BJhUb3uvUg23eFTDzacMN8PMHiItKG8PTHX3k2P7wcARpCXvS9394hrbud/dtx7k+SPAvcBmwE3u\n",
       "/t3YfhRwILAAuMzdL43tnwG+TXrVyA/d/doW/8gi0kFFC6i1tDvDXQX4AWnA3RPAzFYHjgR2BHYA\n",
       "DjCzoaZWywLHAZ8AdjOzNWP7vsDu7j4qE2z7gO8DnwJ2Ao4zs+FD3G9PUg23eFTDzacMd3EvuvtL\n",
       "AGY2N7ZtCPzZ3efH9j+QXiI3lG/7K+7+ZtzOA8AGwEvAQcARMbhf7+73AqsD6wI3xfeuDKwDPDmU\n",
       "H0xEuq9sNdxunDR7AtjKzJYm/dN+O+DcGu/pAzCz5YHlM+3rmNmqpJfUbUFaLsDdnwMmmNmypCWH\n",
       "LYFXgH8Ae7n7G637cVLZmbb6M0W97t7r7Cxb/VliGV/vuNPoQvWnmdfDaL2iZbC1hCSpOrFN08zs\n",
       "T+6+zSDPszXcn7r7JTW2cyHwFvAmsI+7bxbbHwbuBzYFrnD3H8f2icDHgJWA8zNlhe2BM0gD/Qvu\n",
       "fkBmH2OBvYHL3f2cWu1ZU6ZMSfb45nUNfTbSXq/df163uyAV7r1rCmPGjGlZhJwyZUpy+M3v1Fzv\n",
       "ot2Xael+m9HWgNtu2RNo3dQrAbeX5sPtlYDbS/PhtiPgHnnLvJrr/eQzSxcm4JZ9HG55f1uISNNK\n",
       "VlEod8DtL1FIa/RKdttLeiW7bZey1XBLHXBFZMlWsnirK81kEY3DLR6Nw803bFio+SgSZbgiUloq\n",
       "KUhpqYZbPKrh5itZvFXAFZHyKluGqxquDFANt3hUw83X1xdqPopEGa6IlFZ9GW714fpmdgTwJdKr\n",
       "WI9y98fMbBdgbFxlrLtPjesO2t4IZbgyQDXc4lENN18ItR/VxLlZDnH3TwBfBL5nZgEYD+waH+Pi\n",
       "un2V7XHdhijDFZHSajLDDcDScZrWWcBapDMXTnP3OQBmNt3MRpAmp4u1AxsDjzXSXwVcGdBLcyn0\n",
       "il6aS6EdmqnRuvtbZvY94Lekd+NdhTTozooTYEE6E+FqpMF5sHYFXBFZMjQ7SCHe9eVaADN7EHiR\n",
       "dK7so0iD7AXAq6QZ7mDtDRlSDTdOlyg9Rtlt8Si7zdeqOz6Y2e7AX4HHgU0yi0a4++PA9CrtDama\n",
       "4ZrZlgxe/AjAto3uSESk1Zod9mVmPwVGko5SONDdF5rZeNJ7MUI8aebuCwZrb1ReSWEK8Jcqy2YN\n",
       "ZWdSbKrhFo9quPlaUFL48iBtk4HJ9bY3Ii/g/s3dd25m4yIi7VS2K83yAu7YnGXSg5TdFo+y23wl\n",
       "i7fVA+5QrqIQEemkvr5yXbtVd2/NbJV2dkS6T3MpFI/mUsjXzJVm3VBzHG4crXApMBz4ULzE7RJ3\n",
       "P7TdnRMRyVO2Gm49Ge4Pgc8B/wRw94Wkl79Jj1ENt3hUw81Xtgy3noCbuPvTFW3D29EZEZFG9OL0\n",
       "jK+a2W5AMLMVge9QfXyulJjG4RaPxuHm6ytaCltDPRnuV4GDgc1IJ2pYHjihnZ0SEalH2UoKNTNc\n",
       "d38F2L8DfZEuU3ZbPMpu85XtpFnds4WZ2dLAfHevPn26iEgHDStYjbaWmiUFM9vKzB4AXgBeNrPJ\n",
       "ZrZR+7smnaZxuMWjcbj5ylZSqKeGewlwsru/z93fB1wIXNHebomI1Bbq+K9I6gm4/3L3W/tfuPt1\n",
       "wIL2dUm6RTXc4lENN9+wvlDzUST1BNxbzWzH/hdm9lng9vZ1SUSkPmUrKeRNQP4m6QTkATjVzObG\n",
       "RcsD/yK9g6X0EI3DLR6Nw81XtnG4ebOFrdDJjoiINKpk8VY3kZRFlN0Wj7LbfM1cumtmKwE3ZJq2\n",
       "cPf3mtkk0tvuzAUmufvlcf1dWDRP+NihTGFbz2xhywKfB9YmLS8EYE13P7HRnYmItFIzJQV3fwPY\n",
       "GcDMNge+FhclwH7u/kz/unGWxPHALrHpVjO7o9HrEuo5aXYtsB+wG7ARsDcwv5GdSDloHG7xaBxu\n",
       "vlDHo07HAv9TsemsEcA0d5/j7nNI7+K7caP9raeksI67f9zMDo87ORn4eaM7EhFptVZc2mtmqwHr\n",
       "uftDsWk2cKWZzQSOj7dDXxWYZWYT4zqvA6uRzi9Tt3oy3Jfj/6cB27n7LGC9RnYi5aAabvGohpuv\n",
       "ReNwDwcu6n/h7se6+/bAacBZsXkGsDJpwnlKfP5qo/2tJ+D+wczeB9wD7GVm9wF/b3RHIiKt1uw4\n",
       "XDNbCtgDuG6QxXOBefH5dGCTzLIRMfNtSD2zhQ3cvdfMdiI9e/fXRnckxadxuMWjcbj5WlBS+Hfg\n",
       "xngnGwDM7CrSQQKzgaMB3H2BmY0HbourjRvKzhoaFububwEPDmVHIiKt1uyVu+5+zSBtg05H6+6T\n",
       "gcnN7E/jcGWAstviUXabr2xXmg3ppu5mdmGrOyIi0qi+EGo+iiRvLoUtSQcAVwrAtm3rkXSNarjF\n",
       "oxpuvoLF05rySgpTqH6zyFlt6IuISEOKdlfeWvIC7t/cfeeO9US6Ttlt8Si7zVe0kkEteQF3bM4y\n",
       "EZGuK1m8zZ2eseGZcJZkU/y73e5C0x784z1s8W87dLsbLfHXp3qj6tVLx6QdevauvSIiRTNMAVfK\n",
       "SplU8eiY5CvZOTMFXBEpr7IF3CFd+CC96cE/3tPtLkgFHZN8IYSajyJRhisipTWsZCmjAq4MUL2w\n",
       "eHRM8vXSOFwRkUIrWYJbuv5KG6leWDw6JvmanYC80+q5a+/qwARgXXf/tJkF4Gh3P6/tvRMRyVHn\n",
       "LXQKo56SwsXAL4i3EHb3xMz2BxRwe4zqhcWjY5Kv2XhrZusCV5DGwj+5+wlmtguLpjYY23/VbbX2\n",
       "hvpbxzqrxlnRF2baVIoQka5rwXy4ZwOnuPuoGGz7gPHArvExDmCw9vjXfmP9rWOdBWa2dv8LM/s8\n",
       "mp6xJ6leWDw6JvmaqeGa2TBgI3e/L9M8Apjm7nPcfQ4w3cxGDNYObNxof+spKZwI3AKsZ2YPAsOB\n",
       "vRrdkYhIqzU5l8L7gGXN7HpgJeB/gBeBWWY2Ma7zOrAa6Y0XBmt/rJEd1nPX3gfMbGvgQ8B84FF3\n",
       "X9DITqQcVC8sHh2TfHXVcAe7b01qBmng3BsYBtwLfBlYGTiKNMheALxKWg0YrL0hdY3Ddfd3gIca\n",
       "3biISDs1E3DdfZ6ZPQus5e7Pm9nbwOPAJpnVRrj747H88K72Rvtbz7CwEwZpTtz9h43uTIpNc68W\n",
       "j45JvrqGhS3MXXoScLGZvRdwd/+XmY0HbovLx5EuWDBYe6PqyXBXZPHfEVujk2YiUgDNXtjg7s8A\n",
       "u1e0TQYmD7LuoO2NqKeGOy772syWJh1KIT1GmVTx6JjkK9tcCg2Pp3X3ecAabeiLiEhD+kLtR5HU\n",
       "U8O9saJpDeD59nRHukn1wuLRMcnXi7fYOafi9Ux314gFEem6ksXbumq4d3agH1IAyqSKR8ckX9FK\n",
       "BrXUrOGa2Qad6IiISKNaMJdCR9Vz0uzXbe+FFIKu2y8eHZN8w/pCzUeR1FPDndP2XoiIDEHBEtia\n",
       "6gm4l5jZOcAZ2UZ3n9meLkm3qF5YPDom+co2T2w9AfdU0ivNvpBpS4AN29IjEZE6Fe026LXUM0rh\n",
       "Ax3ohxSAxnwWj45Jvl4chysiUkjlCrc5JRAz+1YnOyLdp0yqeHRM8pXtrr15Nefdc5aJiHTdsBBq\n",
       "Pookr6SwlJmtWm2hRin0HtULi0fHJF8vnTT7OPBAlWUapSAiXVeucJsfcO9391Ed64l0nTKp4tEx\n",
       "yddLGa6ISKG1okZrZsOBacAEd7/AzCYBI4G5wCR3vzyutwswNr5trLtPbXRfeSfNzm10Y1Juum6/\n",
       "eHRM8oU6HnU4ksXLpwmwn7vvnAm2fcB4YNf4GGdmDUf7qgHX3a9pdGMiIp3U7LAwM1se+BRwA4vH\n",
       "58p3jgCmufscd58DTAc2brS/KinIANULi0fHJF9f86fNjgXOA9bMtM0GrjSzmcDx8XboqwKzzGxi\n",
       "XOd1YDXgscb6KyJSUs3Mhxtvjb6Du99CJqN192PdfXvgNOCs2DwDWBk4GTglPn+14f42+gbpXaoX\n",
       "Fo+OSb4mSwrbA8ua2S9J67iHmNmmmeVzgXnx+XRgk8yyETHzbYhKCiJSWs2UFNz9ZuBmADM7GHiP\n",
       "uz9sZlcBa5OWFo6O6y4ws/HAbfHt44ayz5AkyZA7LKkpU6Yky62/Rbe7IRlFu7WKwFtPP8CYMWNa\n",
       "dmCmTJmSzF9785rrLfXPh1q632YowxWR0irbL1bVcGWA6oXFo2OSry/UfhSJMlwRKa1QRw23SEVT\n",
       "BVzAzC4DRgPHuPtvarX3Ko35LB4dk3z1lBQWdKAf9VJJAXD3Q4BJ9baLSDGopNAGZna/u289yPNH\n",
       "gHuBzYCb3P27sf0o4EDSX26Xufulsf0zwLdJ/8r4obtf2/EfpsA092rx6Jjkq6ekUCRlz3CXBY4D\n",
       "PgHsZmb9l+ftC+zu7qMywbYP+D7pddM7AcfFWYJEpKTKdoudUmS4OV5x9zcBzOwBYAPgJeAg4Agz\n",
       "Wx243t3vBVYH1gVuiu9dGVgHeLLjvS4oZVLFo2OSr2i30KmlLAG3DwZm9lk+075OvA3Q68AWpOUC\n",
       "3P05YIKZLUtactgSeAX4B7CXu7/R6g5m//TrH8qj19173RdCofqj1zByreVotXKF25JcaWZmFwJv\n",
       "AW8C+7j7ZrH9YeB+YFPgCnf/cWyfCHwMWAk4P1NW2B44g7SG+4K7H5DZx1hgb+Bydz+nVntWr1xp\n",
       "1kv1wrINiK+ml45JO640e88Htqy936dau99mlCLgVpM9gdZNCrjFo4BbPO0IuCt8YKua67351J8L\n",
       "E3DLUlKopry/LQqoV77YvUTHJF/Rhn3VUuqA6+7bdLsPItJFJQu4ZR8WJi2k6/aLR8ckX6jjvyIp\n",
       "dYYrIku2spXqFXBlgOqFxaNjkk8BV0SkQ5opGZjZ6cB2wELgcHd/wsx2AcbGVca6+9S47qDtjVIN\n",
       "VwaoXlg8Oib5mrm0191PdfdPkgbSk8wsAOOBXeNjHAxMC7BYe1y3YcpwRaS0WlRS2Jb0KtQRwDR3\n",
       "nwNgZtPNbARpYrpYO7AxDd4iHRRwJUP1wuLRMcnX7CgEM7ubdJ6VUaR35Z0Vr1SFdMqA1UgHnw3W\n",
       "3nDAVUlBREqr2dnC3H1H4EvAz4AZpJNanQycEp+/mtPeMAVcGaB6YfHomOQLdTzq8CLpX/uPk2a5\n",
       "/Ua4++PA9CrtDVNJQURKKzRRxDWzq0nLCe+Q3kZroZmNB26Lq4wDcPcFg7UPqb9lnrymKHpl8ppe\n",
       "0iuT1/SSdkxes/aHt6253j//8QdNXiMi0qxCRNEGqIYrA1QvLB4dkxpaVMTtFGW4IlJaZSsdKeDK\n",
       "AI35LB4dk3zlCrcKuCJSZiWLuKrhygDVC4tHxySf5sMVEekQ3WJHSkv1wuLRMalBAVdEpDOKVjKo\n",
       "RTVcGaB6YfHomOTrC7UfRaIMV0TKq2ABtRYFXBmgemHx6JjkK1tJQQFXREqrZBeaqYYri6heWDw6\n",
       "JvlUwxUR6ZiCRdQaFHBlgOqFxaNjkq9sJQUFXBEprZLFWwVcWeTBP96jjKpgdEzyNTM9o5mNAs4B\n",
       "7nL3E2PbJGAkMBeY5O6Xx/ZdgLHxrWPdfepQ9qmAKyLl1VyKOxw4E9gu05YA+7n7M/0NZtYHjAd2\n",
       "iU23mtkd7t7w/ck0SkEGKJMqHh2TfM3c8MHdbwdmVtls1ghgmrvPcfc5pHfx3Xgo/VWGKyKl1YaT\n",
       "ZrOBK81sJnB8vB36qsAsM5sY13kdWA14rNGNK8OVARrzWTw6JvlCCDUfjXD3Y919e+A04KzYPANY\n",
       "GTgZOCU+f3Uo/VWGKyKl1YIEt9om5gLz4vPpwCaZZSNi5tswBVwZoHph8eiY5GumpGBmJwG7AWuZ\n",
       "2UrufoSZXQ2sRVpaOBrA3ReY2XjgtvjWcUPdpwKuiJRWM8PC3H0CMKGibb8q604GJg95Z5FquDJA\n",
       "9cLi0THpLcpwRaS0dGmvlJbqhcWjY5JP8+GKiHRI0aZfrEU1XBmgemHx6JjU0MylZl2gDFdESksl\n",
       "BSkt1QuLR8ckn06aiYh0SNkCrmq4MkD1wuLRMckX6vivSJThtsicZx7sdhea9uG1l++Jn6OXjFxr\n",
       "Od56+oFud6Ow7rt7Sre70JCQJA3PoSsiIkOgkoKISIco4IqIdIgCrohIhyjgioh0iAKuSImZmUYa\n",
       "lYgCrjTNzDYys6W73Y8ljZmdAuxuZit0uy9SHwVcaYqZ/RT4EbCdmQ3vdn+WFGb2VWA/YEtgewXd\n",
       "clDAlSEzs3WA54BrgO2Abc1s2bisWJf49J5fuvvmwB+BHYBRZrZSl/skNaj+I0Pm7i8AYwHM7CvA\n",
       "aNJf4ncAywBvd61zvW82gLvfbGbLk/7Ce9XMEiC4+/1d7Z0MShmuNMzMVsw8XwrA3S8BngG2MrNj\n",
       "gDPM7L1d6mJPMrO9zWwEDNxJdlh8fg3wEPAl4DIKNwus9NOlvdIQM/sx8CJwpbs/FduGufuC+PzL\n",
       "wN7Aue5+a9c62mPM7Czg88BEYIq7PxLbs5/9p4EZ7v7n7vVU8ijgSt3MbFfgWODPwNPAHZmgu5S7\n",
       "z4/P13T3l8wsuLv+gbWAmW0PPAV8EZhLGnT/UWXdAKDPvngUcKVhZrYB8BXSoDvV3Z+osp4CbpPM\n",
       "rM/dF/Z/lmb2QeAA4HXgl8Bw4E13f72rHZW6KOBKTYNlTGa2KemwpIdIT46tRHrmfGFXOtmDsuWC\n",
       "ivb1gQOBVYGtgMPdfVqn+yeNU8CVXBWlghXdfXZm2abAwcCuwNfd/c7u9LL3ZDLbPmAf4B13vz6z\n",
       "fD3Sz/5ed7+jW/2UxijgSk3xbPgk4GXgDeAsd/9XXPZBYBl3f1QlhNaKf1ncAdwE7Ay8BBzr7m/G\n",
       "5Uu7+zzVbMtD43ClHqeTnii7Ebif9EKHhwHc/cku9qvXrQf8yd3PNrMdgRuyC919Xvy/Am1JaByu\n",
       "vEv2KrGY3c4nzWxPIz1Z9oaZ7VH5Pn3xm9M/rjbjLeCzZvY4cC7wV+BaM9u4452TllDAlcXEEzVJ\n",
       "fD48nrS5C/gB8Brwe9Lygq4ia6H+E2Rm1mdm/25mK7j7DODLpL/wRgK/Aia6++Nd7awMmWq4MqDi\n",
       "RM0E0i/4/XE40mdJRyXMB65z9xtVs22t+LnfDNzg7hf2n7CMJ8jeByxw9/+nmm15KeDKYuKX+VfA\n",
       "A+5+ZpygZp67v2JmywEL3f1tfelbz8wOAj7i7t8ys8OAbYAL3f3BivX0i66kFHBlMWa2IXA26XX5\n",
       "/w3MAD4D7Onuc7vYtZ7T/xdF5vVBpJ/174GpwKeB19x9Und6KK2mGu4SrvJETbxq7GFgFHAFcBVp\n",
       "7VYTjLdQ9gqyOJ4Zd7+CdNjXeUBC+kvvsS52U1pMGe4SLHuiBvgaMNvdL80s35z07Pi52UH30pyK\n",
       "z92BFYHngbOAR4A1SP/KuMLdJ6uE0DuU4S7BMl/6HwEbkt6u5VQzWzWu8hFgrLtfrwnFWyd+7gG4\n",
       "ELjZ3T8NPAoc5e6Ju78EnNQfbLvaWWkpBdwlVAy0AJcAa7r7caSToqwPHALg7le5+9360rdGRflm\n",
       "DWAPYBUAd58AvN/MdorL/xnbE2W3vUMlhSVM5Z+nZjYauAg4IQ71+jTpiZtvAW/ry95a8RfdR9z9\n",
       "/+Jk4peRlhWuB64FDnH3/+tmH6V9lOEueQKAmZ1vZiPjhDOHAGeb2emkA+0nu/tcBdvWqMhstwNu\n",
       "MrPt3P3aXKArAAAGCUlEQVQx0s/+aOAXwKExEFdecSY9QgF3CdFfQsgMQ3oKuNjMNnT3e4HDgc8B\n",
       "97n7b/Wlb53+mm28euwe4DhgopntEIPursA8YOv4Fk1x2aNUUliCxFrseHf/dnx9JOkUfwe7+zQz\n",
       "25J0Uuvj3P23Xexqz4kjPiYBY9z9tXjl3vnA/u7+hzj++XzgP9z9tS52VdpIGe4SJJYI1jezq+Pr\n",
       "n5AOsv+5mb3H3R8A9icdmiQt5O4Pkd6e6KrY9FvgT8TxzXH8854Ktr1NGe4SIntVk5lNBNYhnf1r\n",
       "T9Iywu8rr3yS1oujEMaTzvx1SazZapztEkIBdwlScYfXs0nnW53g7g9qboTOMbORwPCY9coSRAF3\n",
       "CVMRdNeMg+ylC5TZLnkUcHtMPV9ilQ5EukMnzXpInD+1f/LwqsO6FGxFukMZbo/IzD41DPgu8B7g\n",
       "18Bf450DRKTLlOH2iEzW+gvSuzLcRXqt/naw+H3KRKQ7lOH2EDNbCzjH3Q+Irz8JHA98sf/W2iLS\n",
       "PcpwS2yQycNfBN4ys5Nj05PAHDR5uEghKMMtqYpJrL9FeiuWC8xsA+BE0gsb1gR+4O43dLOvIpJS\n",
       "wC2xWJf9DukJss1I77B7Sly2JWkQfkLjPUWKYalud0Cach7p5OH7AJjZ9WY20d2Pj/MiiEiBqIZb\n",
       "bhcDHzWzA+Pr/YAPmdnI7KgEZbcixaCSQsmZ2Wak0/pNcvfLVD4QKS5luCUXb8fyNeBoM1sHGAYa\n",
       "dytSRMpwe0S8m4DG2ooUmDLc3vEWKLMVKTJluCIiHaIMV0SkQxRwRUQ6RAFXRKRDFHBFRDpEl/YK\n",
       "ZjYO+E/gBWAZ4NvufkuL97EVcJa771zRvjcQ3P2aBrf3deB/3X1OE316091XqGO9VYAbSW+6+WN3\n",
       "P6eBfdwJnNDopdbN7FOKSxmuACTAee6+A2DApWa2XCd27O7XNhpso+OA5ZvcfV1DdNz9tfjZXNqu\n",
       "fbR4n1JQynClX/9t0p8ys2dJM6tpAGb2FHA68BVgOeBz7v50XHYg8FXSwPInd/9G/wbN7GDgv0gz\n",
       "5wezOzOzTwA/ANZnkAzOzI4EDia9e0UCjHH3eWa2LHA7sBZwk5nNB/7D3Z+N79sSOIv0iruZwGHu\n",
       "/mpctgVwCfAGcE//z9wsM9sXOBRYkfSXwBfd/dHMKl8ws3OANYCz3f3S+L5hwPeBbUm/ixe4+xWt\n",
       "6JMUkzJcWYyZjQTeCzyRaU6AD7v7tu7+0Uyw3ZQ0CO8Us7GlzeyguOz9wBnAaHf/FDArux93/727\n",
       "j2KQDC4GsL2BT7r7KHff0d3nxffNjft6EfhsXN4fbJeJ2zvA3Xcivd3QhMymf0b65/1o4BZg+NA/\n",
       "qcVMdffdYr9+BnyjYvlbcZ+jgTPMbPXYfhiwMH4Oo4GjzOyDLeqTFJAyXIE00/uqme0JvA7s6+7z\n",
       "K9Y5Y5D3jSHNUG8zM0izu5lx2TbAFHd/Jb6eDOxaZ3/2Ia33Nlqf/RBpZn5l7E8fMBcGaqIrufsd\n",
       "AO5+j5nNbXD7g3L3GWb2MWBzYCSwdsUqt8T1Xjaz+4CPkWbpuwIfMLNt4nrLxZ/hyVb0S4pHAVcg\n",
       "zWAvcPcfNvi+ecD12TJCxnwW/5O90T/fh/Jvcz7wVOWJuWjBELZXFzO7LD79FfAAsG7FKtmffRjw\n",
       "dnw+Dxjr7je2q29SLCopSL+h1DNvAfY1s436GzJzOfwBGGVmq8S2fRvY7rXAf5vZihXbzJpLeguh\n",
       "7PJHgeFm9vnK/rj7G8BLZrZ9bN+D5k+69duLtI59C7Al7/5FY3Gf6wFbsKiefT3wX2a2QsXPIT1K\n",
       "AVf65Z1NH3SZuz9JWsP9uZndY2b3Em/LHksJpwF3A3fW2Mdi7e7uwFXAFDP7HXBXf/DNuBD4tZnd\n",
       "Dnw5vm8BafA7zMzuM7N7gGMy7/kKcG7s57bECX8adLSZ3W1m2Rtzng48BNwG/IP4iyDzs71tZlOB\n",
       "m4Gj3P2t2N9fAtcBd8af8+7+4FvHPqWENHmNiEiHKMMVEekQBVwRkQ5RwBUR6RAFXBGRDlHAFRHp\n",
       "EAVcEZEOUcAVEekQBVwRkQ75/9W4p5moKvxDAAAAAElFTkSuQmCC\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1144aa450>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_true = np.array(predictions.map(lambda x: x.label).collect())\n",
    "y_pred = np.array(predictions.map(lambda x: x.prediction).collect())\n",
    "\n",
    "print classification_report(y_true, y_pred)\n",
    "evaluate(y_true, y_pred, \"GBT, Pipeline (100 iterations)\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}