bayes_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iconv: cannot open input file `vk_feed.csv': No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "!iconv -f cp1251 -t utf-8 vk_feed.csv -o vk_feed_utf.csv "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>status</th>\n",
       "      <th>predict_proba</th>\n",
       "      <th>predict</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ребят краснодар нужны доноры положительная для...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>волгоград прошу максимальный перепост предыдущ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>срочно требуются доноры крови любая цельная кр...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>анонимно люди крик о помощи срочно а точнее в ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>люди крик о помощи срочно а точнее в понедельн...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  status  predict_proba  \\\n",
       "0  ребят краснодар нужны доноры положительная для...       1              1   \n",
       "1  волгоград прошу максимальный перепост предыдущ...       1              1   \n",
       "2  срочно требуются доноры крови любая цельная кр...       1              1   \n",
       "4  анонимно люди крик о помощи срочно а точнее в ...       1              1   \n",
       "5  люди крик о помощи срочно а точнее в понедельн...       1              1   \n",
       "\n",
       "  predict lang  \n",
       "0    True   ru  \n",
       "1    True   ru  \n",
       "2    True   ru  \n",
       "4    True   ru  \n",
       "5    True   ru  "
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('cleaned_data.csv', index_col=0)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from langdetect import detect\n",
    "\n",
    "def detect_language(text):\n",
    "    try:\n",
    "        lang = detect(text)\n",
    "        return lang\n",
    "    except:\n",
    "        return '-'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#data['lang']= data['text'].apply(lambda x: detect_language(x))\n",
    "#data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>status</th>\n",
       "      <th>predict_proba</th>\n",
       "      <th>predict</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lang</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ru</th>\n",
       "      <td>26108</td>\n",
       "      <td>26108</td>\n",
       "      <td>26108</td>\n",
       "      <td>26108</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       text  status  predict_proba  predict\n",
       "lang                                       \n",
       "ru    26108   26108          26108    26108"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.groupby(by='lang').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = data[data['lang'] == 'ru']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>predict_proba</th>\n",
       "      <th>predict</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>status</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1838</td>\n",
       "      <td>1838</td>\n",
       "      <td>1838</td>\n",
       "      <td>1838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24270</td>\n",
       "      <td>24270</td>\n",
       "      <td>24270</td>\n",
       "      <td>24270</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         text  predict_proba  predict   lang\n",
       "status                                      \n",
       "0        1838           1838     1838   1838\n",
       "1       24270          24270    24270  24270"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.groupby(by='status').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from html2text import html2text\n",
    "\n",
    "def preprocess(text):\n",
    "    text = html2text(text)\n",
    "    text = re.sub('[\\W]', ' ', text)\n",
    "    text = re.sub('_', ' ', text)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    text = re.sub('[A-Za-z0-9]', '', text) #removing english\n",
    "    text = text.lower()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['text'] = data['text'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26108, 557793)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import RussianStemmer\n",
    "\n",
    "class Preprocessor:\n",
    "    def __init__(self, stemmer, analyser):\n",
    "        self.stemmer = stemmer\n",
    "        self.analyser = analyser\n",
    "        \n",
    "    def __call__(self, doc):\n",
    "        return (self.stemmer.stem(w) for w in self.analyser(doc) if len(w) > 3)\n",
    "    \n",
    "preporcessor = Preprocessor(RussianStemmer(), CountVectorizer(ngram_range=(1,3)).build_analyzer())\n",
    "\n",
    "vectorizer = CountVectorizer(stop_words=stopwords.words('russian'), analyzer=preporcessor)\n",
    "V = vectorizer.fit_transform(data['text'])\n",
    "\n",
    "print(V.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = V\n",
    "y = data['status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "only integer arrays with one element can be converted to an index",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-322-8e32068bb343>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtrain_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_index\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkfold\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtest_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: only integer arrays with one element can be converted to an index"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "kfold = StratifiedKFold(y, n_folds=4, shuffle=True)\n",
    "\n",
    "for train_index, test_index in kfold:\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    \n",
    "    model = MultinomialNB()\n",
    "    model.fit(X_train, y_train)\n",
    "    print(model.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 325,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.996629385628926"
      ]
     },
     "execution_count": 326,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "\n",
    "Feature = namedtuple('Feature', ['word', 'weight'])\n",
    "word_dict = {}\n",
    "\n",
    "for key, value in vectorizer.vocabulary_.items():\n",
    "    word_dict[value] = Feature(key, model.coef_[0][value])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Feature(word='кров', weight=-4.0573687612639882),\n",
       " Feature(word='нужн', weight=-4.2801885065018483),\n",
       " Feature(word='срочн', weight=-4.3985958338801439),\n",
       " Feature(word='групп', weight=-4.6104997009582025),\n",
       " Feature(word='донор', weight=-4.6587437647053962),\n",
       " Feature(word='отрицательн', weight=-4.9339431916588401),\n",
       " Feature(word='помощ', weight=-5.0349698966569525),\n",
       " Feature(word='пожалуйст', weight=-5.0750081033665975),\n",
       " Feature(word='друз', weight=-5.1076362841449816),\n",
       " Feature(word='очен', weight=-5.1415375671885455),\n",
       " Feature(word='помог', weight=-5.1856337083440858),\n",
       " Feature(word='переливан', weight=-5.2237926640223664),\n",
       " Feature(word='может', weight=-5.2553955473543601),\n",
       " Feature(word='треб', weight=-5.2565406086395896),\n",
       " Feature(word='сдат', weight=-5.26324928134754),\n",
       " Feature(word='люб', weight=-5.264826004873675),\n",
       " Feature(word='положительн', weight=-5.4225741349358563),\n",
       " Feature(word='помоч', weight=-5.4344558316958924),\n",
       " Feature(word='репост', weight=-5.4521738128269224),\n",
       " Feature(word='больниц', weight=-5.4682795942698164),\n",
       " Feature(word='нуж', weight=-5.4701788567053766),\n",
       " Feature(word='человек', weight=-5.5615963851868608),\n",
       " Feature(word='спасиб', weight=-5.5898894221112174),\n",
       " Feature(word='мо', weight=-5.63451091223821),\n",
       " Feature(word='всем', weight=-5.6910383250873045),\n",
       " Feature(word='резус', weight=-5.7049514205089142),\n",
       " Feature(word='есл', weight=-5.7136776789342605),\n",
       " Feature(word='ког', weight=-5.7531380721385643),\n",
       " Feature(word='прош', weight=-5.7576396293073113),\n",
       " Feature(word='ребят', weight=-5.765031779906078),\n",
       " Feature(word='перв', weight=-5.7853362896825535),\n",
       " Feature(word='ребенк', weight=-5.7989115938826181),\n",
       " Feature(word='ест', weight=-5.8012538556482385),\n",
       " Feature(word='личк', weight=-5.8137012226138074),\n",
       " Feature(word='пиш', weight=-5.8139609111139148),\n",
       " Feature(word='завтр', weight=-5.8264297224985695),\n",
       " Feature(word='можн', weight=-5.8509371496052918),\n",
       " Feature(word='ваш', weight=-5.8642937735379306),\n",
       " Feature(word='сдава', weight=-5.9109467685915105),\n",
       " Feature(word='телефон', weight=-5.9145266715651488),\n",
       " Feature(word='станц', weight=-5.9485875633244065),\n",
       " Feature(word='центр', weight=-5.9649244235014054),\n",
       " Feature(word='операц', weight=-5.9662938156818841),\n",
       " Feature(word='сдач', weight=-5.9829846941744949),\n",
       " Feature(word='реанимац', weight=-5.9883123649684649),\n",
       " Feature(word='отделен', weight=-5.9888129201888596),\n",
       " Feature(word='люд', weight=-6.0289329031454768),\n",
       " Feature(word='звон', weight=-6.0428000610675463),\n",
       " Feature(word='мам', weight=-6.0537088300769586),\n",
       " Feature(word='вопрос', weight=-6.0656151129182811),\n",
       " Feature(word='адрес', weight=-6.0686638159349471),\n",
       " Feature(word='состоян', weight=-6.0933830125013362),\n",
       " Feature(word='тяжел', weight=-6.0944656404087665),\n",
       " Feature(word='наход', weight=-6.106458219860385),\n",
       " Feature(word='знаком', weight=-6.1069089802726131),\n",
       " Feature(word='девочк', weight=-6.1319301971870459),\n",
       " Feature(word='заран', weight=-6.1327794580532036),\n",
       " Feature(word='обраща', weight=-6.1433554638426289),\n",
       " Feature(word='равнодушн', weight=-6.1473625602453525),\n",
       " Feature(word='леж', weight=-6.1548825868669264),\n",
       " Feature(word='реб', weight=-6.2000297283802119),\n",
       " Feature(word='отзов', weight=-6.2226875729889137),\n",
       " Feature(word='жизн', weight=-6.2247222706563994),\n",
       " Feature(word='донорск', weight=-6.2314171195038028),\n",
       " Feature(word='дорог', weight=-6.2519228050190883),\n",
       " Feature(word='просьб', weight=-6.2546209318964854),\n",
       " Feature(word='номер', weight=-6.2743423137889653),\n",
       " Feature(word='друг', weight=-6.2794599834700726),\n",
       " Feature(word='втор', weight=-6.3127785455453864),\n",
       " Feature(word='паспорт', weight=-6.3335681792660719),\n",
       " Feature(word='возможн', weight=-6.3365202526442674),\n",
       " Feature(word='максимальн', weight=-6.3382259584318943),\n",
       " Feature(word='необходим', weight=-6.3607946274457836),\n",
       " Feature(word='ком', weight=-6.3801276056212872),\n",
       " Feature(word='сейчас', weight=-6.3849508793550731),\n",
       " Feature(word='наш', weight=-6.4129894133944649),\n",
       " Feature(word='тромбоцит', weight=-6.4150928360593351),\n",
       " Feature(word='напиш', weight=-6.4167649792582688),\n",
       " Feature(word='сможет', weight=-6.4355656459837451),\n",
       " Feature(word='подробн', weight=-6.4516019642516751),\n",
       " Feature(word='больш', weight=-6.4544098439401996),\n",
       " Feature(word='попа', weight=-6.4568055285581991),\n",
       " Feature(word='областн', weight=-6.4592215525912149),\n",
       " Feature(word='собак', weight=-6.4593023305560413),\n",
       " Feature(word='добр', weight=-6.4598452499464232),\n",
       " Feature(word='утр', weight=-6.461313428621045),\n",
       " Feature(word='котор', weight=-6.4737093325012856),\n",
       " Feature(word='сдела', weight=-6.4800226108255865),\n",
       " Feature(word='город', weight=-6.4987315006562145),\n",
       " Feature(word='вниман', weight=-6.5120350371527049),\n",
       " Feature(word='лейкоз', weight=-6.5142641357022466),\n",
       " Feature(word='понедельник', weight=-6.5206092829282518),\n",
       " Feature(word='трет', weight=-6.54676316980217),\n",
       " Feature(word='диагноз', weight=-6.5546342575381269),\n",
       " Feature(word='сегодн', weight=-6.5546680583950652),\n",
       " Feature(word='буд', weight=-6.5615367219918097),\n",
       " Feature(word='год', weight=-6.5783357338700315),\n",
       " Feature(word='маленьк', weight=-6.592859021656718),\n",
       " Feature(word='мальчик', weight=-6.5979295149087598),\n",
       " Feature(word='хорош', weight=-6.6042371293109152)]"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_values = list(word_dict.values())\n",
    "feature_values.sort(key=lambda x: x.weight, reverse=True)\n",
    "feature_values[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predict_proba = list(map(lambda x: x[1], model.predict_proba(X)))\n",
    "data['predict_proba'] = predict_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "y_true = data['status']\n",
    "fpr, tpr, thresholds = roc_curve(y_true, predict_proba)\n",
    "auc_score = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_roc(fpr, tpr, auc_score):\n",
    "    plt.figure()\n",
    "    plt.plot(fpr, tpr)\n",
    "    plt.xlim([0.0, 0.5])\n",
    "    plt.ylim([0.5, 1.0])\n",
    "    plt.xlabel('fpr')\n",
    "    plt.ylabel('tpr')\n",
    "    plt.title('ROC Curve. AUC = {0:.2f}'.format(auc_score))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEZCAYAAACNebLAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF4xJREFUeJzt3X2wZHV95/H3Z0ACRFAe4mqAQREFgomERGKMu0wFE3BF\nZ4v4wKTc+JBYMS6y2ZAErEqWIYmurMZlXdTU7iY+4+hqBTCrBIVcJaJCCUTEgcGoPAiaII9qdJmZ\n7/5xeqBpzr2373B/t+cM71dV1z3n9K9P//o3d86nz+/bp2+qCkmSJq2adQckSTsmA0KS1MuAkCT1\nMiAkSb0MCElSLwNCktTLgJAk9TIgNLUk30zygyT3JrktybuT7DnR5jlJLhm1uSvJBUmOmGizV5Jz\nktw0arcpyduS7LvAc5+a5Nok30tyc5IPJzmy1Wt9pJLMJbkzyWMmtv9dkldPbDs2yS0T25q/3iT7\nJPnr0XN8I8m6Bdo+Lsl7knwnybeTnDlx/3OSfHH073lNkl9azr5qNgwILUUBL6iqvYGjgJ8F3rDt\nziS/CPwt8NfAk4CnAF8GPpfkyaM2jwEuBY4AfnW0r+cA3wWO6XvSJG8HXg+cAuwDPB04H3jBUl9A\nkl2W+pjteI6DgecCW4EXTfmwB65YXc7Xu4h3Aj8EfgJ4OfCuyTAfcw6wB7Aa+AXg3yd5xai/+wAX\nAGcDjwPeAnw8yeOWub9aaVXlzdtUN+AbwC+PrZ8NfHxs/bPA/+h53CeA94yWfwu4Hdhjyuc8FNgM\n/NwCbf4OePXY+iuAy8bWtwKvAzYB/wi8C3jLxD7OB353tPwk4KPAP43av36J4/THwGXAW8fHp6+v\no23HAjePlp+22Otdpn/LPYEfAU8d2/Y+4E3ztP9n4Oix9TcAnxktvwC4dqL9DcCrZv076+2R3TyD\n0HZJciDwfODG0foedGcCH+1p/hHgV0bLxwEXVdW/TPlUxwG3VNWXltjFye+QWQs8C/gp4Dzgpdvu\nSPJ44FeBDyUJ8HHgarqgOA74j0l+hen9BvCB0fMcn+QnlvDYJb/eJO8YTefdOfZz2/I18zzs6cDm\nqvrHsW3/ACw0jbVqYvkZ27owuj2kW2P3a6AMCC3V+UnuBW4GvgOsH23fl+736faex9wO7D9a3m+e\nNvNZavv5vKmq7qmqH1XVZUAlee7ovhcDl1fVd+imufavqjdW1Zaq+ibwv4GTp3mS0T5XAx+pqquA\nrwG/voR+7ssSX29V/Yeq2qeq9h37uW35qHke9ljgnolt9wB7zdP+IuCMJI9NcijwKrqzEIDLgZ9M\n8rIku46mnp46dr8GyoDQUq2trm5wLHA4Dx7476KbynlSz2OeBNwxWv7uPG3ms9T287l1Yv3DwLai\n7K8DHxwtrwYOGH8XTjed8oQpn+c3gIur6q7R+ofopry22Qw8ZuIxjwHuHy0v1+tdzPeAvSe27Q3c\nN0/719PVK26kqzGdx2hMq+pOujO004Bv052NfYqHj7kGxoDQUgVg9C78vcCfj9Z/AHweeEnPY14K\nfHq0/Gm6aZc9pny+S4ADkxy9QJvv89B3q0/saTM55fQh4MVJthVdPzbafgvw9Yl34Y+rqhcu1tEk\nu9O91mOT3J7kduB3gWcm+elRs5uBJ0889BDgptHyNK938nnfleS+0SeIxm/3Jbl2nodtAnZN8tSx\nbc8ErutrXFV3V9XLq+pJVfXTwC7AFWP3X1ZVx1TV/nQhefj4/RqoWRdBvA3nxsOL1PvTvRP9mdH6\nL9G9Az2FbgpjH+DPgDsZFUOB3YAv0hWuD6MLnP3o3qWfMM/z/ne6ouexdO+2fwx4GfCHo/v/jO6T\nUXvQFbU3AZ8de/xW4JCe/V4HXAx8bGzbKuBK4A+B3ekOhEcCPz/F+KyjO1M6gO6MY9ttjlFRnO7d\n9beBZ43Wnw58FXjNtK93Gf89z6M7c9pz9G93F3DEPG0P4cFpxOfTFfAPH7v/KGBXurOQcxj7kIC3\n4d5m3gFvw7kBXx8PiNG2dwD/Z2z9OXSf1LkPuJuu4HvExGP2At5G9276Xrppi7cC+yzw3K8HvjIK\npFvozgCOGN23H93Ha++h+/TQf54IiC3zBMQfje47aWL7E0cHz9vppnwu3/a66aajrp2nj58E/mvP\n9pcAtwGrRuuvHL2Wu0dh9gdLeb3L+O+5D9100feAbwIvG7vvucC9E6/hW6O2VwHPm9jXeaPXc9eo\nr/vP+vfV2yO/ZfSP20SSvwROBL5TVT8zT5u3070j+T7wyqqa71MXkqQV1LoG8W7g+PnuTPJ8uqmH\npwG/DfxF4/5IkqbUNCCq6u/pTjnns5bu4hyq6ovA45L8q5Z9kiRNZ9afYjqAbn51m2+NtkmSZmzW\nATF59SU8/OOIkqQZ2HXGz38rcNDY+oF0n/Z4mCQGhyRth6rqezO+qJU4g+j7npZtLqS7qIYkzwbu\nru7rDnot9HGsc84pupOPYt264o47Zv8RsVa3M888c+Z92FFujoVj4VgsfHskmp5BJDkPWAPsl+Rm\n4Ey6C6Wqqv5nVX0iyb9N8jW6j7m+aqnPcccdcNBB8MMfwmmnwVvfupyvQJIevZoGRFUt+iVlVXXK\nI3mOU0/twuGee2DvyW+WkSRtt1kXqR+xiy6C97//0RUOa9asmXUXdhiOxYMciwc5Fsuj6ZXUyylJ\n9fX1oIPg8su7n5Kkh0pC7cBF6qY2b4ZdZ/1ZLEnaCRkQkqReBoQkqddOERC77DLrXkjSzmfwAbFl\ni2cQktTC4APCKSZJasOAkCT1GnRAbN3a/Vw16FchSTumQR9aLVBLUjuDDggL1JLUzqADwvqDJLVj\nQEiSehkQkqRegw8Ii9SS1MagA8IitSS1M+iAcIpJktoxICRJvQwISVKvwQeERWpJamPQAWGRWpLa\nGXRAOMUkSe0YEJKkXgaEJKnX4APCIrUktTHogLBILUntDDognGKSpHYMCElSLwNCktRr8AFhkVqS\n2hh0QFiklqR2Bh0QTjFJUjsGhCSplwEhSeo1+ICwSC1JbTQPiCQnJLk+yaYkp/fcvzrJp5P8Q5JL\nk/zktPu2SC1J7TQNiCSrgHOB44EjgXVJDp9o9lbgPVX1TOBPgDdPu3+nmCSpndZnEMcAN1bVTVV1\nP7ABWDvR5qeASwGqaq7n/nkZEJLUTuuAOAC4ZWz91tG2cdcAvwaQ5CTgsUn2mWbnBoQktdP68Jqe\nbTWx/gfAuUleCXwW+BawuW9n69evf2B5zZo1bN68xiK1JI2Zm5tjbm5uWfaVqsnj9fJJ8mxgfVWd\nMFo/A6iqOnue9j8ObKyq1T331WRf3/hG+MEPup+SpIdLQlX1vVlfVOsppiuBQ5McnGQ34GTgwvEG\nSfZLsq3zbwD+atqdO8UkSe00DYiq2gKcAlwMXAdsqKqNSc5KcuKo2RrghiTXA08Apj4fMCAkqZ3m\nh9equgg4bGLbmWPLHwM+tj373rwZ9tzzkfVPktTPK6klSb0GHRBeSS1J7Qw6IKxBSFI7BoQkqZcB\nIUnqNfiAsEgtSW0MOiAsUktSO4MOCKeYJKkdA0KS1MuAkCT1GnxAWKSWpDYGHRAWqSWpnUEHhFNM\nktSOASFJ6mVASJJ6DT4gLFJLUhuDDgiL1JLUzqADwikmSWrHgJAk9TIgJEm9Bh8QFqklqY1BB4RF\naklqZ9AB4RSTJLVjQEiSehkQkqRegw8Ii9SS1MagA8IitSS1M+iAcIpJktoxICRJvQwISVKvwQeE\nRWpJamOwAVEFW7caEJLUymADYsuWLhySWfdEknZOgw0I6w+S1JYBIUnq1TwgkpyQ5Pokm5Kc3nP/\nQUkuTXJVkmuSPH+a/VqglqS2mgZEklXAucDxwJHAuiSHTzT7I+DDVXU0sA545zT79ipqSWqr9RnE\nMcCNVXVTVd0PbADWTrTZCuw9Wn488K1pduwUkyS11foQewBwy9j6rXShMe4s4OIkpwJ7As+bZscG\nhCS11foMou9DqDWxvg54d1UdBLwA+MA0OzYgJKmt1ofYW4HVY+sHArdNtPlNuhoFVfWFJLsn2b+q\n7pjc2fr16x9YftrT1rDLLmuWu7+SNGhzc3PMzc0ty75SNfmGfvkk2QW4ATgOuB24AlhXVRvH2vxf\n4CNV9d4kRwCfqqoDe/ZV433dtAlOPLH7KUnql4Sq2q5LiptOMVXVFuAU4GLgOmBDVW1MclaSE0fN\nfh94TZJrgA8Cr5hm304xSVJbzQ+xVXURcNjEtjPHljcCz13qfg0ISWrLK6klSb0GHRBeSS1J7Qw2\nILySWpLaGmxAOMUkSW0ZEJKkXgaEJKnXoAPCIrUktTPYgLBILUltDTYgnGKSpLYMCElSr0EHhDUI\nSWpn0AHhGYQktTPYgLBILUltLRgQSVYlec5KdWYpPIOQpLYWDIiq2gr8+Qr1ZUkMCElqa5oppouT\n/FqS7fqLRK1YpJaktqZ5D/57wI8DW5L8CxCgqmrvpj1bhGcQktTWoofYqtprJTqyVBapJamtqQ6x\nSU6i+7OgBVxWVec37dUUPIOQpLYWrUEkeSfwWuBa4CvAa5O8o3XHFmNASFJb0xxijwWeUVUFkOS9\ndGExUxapJamtaT7FdAOwemz9IODLbbozPWsQktTWNIfY/YCNSa4YrT8L+HySCwGq6kWtOreQzZth\nzz1n8cyS9OgwTUDsATx/bD3A2cCfNunRlKxBSFJb0xxid62qz4xvSLLH5LaVZkBIUlvzHmKT/A7w\nOuCQJOM1h72Az7Xu2GIsUktSWwu9Bz8P+CTwX4AzxrbfV1V3Nu3VFCxSS1Jb8x5iq+oe4B5g3cp1\nZ3pOMUlSW4P9exAGhCS1ZUBIknoNOiAsUktSO4MNCIvUktTWYAPCKSZJasuAkCT1MiAkSb0GHRAW\nqSWpneYBkeSEJNcn2ZTk9J7735bk6iRXJbkhyVRXaVuklqS2mh5ik6wCzgWOA24DrkxyQVVdv61N\nVf3eWPtTgKOm2bdTTJLUVusziGOAG6vqpqq6H9gArF2g/TrgQ9Ps2ICQpLZaB8QBwC1j67eOtj1M\nktXAk4FLp9mxASFJbbUOiPRsq3nangx8dNvfvl6MRWpJaqv1e/Bbeejfsz6QrhbR52S6vz8xr/Xr\n1z+wfPfda9h11zWPrHeStJOZm5tjbm5uWfaVKd+wb9/Ok12AG+iK1LcDVwDrqmrjRLvDgE9W1SEL\n7OshJxfPfCa8733dT0lSvyRUVd9szqKaTjFV1RbgFOBi4DpgQ1VtTHJWkhPHmp5MV8CemjUISWqr\n6RnEcpo8gzjsMLjwwu6nJKnfDnsG0ZJFaklqa7AB4ZXUktTWYAPCGoQktWVASJJ6GRCSpF6DDgiL\n1JLUzmADwiK1JLU12IBwikmS2jIgJEm9BhkQVdYgJKm1QQbE1q2QwKpB9l6ShmGQh1gL1JLU3iAD\nwvqDJLVnQEiSehkQkqRegw0IP8EkSW0NMiAsUktSe4MMCKeYJKk9A0KS1MuAkCT1GmxAWKSWpLYG\nGRAWqSWpvUEGhFNMktSeASFJ6mVASJJ6DTYgLFJLUluDDAiL1JLU3iADwikmSWrPgJAk9TIgJEm9\nBhsQFqklqa1BBoRFaklqb5AB4RSTJLVnQEiSehkQkqRegw0Ii9SS1FbzgEhyQpLrk2xKcvo8bV6a\n5Lok1yb5wGL7tEgtSe01PcwmWQWcCxwH3AZcmeSCqrp+rM2hwOnAL1bVvUn2X2y/TjFJUnutzyCO\nAW6sqpuq6n5gA7B2os1rgHdU1b0AVXXHYjs1ICSpvdYBcQBwy9j6raNt454OHJbk75NcnuT4xXZq\nQEhSe60Ps+nZVj19OBT4N8Bq4LIkR247oxi3fv16AD73Odh33zXAmmXsqiQN39zcHHNzc8uyr1RN\nHq+XT5JnA+ur6oTR+hlAVdXZY23eBXy+qt43Wv80cHpVfWliX7Wtr29+M9x9d/dTkjS/JFRV35v1\nRbWeYroSODTJwUl2A04GLpxocz7wywCjAvXTgK8vtFOnmCSpvaYBUVVbgFOAi4HrgA1VtTHJWUlO\nHLX5W+C7Sa4DLgF+v6ruWmi/BoQktdf8MFtVFwGHTWw7c2L9NOC0afe5eTPsvvvy9E+S1M8rqSVJ\nvQYZEF5JLUntDTIgrEFIUnsGhCSplwEhSeo12ICwSC1JbQ0yICxSS1J7gwwIp5gkqT0DQpLUy4CQ\nJPUabEBYpJaktgYZEBapJam9QQaEU0yS1J4BIUnqZUBIknoNNiAsUktSW4MMCIvUktTeIAPCKSZJ\nas+AkCT1MiAkSb0GGxAWqSWprUEGhEVqSWpvkAHhFJMktWdASJJ6GRCSpF6DDQiL1JLU1iADwiK1\nJLU3yIBwikmS2jMgJEm9DAhJUq/BBoRFaklqa5ABYZFaktobXEBs3drdVg2u55I0LIM7zG47e0hm\n3RNJ2rkNLiAsUEvSymgeEElOSHJ9kk1JTu+5/xVJ/inJVaPbqxfanwVqSVoZTQMiySrgXOB44Ehg\nXZLDe5puqKqjR7e/WmifFqhhbm5u1l3YYTgWD3IsHuRYLI/WZxDHADdW1U1VdT+wAVjb027qioJT\nTP7yj3MsHuRYPMixWB6tA+IA4Jax9VtH2yadlOSaJB9JcuBCOzQgJGlltA6IvjODmli/EHhyVR0F\nXAK8d6EdWoOQpJWRqsnj9TLuPHk2sL6qThitnwFUVZ09T/tVwJ1V9fie+9p1VJJ2YlW1XRcGtJ6s\nuRI4NMnBwO3AycC68QZJnlhV3x6trgW+2rej7X2BkqTt0zQgqmpLklOAi+mms/6yqjYmOQu4sqr+\nBjg1yYuA+4E7gVe27JMkaTpNp5gkScO1w11JPcWFdbsl2ZDkxiSfT7J6Fv1cCVOMxb9O8qUk9yc5\naRZ9XClTjMV/SnLd6NNwn0py0Cz6uRKmGIvfTvLlJFcn+ew81x7tFBYbi7F2L06yNcnRK9m/lbTc\nFyUDUFU7zI0usL4GHAw8BrgGOHyize8A7xwtv4zuIruZ931GY7EaeAbwHuCkWfd5xmNxLLD7aPm1\nj/Lfi8eOLb8Q+OSs+z2rsdg2HsBngMuBo2fd7xn+XrwCePtS9rujnUFMc2HdWh78KOxHgeNWsH8r\nadGxqKqbq+orPPyjwzubacbiM1X1w9HqF+i/3mZnMM1YfG9s9bHA1hXs30qa9kLcPwXOBn60kp1b\nYct+UTLseFNM01xY90CbqtoC3J1k35Xp3oqa9iLDR4OljsVvAp9s2qPZmWoskrwuydeANwOnrlDf\nVtqiY5HkKODAqvrESnZsBpb9omTY8QJimgvrJtukp83OYJqxeLSYeiySvBz4OeAtTXs0O1ONRVW9\ns6oOBU4H/rh5r2ZjwbFIEuC/Aact8pidwbJflAw7XkDcSjevvs2BwG0TbW4BDgJIsguwd1XdtTLd\nW1HTjMWjxVRjkeR5wBuAF45Os3dGS/29+DDw75r2aHYWG4u96L4kdC7JN4BnAxfspIXqRX8vququ\nsf8X/4vujdSCdrSAeODCuiS70V1Yd+FEm4/TFVsAXgJcuoL9W0nTjMW4nfWdEUwxFkl+FvgL4EVV\n9d0Z9HGlTDMWh46tnghsWsH+raQFx6Kq7q2qJ1TVIVX1FLra1Aur6qoZ9belaX4vnji2Ou9FyQ8x\n6+p7TzX+BOAG4EbgjNG2s4ATR8s/BnxkdP8X6E6ZZt7vGY3Fz9OdUd0H/DNw7az7PMOx+BTd1fpX\nAVcD58+6zzMci3OAr4zG4hLgiFn3eVZjMdH2UnbSTzFN+XvxptHvxdWj34unL7ZPL5STJPXa0aaY\nJEk7CANCktTLgJAk9TIgJEm9DAhJUi8DQpLUy4CQliDJqUm+muT9s+6L1JrXQUhLkGQjcEJV3TRF\n21VVtbN+k6oeBTyDkKaU5F3AU4CPJ7k7yfuSXJLkhiS/NWpzbJJLk3wQ+PJMOyw9Qp5BSEuQ5Ot0\nX3HyerovwfsFui+Fu5ruO/kPA/4GOLKqbp5VP6Xl4BmEtP0uqKr/V92XA15KFxAAVxgO2hkYENL2\nGz/9Hv+7JN+fQV+kZWdASEsz/rXqa5PslmQ/ur+JfeWM+iQ1YUBISzN+1nAF8AngcuBPqurbs+mS\n1IZFamk7JDkTuK+q3jbrvkiteAYhSerlGYQkqZdnEJKkXgaEJKmXASFJ6mVASJJ6GRCSpF4GhCSp\n1/8Hm4c6rAc8CxgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f01d9119240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_roc(fpr, tpr, auc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precision, recall, pcrt = precision_recall_curve(y_true, predict_proba)\n",
    "pcrt = np.insert(pcrt, 0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f01dbd7a978>"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEPCAYAAABV6CMBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHRJJREFUeJzt3XuUHOV95vHvo8sI0AULgZyVYEi4GBAEEWwLmWDTMU4Q\nPhh5ESFoTQDb2QV7sXOyBy2wjpcBhxg5nMR4tYljHzC2IVGIlwQZzEGWx42MiUGALiBGFxtFSELg\ngLgIIYQ0+u0fVa1pNT3qqtFUz0zr+ZxTZ6reeqvqnToz9XS91VWliMDMzCyrYQPdADMzG1ocHGZm\nlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5FBockm6X9JKkFfuo8w1JayUtk3RaVfnlktZIWi3psiLb\naWZm2RV9xvEd4NzeZko6Dzg2Io4HrgS+mZaPB/438EHgDOAGSYcW3FYzM8ug0OCIiEeAV/dRZSbw\nvbTuY8Chkt5LEjYLI+L1iHgNWAjMKLKtZmaWzUBf45gMbKia3piW1ZZvSsvMzGyADXRwqM501Ckn\nLTczswE2YoC3vxE4qmr6SOCFtLxUU/7TeiuQ5EAxM+uDiKj3Ib2hZpxxiPpnEAALgMsAJE0HXouI\nl4CHgN+XdGh6ofz307K6IsJDBDfccMOAt2GwDN4X3hfeF70PnZ3793m70DMOSf9AcuYwQdLzwA1A\nGxAR8a2I+JGkj0v6JbAN+DTJzFclfQV4gqSL6sZILpKbmdl+2rx5/5YvNDgi4r9kqHN1L+V3Anf2\nc5PMzFpaBGzfDq+/Dm+8AW+9lYyvXw+/+hWsWAGdnfu3jYG+xmH9qFQqDXQTBg3vix7eFz2Gyr6I\nSA7wL74Ib76ZBEC9oRIOtUNbG4wblwyHHAKjRsHxx8Oxx8KsWfClL8G0aX1vnyKG9rVlSTHUfwcz\nK8bu3bB1K2zZknwK374d3n4b3nkHurth165kqDfeqOydd2Dnzp6ht+ldu3pvXwQ89xy89FLP9Dvv\nJO2MSA7ukyf3hMC4cXDooXtP184bOxZGjmy8byQRfbw47jMOsxa1e3dyoKsMlQNfZdi5E3bsSA6k\n1T+zjNcr6+4u9veJ6Gnzjh3JAbbReHd3ciAdPz755H3wwXDQQckn8hEjkmH48L1/9jZeW9bWBqNH\nJwfpkSOT6cp49fSIEaB9HJ7f+1445pieOm1tSTuHDy92f+4PB4cNGRHJUDkg7t797qFeeaOy7dvh\n179ODqzVB9t643nn79qVdB1s3fruT677Oqj3Rxn0HOSqD36VYeTI5CA6alQyVI/XTteOH3rou8tH\nNOFo0taWDJVtVsbrlVXatK+DtvWNu6paTHd3cjFs27bk59tv1z+45J1+553k4Pfmm/W3+9Zb8Mor\ne5dVDpy1Q+X0vbfyHTvg5ZeTn9UH5IjkIDBsWM8wfPje072V7avuqFEwcWJyIK3UqRxcq5erLcsy\nf/jwpAthzJieT5/V8+od0PtSp94ywwb69l4b1Panq8rBkcPu3ckBtHp4+23YsOHd/Zu1Q+XAuK9h\n06ae0+t6n4xrP9nu3p3UrwTFtm3J9CGHJKfQhxySfCLsjwNT5WLb6NH1P8EddBAcfvje86Seg2Xt\n0Fv5iBHJtg4/PDldrz3Q+9OjWf9wcPTD7xCRfJp+4glYuDD5utq6dXuHRHd3z6lwpQ+zrQ2OOio5\nyFX6NisHxt6G3uYfcURycK79dFxvvPJz5MjkYF4ZDj7YB1cza8zBkfF32LYNnn02CYSXX4YXXoD7\n74fnn0/6oUeNSkLg4ovhnHPgxBN7gqKtLTlY+6BsZq3AwdHL7/DKK3DbbbB0KTzzTPKVtxNOSL7L\nfMQRSXfIWWfBBz+YfNJvxsU9M7PBwF/HrbJrFzz4IHzta/DYY3DBBfDpT8MppySBMZi/4mZmNhS0\nTHB0dyfdTpddlnQxXXUVLFqUdDWZmVn/aYmuqnvvDf7kT+Cww+DKK+Gaawa6VWZmg9sBf41j0qTg\nvvvgAx8Y6NaYmQ0N+xMcLXGL0G23OTTMzJqlJYLjzDMHugVmZgeOlgiO8eMHugVmZgeOlggO35Rn\nZtY8LREcfpibmVnztMQh18FhZtY8hR9yJc2QtErSGknX1pnfLmmRpOWSOiVNqpo3V9LTklZIuri3\nbTg4zMyap9BDrqRhwDzgXOBkYLakE2uq3QrcGRFTgZuAW9JlPw6cBpwKTAfmSBpTfzvFtN/MzN6t\n6M/q04C1EbE+InYC84GZNXWmAJ0AEVGumj8FeDgSbwHLgRn1NuLgMDNrnqKDYzKwoWp6Y1pWbRkw\nC0DShcAYSeNJguI8SQdLOhz4PeCogttrZmYNFB0c9c4Fap9xMgcoSXoS+DCwCdgVET8GHgQeBe5O\nf+4qsK1mZpZB0U/H3Qi0V00fCbxQXSEiNtNzxjEamBURW9N5fwn8ZTrvbmBtvY10dHTsGS+VSpRK\npf5qv5lZSyiXy5TL5X5ZV6EPOZQ0HFgNnANsBh4HZkdEV1WdCcCWiAhJf0FyttGRXlh/T0RskXQq\ncBdwWkTsrtlG0945bmbWKgbti5wiolvS1cBCkm6x2yOiS9KNwJKIuB8oAV+VtBtYDPz3dPGRwM8k\nBfAG8Kna0DAzs+ZriceqD/Xfwcys2Q74x6qbmVnzODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zM\ncnFwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPL\nxcFhZma5ODjMzCwXB4eZmeVSeHBImiFplaQ1kq6tM79d0iJJyyV1SppUNW+upGckrZT09aLbamZm\njRUaHJKGAfOAc4GTgdmSTqypditwZ0RMBW4CbkmX/RBwZkScApwCTJP0kSLba2ZmjRV9xjENWBsR\n6yNiJzAfmFlTZwrQCRAR5ar5ARwk6SDgYGAE8FLB7TUzswaKDo7JwIaq6Y1pWbVlwCwASRcCYySN\nj4hfAGVgM7AJeCgiVhfcXjMza2BEwetXnbKomZ4DzJN0BbCYJCR2SToWOBGYlK5nkaSHIuKR2hV2\ndHTsGS+VSpRKpf5ou5lZyyiXy5TL5X5ZlyJqj+P9R9J0oCMiZqTT1wEREXN7qT8a6IqIdknXAKMi\n4uZ03peB7RFxa80yUeTvYGbWiiQREfU+3DdUdFfVEuA4SUdLagMuARZUV5A0QVKl8dcDd6TjzwNn\nSxouaSRwNtBVcHvNzKyBQoMjIrqBq4GFwEpgfkR0SbpR0vlptRKwWtIqYCJwc1r+A+A54GlgKbA0\nIh4osr1mZtZYoV1VzeCuKjOz/AZzV5WZmbUYB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4O\nMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjM\nzCwXB4eZmeXi4DAzs1wKDw5JMyStkrRG0rV15rdLWiRpuaROSZPS8pKkpZKeSn9ul3RB0e01M7N9\nU0QUt3JpGLAGOAd4AVgCXBIRq6rq3AMsiIi7JJWAz0TEZTXrGQ+sBY6MiLdr5kWRv4OZWSuSRESo\nL8sWfcYxDVgbEesjYicwH5hZU2cK0AkQEeU68wEuAh6sDQ0zM2u+ooNjMrChanpjWlZtGTALQNKF\nwJj0DKPaJcA/FtVIMzPLbkTB6693GlTbrzQHmCfpCmAxsAnYtWcF0m8ApwAP9baRjo6OPeOlUolS\nqdTX9pqZtaRyuUy5XO6XdRV9jWM60BERM9Lp64CIiLm91B8NdEVEe1XZF4EpEXFVL8v4GoeZWU6D\n+RrHEuA4SUdLaiPpclpQXUHSBEmVxl8P3FGzjtm4m8rMbNAoNDgiohu4GlgIrATmR0SXpBslnZ9W\nKwGrJa0CJgI3V5aXdDTJN6keLrKdZmaWXaFdVc3griozs/wGc1eVmZm1GAeHmZnl4uAwM7NcMt3H\nIWkUyU16v1m9TETcVEyzzMxssMp6A+B9wOvAk8CO4ppjZmaDXdbgOLJyE5+ZmR3Ysl7jeFTSbxfa\nEjMzGxIy3cch6VngOGAdSVeVSB4dcmqxzWvM93GYmeW3P/dxZO2qOq8vKzczs9aT+c5xSVOBD6eT\nP4uI5YW1KgefcZiZ5Vf4neOS/hS4m+RZUhOBuyR9oS8bNDOzoS3rNY4VwIciYls6PRr4N1/jMDMb\nmprxrCoB3VXT3dR/SZOZmbW4rBfHvwM8Julf0ulPArcX0yQzMxvM8lwcPx04i+RMY3FELC2yYVm5\nq8rMLL/96araZ3BIGhcRb0g6rN78iNjSl432JweHmVl+RQbH/RFxvqR1QHXFyg2Ax/Rlo/3JwWFm\nll9hwTEUODjMzPJrxn0cv5t+BRdJl0r6a0ntfdmgmZkNbVm/jvt3wFvp3eP/E1gPfD/LgpJmSFol\naY2ka+vMb5e0SNJySZ2SJlXNO0rSQ5KelfSMw8rMbOBlDY5daX/QTOC2iLgNGNtoIUnDgHnAucDJ\nwGxJJ9ZUuxW4MyKmAjcBt1TN+x4wNyKmANOAX2dsr5mZFSRrcGyVdD1wKfCApOHAyAzLTQPWRsT6\niNgJzCcJn2pTgE6AiChX5ks6CRgeEZV5b0XE2xnba2ZmBckaHH9E8jj1z0bEi8Bk4K8yLDcZ2FA1\nvTEtq7aM5LW0SLoQGCNpPPA+4HVJ/0/Sk5LmSvLd6mZmAyzTneNpWPx11fTzJN1IjdQ70Nd+BWoO\nME/SFcBiYBOwK23bWcBpJOFzD3AFyV3se+no6NgzXiqVKJVKGZpmZnbgKJfLlMvlfllXo/s4HomI\nsyRtpf59HOP2uXJpOtBRee2spOvS5eb2Un800BUR7ZLOAL4aER9N510KnBERX6hZxl/HNTPLqbAX\nOUXEWenPhhfCe7EEOE7S0cBm4BJgdnUFSROALenR/3rgjqplx0uaEBGvAB9Ny8zMbABlvY9juqSx\nVdNj0jOCfYqIbuBqYCGwEpgfEV2SbpR0flqtBKyWtIrkXR83p8vuBq4BOiVVXhr17Wy/lpmZFSXr\n+ziWAqdX+oTSr9k+ERGnF9y+htxVZWaWX1Pex1F9dE7PBrI+kt3MzFpI1uB4TtIXJY1Mhz8Fniuy\nYWZmNjhlDY6rgDNJviq7ETgD+G9FNcrMzAYvPx3XzOwA1Iyn475P0k8kPZNOnyrpz/uyQTMzG9qy\ndlV9m+Qei50AEbGC5J4MMzM7wGQNjkMi4vGasl393RgzMxv8sgbHy5KOJX3siKSLSO4ENzOzA0zW\nGwCPAb5F8s2qV4F1wKciYn2xzWvMF8fNzPIr7FlV6cqHAR+IiI+lDyEcFhFb+7IxMzMb+hp2VaV3\niV+djm9zaJiZHdiydlV9GdgO/BOwrVIeEVuKa1o27qoyM8tvf7qqsgbHOt79AiYi4pi+bLQ/OTjM\nzPJrRnAcDHye5I18AfwM+GZEbO/LRvuTg8PMLL9mBMc9wBvA3WnRbOA9EXFxXzbanxwcZmb5Ffqt\nqtQJETG1avqnVS9XMjOzA0jWGwCXpu8PByB9+9/Pi2mSmZkNZlm7qrqAE4Dn06J2oAvYDUREnFpY\nCxu3zV1VZmY5NaOrakZfVm5mZq2n8PdxSJoBfJ2kW+z2iJhbM78duAM4AngFuDQiXkjndQPLAQHr\nI+KTddbvMw4zs5wK/1ZVX6WPK1kDnAO8ACwBLomIVVV17gEWRMRdkkrAZyLisnTeGxExrsE2HBxm\nZjkV/iKn/TANWBsR6yNiJzAfmFlTZwrQCRAR5Zr5ffqlzMysOEUHx2RgQ9X0xrSs2jJgFoCkC4Ex\nksan80ZJelzSo5JqA8fMzAZA1ovjfVXvjKG2X2kOME/SFcBiYBM9L4lqj4gXJf0W0ClpRUSsq11h\nR0fHnvFSqUSpVNr/lpuZtZByuUy5XO6XdRV9jWM60BERM9Lp60i+vju3l/qjga6IaK8z7zvADyPi\n3ppyX+MwM8tpMF/jWAIcJ+loSW0k7ylfUF1B0gRJlcZfT/INKyS9J10GSYeTvETq2YLba2ZmDRQa\nHBHRTfIuj4XASmB+RHRJulHS+Wm1ErBa0ipgInBzWn4S8ISkpcBPgK9WfxvLzMwGRuH3cRTNXVVm\nZvkN5q4qMzNrMQ4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4O\nMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5FB4c\nkmZIWiVpjaRr68xvl7RI0nJJnZIm1cwfK2mjpG8U3VYzM2us0OCQNAyYB5wLnAzMlnRiTbVbgTsj\nYipwE3BLzfyvAOUi22lmZtkVfcYxDVgbEesjYicwH5hZU2cK0AkQEeXq+ZLeD0wEFhbcTjMzy6jo\n4JgMbKia3piWVVsGzAKQdCEwRtJ4SSI5G5kDqOB2mplZRiMKXn+9A37UTM8B5km6AlgMbAJ2AZ8H\nHoiITUmG9B4eHR0de8ZLpRKlUml/2mxm1nLK5TLlcrlf1qWI2uN4/5E0HeiIiBnp9HVARMTcXuqP\nBroiol3SXcBZwG5gLDAS+NuI+F81y0SRv4OZWSuSRET0qTen6OAYDqwGzgE2A48DsyOiq6rOBGBL\nRISkvwB2RURHzXouB94fEV+ssw0Hh5lZTvsTHIVe44iIbuBqkovbK4H5EdEl6UZJ56fVSsBqSatI\nLoTfXGSbzMxs/xR6xtEMPuMwM8tv0J5xmJlZ63FwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi\n4DAzs1wcHGZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1wcHGZmlouD\nw8zMcnFwmJlZLg4OMzPLpfDgkDRD0ipJayRdW2d+u6RFkpZL6pQ0qar8CUlPSXpa0pVFt9XMzBpT\nRBS3cmkYsAY4B3gBWAJcEhGrqurcAyyIiLsklYDPRMRlkkYCRMROSYcAK4EPRcSLNduIIn8HM7NW\nJImIUF+WLfqMYxqwNiLWR8ROYD4ws6bOFKATICLKlfkRsTNdBuBgoE+/oJmZ9a+ig2MysKFqemNa\nVm0ZMAtA0oXAGEnj0+kjJS0H1gNza882zMys+UYUvP56Zwm1/UpzgHmSrgAWA5uAXQARsRGYKuk3\ngPsk/SAi/qN2hR0dHXvGS6USpVKpP9puZtYyyuUy5XK5X9ZV9DWO6UBHRMxIp68DIiLm9lJ/NNAV\nEe115t0B3B8R99aU+xqHmVlOg/kaxxLgOElHS2oDLgEWVFeQNEFSpfHXA3ek5ZMlHZSOjwd+F1hd\ncHvNzKyBQoMjIrqBq4GFJN+Kmh8RXZJulHR+Wq0ErJa0CpgI3JyWnwQ8Jmkp8FPgaxGxssj2mplZ\nY4V2VTWDu6rMzPIbzF1VZmbWYhwcZmaWi4PDzMxycXCYmVkuDg4zM8vFwWFmZrk4OMzMLBcHh5mZ\n5eLgMDOzXBwcZmaWi4PDzMxycXCYmVkuDg4zM8vFwWFmZrk4OMzMLBcHh5mZ5eLgMDOzXBwcZmaW\nS+HBIWmGpFWS1ki6ts78dkmLJC2X1ClpUlo+VdKjkp6WtEzSxUW31czMGis0OCQNA+YB5wInA7Ml\nnVhT7VbgzoiYCtwE3JKWvwX8cUT8NnAe8HVJ44ps71BXLpcHugmDhvdFD++LHt4X/aPoM45pwNqI\nWB8RO4H5wMyaOlOAToCIKFfmR8TaiPhVOr4Z+DVwRMHtHdL8T9HD+6KH90UP74v+UXRwTAY2VE1v\nTMuqLQNmAUi6EBgjaXx1BUnTgJGVIDEzs4FTdHCoTlnUTM8BSpKeBD4MbAJ27VmB9J+A7wFXFNRG\nMzPLQRG1x/F+XLk0HeiIiBnp9HVARMTcXuqPBroioj2dHguUgZsj4t5elinuFzAza2ERUe/DfUMj\n+rshNZYAx0k6GtgMXALMrq4gaQKwJZIEux64Iy0fCfwr8N3eQgP6/oubmVnfFNpVFRHdwNXAQmAl\nMD8iuiTdKOn8tFoJWC1pFTARuDktvxg4C7hC0lJJT0k6tcj2mplZY4V2VZmZWesZMneOZ7iRsE3S\nfElrJf2bpPaBaGczZNgXfyZpZXrj5I8lHTUQ7WyGRvuiqt5FknZLOr2Z7WumLPtC0sXp38bTku5q\ndhubJcP/yFHpDcdPpf8n5w1EO4sm6XZJL0lasY8630iPm8sknZZpxREx6AeSgPslcDQwkuQrvCfW\n1Pkc8Lfp+B+RdIsNeNsHaF+cDRyUjl91IO+LtN4Y4GHgUeD0gW73AP5dHAc8CYxLpw8f6HYP4L74\ne+DKdPwkYN1At7ugfXEWcBqwopf55wEPpONnAL/Ist6hcsaR5UbCmcB30/EfAOc0sX3N1HBfRMTD\nEfF2OvkL3n3vTKvI8ncB8BVgLrCjmY1rsiz74r8C/zci3gCIiJeb3MZmybIvdgOVJ1G8h+Q2gJYT\nEY8Ar+6jykyS2x2IiMeAQyW9t9F6h0pwZLmRcE+dSC7KvybpsOY0r6my7ItqnwUeLLRFA6fhvkhP\nvY+MiB81s2EDIMvfxfuAEyQ9kj4H7tymta65suyLG4E/lrQBuB/4QpPaNtjU7qtNZPigWfTXcftL\nlhsJa+uoTp1WkGVfJBWlS4H3k3RdtaJ97gtJAv4GuLzBMq0gy9/FCJLuqo8A7cDPJJ1cOQNpIVn2\nxWzgOxHxN+n9ZneRPE/vQJP5eFJtqJxxbCT5Q684Enihps4G4CgAScNJ+nH3dYo2VGXZF0j6GMl9\nMZ9IT9dbUaN9MZbkYFCWtA6YDtzXohfIs/xdbATui4jdEfHvwGrg+OY0r6my7IvPAvcARMQvgIMk\nHd6c5g0qG0mPm6m6x5NaQyU49txIKKmN5EbCBTV1fkjPJ8s/JH1wYgtquC8k/Q7wTeCCiHhlANrY\nLPvcFxHxRkRMjIhjIuK3SK73fCIinhqg9hYpy//IvwIfBUgPkscDzzW1lc2RZV+sBz4GIOkkYFQL\nX/MRvZ9pLwAugz1P+ngtIl5qtMIh0VUVEd2SKjcSDgNuj/RGQmBJRNwP3A58X9Ja4BWSP5aWk3Ff\nfA0YDfxz2l2zPiI+OXCtLkbGfbHXIrRoV1WWfRERD0n6A0krSZ4Hd00rnpVn/Lu4Bvi2pD8juVB+\nee9rHLok/QPJTdYTJD0P3AC0kTz66VsR8SNJH5f0S2Ab8OlM602/hmVmZpbJUOmqMjOzQcLBYWZm\nuTg4zMwsFweHmZnl4uAwM7NcHBxmZpaLg8MOaJIOlfS5dPxsST8sYBuXS/o/OZdZV+9Za5JukPQ/\n+q91Zvk5OOxANx74fDqe6flmkvryf5P3hinfYGWDloPDDnRfBY6R9BTJo9fHSvpnSV2Svl+plJ4B\nfFnSYuAiScdIelDSEkkPS3pfWu8P05ckLZVUrtrO5LT+aklzq9Y7W9KKdLilqr6q6nwpfSnRQuCE\nYnaDWXZD4pEjZgW6Djg5Ik6XdDbJ85ymAC8CP5d0ZkQ8mtbdHhEfAZC0iORFQL+SNA34O5J3wHwZ\n+IOI2CxpXNV2ppK8UGcnsFrSN0gedXEL8DvAa8CPJV0QEXueq5Q+kPHidPk24CngiUL2hFlGDg6z\nvT0eEZsBJC0DfpPkzYEA/5SWjwbOpOdZYJC8aQ7g58B3Jd0D3Fu13p9ExJvp8itJ3k53OPDTiNiS\nlt9N8sjz6gfyfRj4l4jYAeyQVPuwPrOmc3CY7a36LYHd7P0/si39OQx4NSLe9Xj2iPicpA8C5wPL\nJE2ts97d6Xr39dTSvVabse1mTeFrHHag20ry3g7I+OTciNgKrJN0UaVM0qnpz2MiYklE3AD8B3u/\n66DWY8BHJB2WvkNmNlCuqbMY+M+SRkkaC3wiSxvNiuQzDjugRcQWST+XtALYDlS/iyB6GQf4FPBN\nSX9O8n80H1gB/JWkysuRFkXEivT9KHttNt32i5KupycsHqh6FHylztK022s58O8kQWI2oPxYdTMz\ny8VdVWZmlouDw8zMcnFwmJlZLg4OMzPLxcFhZma5ODjMzCwXB4eZmeXi4DAzs1z+P6rvGsSMLRHo\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f01dbd6a048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(pcrt, precision)\n",
    "plt.xlabel('threshold')\n",
    "plt.ylabel('precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9, 1.0)"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEPCAYAAACDTflkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFv1JREFUeJzt3X+wJWWd3/H3Z4ZBEBCHUVF+DCro8kNB2RJHo+4Fa5fR\nEEjEGCe7UaxKhXJDYmorFLjJhsFoCbtUxXXZirUpgbhhM9HsrqsSF3YcDq4aBeX3ODOgAWRwFnX5\nIQoMM/DNH92XORzuzNzb9/b9MfN+VXXd7qef0+fpp849n/t03+5OVSFJUheL5roBkqSFyxCRJHVm\niEiSOjNEJEmdGSKSpM4MEUlSZ72GSJLPJnkwye27qPPpJHcnuTXJG4bKP5jkriSbknygz3ZKkrrp\neyRyJXD6zlYmeRdwdFW9BjgX+ExbvhT4T8CbgDcDFyU5uOe2SpKmqNcQqapvAA/vospZwOfaut8B\nDk5yKE3wXFdVj1bVI8B1wMo+2ypJmrq5PidyOHD/0PLmtmy0/IG2TJI0j8x1iGSC5ZqgnLZckjSP\n7DPH778ZOHJo+Qjgx2352Ej59RNtIInhIkkdVNVEf7BPyWyMRMLEIwuALwEfAEiyAnikqh4ErgV+\nPcnB7Un2X2/LJlRVTlVcdNFFc96G+TLZF/aFfbHraab0OhJJ8mc0I4plSX4EXATsC1RV/UlV/Z8k\n707yA+CXwIdoVj6c5D8D36U5jHVxNSfYJ3T11bBoESTNND4/U2WLF8M++zTTkiU75nc2ZRfZPtF7\njP+UpIWm1xCpqn8+iTrn7aT8KuCqybzPNddAFTzzTPNzeH4myp5+upm2bYPt23c+ja/f9f422x19\nD9gRKMPhMlHg7Gz+scfgiismDsLZMhy0S5ZMPO0uaGfC+vXNpIXVF4sXw9KlcMghsP/+O8qHP++j\nn//dlQ+X3XEHrFkzcd2XvARe+cqd/7G4aK7PIM9TmclhzVxIUgt9H+D54bWz+V2t/+Y3B6xYMfa8\n9bO5D8OBurNpd0E7E9avH3DCCWP9v9ECsJD6Yvt2ePhh+Pu/h61bd5QPf9Yn+vzvrny8bMuWAS99\n6fN/R555Bn74w+a9R/9YHF9etGhHoOy7LxxxRDMdcshzp4MO2vkfT8PLL3oRvPSlTWjORUAloWbg\nnIghIkm7MR4246GydSv86EewZQs89NCO6eGH4dFHJ/5DarTs5z+Hn/wEfvELWLasCZTxab/9mqAa\nn5Ysee7yMcfAwQfDySc3wdVlZG+ItAwRSQvZtm3ws5/BT3+6Y9q6FZ56qpm2bdsx/9RT8OSTcNdd\n8OCDcOedTdlxx8G73w1jY3DssXDoofCCF+z6fQ2RliEiaW/25JNw442wdi3ccENzWO7BB5vzuNCc\nW3r5y+HEE+Gyy+Doo5tyQ6RliEjSc43/kxDA4483h93+8A/hj/8Y/uqv4MwzDZFnGSKSNDlXXgnn\nnQe33w7HHDMzITLXV6xLkmbJhz4E99zTHNaaKY5EJGkvsnkzvP718MgjMzMS8fIZSdqLHHFEcy3L\nTDFEJGkv86pXzdy2DBFJ2ssceujMbcsQkaS9zCGHzNy2DBFJ2ssYIpKkzgwRSVJny5bN3LYMEUna\ny+y338xtyxCRpL3MTD6/xBCRpL3MTD5Z1BCRpL2MIxFJUmeGiCSpMw9nSZI6cyQiSerMkYgkqTNH\nIpKkzgwRSVJnHs6SJHXmSESS1JkjEUlSZ45EJEmdGSKSpM48nCVJmhcMEUlSZ4aIJKkzQ0SS1Jkh\nIknqrPcQSbIyycYkdyW5YIL1y5OsTXJbknVJDhtad2mSO5LcnuR9fbdVkjQ1vYZIkkXA5cDpwAnA\nqiTHjlS7DLiqqk4CPgZc0r723cAbgBOBFcD5SQ7ss72SpKnpeyRyCnB3Vd1XVduANcBZI3WOB9YB\nVNVgaP3xwA3VeBy4DVjZc3slSVPQd4gcDtw/tLy5LRt2K3A2QJL3AAcmWUoTGu9Ksn+SlwCnAkf2\n3F5J0hTs0/P2J7ouskaWzwcuT3IO8HXgAWB7Vf1NkjcB3wJ+0v7cPtGbrF69+tn5sbExxsbGpttu\nSdqjDAYDBoMBAPfcM3PbTdXod/oMbjxZAayuqpXt8oVAVdWlO6l/ALChqpZPsO5q4E+r6q9HyqvP\nfZCkPc3118Npp4WqmvYNUPo+nHUTcEySo5LsC7wf+NJwhSTLkmfv5PJR4Iq2fFGSQ9r5E4HXA9f1\n3F5J2uPN5L2zej2cVVVPJzmP5st/EfDZqtqQ5GLgpqr6CjAGfDLJMzSHs/51+/IlwN8mKeDnwG9W\n1TN9tleSNDW9Hs6aDR7OkqSpGQzg1FMXxuEsSdIezBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aI\nJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVm\niEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlS\nZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOus9RJKsTLIxyV1JLphg/fIka5PclmRdksOG1l2a5M4k\n65N8qu+2SpKmptcQSbIIuBw4HTgBWJXk2JFqlwFXVdVJwMeAS9rXvgV4a1W9DngdcEqSd/TZXknS\n1PQ9EjkFuLuq7quqbcAa4KyROscD6wCqajC0voD9kuwH7A/sAzzYc3slSVPQd4gcDtw/tLy5LRt2\nK3A2QJL3AAcmWVpV3wYGwBbgAeDaqtrUc3slSVPQd4hkgrIaWT4fGEvyPeDtNIGxPcnRwLHAYTTB\n884kb+uzsZKkqdmn5+1vBpYPLR8B/Hi4QlVtYcdI5ADg7Kp6LMm5wLer6ol23VeBFcA3Rt9k9erV\nz86PjY0xNjY2ozshSQvdYDBgMBgAcO+9M7fdVI0ODGZw48liYBPwTprDUjcCq6pqw1CdZcBDVVVJ\nPg5sr6rVSd4H/EvgXTQjpq8C/6Wqrhl5j+pzHyRpTzMYwKmnhqqa6GjRlPR6OKuqngbOA64D1gNr\nqmpDkouTnNFWGwM2JdkIvAz4RFv+v4H/B9wB3ALcMhogkqS51etIZDY4EpGkqVkwIxFJ0p7NEJEk\ndWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLU2S7vnZXkyzz/honPqqozZ7xFkqQFY3c3YLxs\nVlohSVqQdhkiVXXDbDVEkrTw7O5w1h3s+nDWiTPeIknSgrG7w1ln7Ga9JGkvtrvDWffNVkMkSQvP\npP7FN8mKJDcl+UWSp5I8neTnfTdOkjS/TfY6kcuBVcDdwP40Txz8o74aJUlaGCb9jPWq+kGSxe3T\nCq9M8q0e2yVJWgAmGyKPJ9kXuDXJ79M8L/2A/polSVoIJns461+0dc8DfgkcCZzdV6MkSQvDZEci\nPwOeqqongYuTLAZe0F+zJEkLwWRHIl8DXji0vD+wduabI0laSCYbIvtV1S/GF9r5F+6iviRpLzDZ\nEPllkpPHF5L8KvBEP02SJC0Ukz0n8u+ALyT5cbv8CuCf9dMkSdJCMakQqaqbkhwL/AoQYGNVbeu1\nZZKkeW+ytz15IXAB8JGqugN4ZRJvzihJe7nJnhO5EngKeEu7vBn4eC8tkiQtGJMNkaOr6veBbQBV\n9QTNYS1J0l5ssiHyVJL9aR9QleRoYGtvrZIkLQi7PbGeJMBngL8GjkxyNfAPgHP6bZokab7bbYhU\nVSX5CPAbwAqaw1gfqaqf9d04SdL8NtnrRL4NvLqqrumzMZKkhWWyIXIqcG6S+2ju4huaQcqJvbVM\nkjTvTTZE3tVrKyRJC9Jkr1i/r++GSJIWnsn+i68kSc/Te4gkWZlkY5K7klwwwfrlSdYmuS3JuiSH\nteVjSW5JcnP784kkZ/bdXknS5PUaIkkWAZcDpwMnAKvaGzkOuwy4qqpOAj4GXAJQVYOqemNVnQyc\nRnNC/7o+2ytJmpq+RyKnAHdX1X3tXX/XAGeN1DkeWAdNcEywHuC9wFfbx/NKkuaJvkPkcOD+oeXN\nbdmwW4GzAZK8BzgwydKROu8H/mdfjZQkddN3iEx0k8YaWT4fGEvyPeDtwAPA9mc3kLwceB1wbV+N\nlCR1M9nrRLraDCwfWj4C+PFwharawo6RyAHA2VX12FCV9wF/WVVP7+xNVq9e/ez82NgYY2Nj0223\nJO1RBoMBg8EAgHvvnbntpmp0YDCDG08WA5uAdwJbgBuBVVW1YajOMuCh9h5dHwe2V9XqofX/F7iw\nqm7YyXtUn/sgSXuawQBOPTVU1bQf6dHr4ax29HAezX9VrQfWVNWGJBcPPRlxDNiUZCPwMuAT469P\nchRwxM4CRJI0t3odicwGRyKSNDULZiQiSdqzGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVm\niEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlS\nZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCR\nJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKmz3kMkycokG5PcleSCCdYvT7I2yW1J1iU5bGjdkUmu\nTfL9JHcmWd53eyVJk9driCRZBFwOnA6cAKxKcuxItcuAq6rqJOBjwCVD6z4HXFpVxwOnAD/ps72S\npKnpeyRyCnB3Vd1XVduANcBZI3WOB9YBVNVgfH2S44DFVTW+7vGqerLn9kqSpqDvEDkcuH9oeXNb\nNuxW4GyAJO8BDkyyFHgt8GiSP0/yvSSXJknP7ZUkTcE+PW9/oi/9Glk+H7g8yTnA14EHgO00bXsb\n8AaaIPo8cA5w5egGV69e/ez82NgYY2Nj0223JO1RBoMBg8EAgHvvnbntpmr0O30GN56sAFZX1cp2\n+UKgqurSndQ/ANhQVcuTvBn4ZFWd1q77LeDNVfVvRl5Tfe6DJO1pBgM49dRQVdM+utP34aybgGOS\nHJVkX+D9wJeGKyRZNnSY6qPAFUOvXZpkWbt8GvD9ntsrSZqCXkOkqp4GzgOuA9YDa6pqQ5KLk5zR\nVhsDNiXZCLwM+ET72meAfw+sS3JbW/e/9dleSdLU9Ho4azZ4OEuSpmYhHc6SJO3BDBFJUmeGiCSp\nM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohI\nkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeG\niCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSps95DJMnKJBuT3JXkggnW\nL0+yNsltSdYlOWxo3dNJbk5yS5Iv9t1WSdLU9BoiSRYBlwOnAycAq5IcO1LtMuCqqjoJ+BhwydC6\nX1bVyVX1xqr6x322dU8wGAzmugnzhn2xg32xg30x8/oeiZwC3F1V91XVNmANcNZIneOBdQBVNRhZ\nn57bt0fxF2QH+2IH+2IH+2Lm9R0ihwP3Dy1vbsuG3QqcDZDkPcCBSZa2616Q5MYk30oyGj6SpDnW\nd4hMNJKokeXzgbEk3wPeDjwAbG/XLa+qU4DfBD6V5FW9tVSSNGWpGv1On8GNJyuA1VW1sl2+EKiq\nunQn9Q8ANlTV8gnWXQl8uar+YqS8vx2QpD1YVU37lME+M9GQXbgJOCbJUcAW4P3AquEKSZYBD1WT\nZh8FrmjLXww8XlVPJXkJ8FbgeeEzE50gSeqm18NZVfU0cB5wHbAeWFNVG5JcnOSMttoYsCnJRuBl\nwCfa8uOA7ya5Bfga8Mmq2thneyVJU9Pr4SxJ0p5tXl+xPs0LFT/Yvm5Tkg/MbstnXte+SHJS+99t\ndyS5Ncn7Zr/1M2s6n4t2/UFJNif59Oy1uh/T/B05Msm1Sb6f5M4kzzsXuZBMsy8ubftgfZJPzW7L\nZ1aSzyZ5MMntu6jz6SR3t98Jbxgqn/r3ZlXNy4km4H4AHAUsoflX4GNH6nwe+K12fgz4XDu/FPgh\ncDDw4vH5ud6nOeqL1wBHt/OvAH4MvGiu92ku+mJo/aeA/wF8eq73Zy77ArgeOK2dfyGw31zv01z0\nBfAW4G/b+QDfAt4x1/s0jb54G/AG4PadrH8XcE07/2bg2+18p+/N+TwSmc6FiqcD11XVo1X1CM05\nmZWz0up+dO6Lqrq7qn7Yzm8BfgK8dJba3YdpXcCa5Fdpzr1dNyut7VfnvkhyHLC4qsbXPV5VT85W\nw3swnc9FAfsl2Q/Yn+Yfjh6cjUb3oaq+ATy8iypnAZ9r634HODjJoXT83pzPITKdCxVHX/vABK9d\nSKZ70SZt+SnAkvFQWaA690WS0Nxm53z2jLshTOdz8Vrg0SR/nuR77eGchdwnnfuiqr4NDGj+g/QB\n4Nqq2tR7i+fOzvqq0/fmfA6R6VyoOJnXLiTTvWiTJK+g+evjnJ7aOFum0xe/TTOMf2AX21pIptMX\n+9Ac9vgd4E3A0Szsz0bnvkhyNHAscBjNl+Y7k7ytz8bOsdG+Ck1fdfre7Ps6kenYDAyf6DuC5nj+\ns9rDM+N/WRwAnF1VjyXZTHPMc/i11/fa2n517ot2+SDgK8DvVtVNs9Li/kznc/EW4G1Jfhs4CFiS\n5LGq+t3ZafqMm+7vyC1VdV+77os0x8evnI2G92A6fXEuzXmBJ9p1XwVWAN+YjYbPgc3AkUPL433V\n7Xtzrk8C7eLk0GJ2nCjbl2YoetxInWXs+Dflj9NcHQ/PPUE0Pv/iud6nOeqLJTTX2fzbud6Pue6L\nkTofZOGfWJ/O52IRcAuwrF2+AvjwXO/THPXF+2iO/y9uf1/WAv9wrvdpmv3xSuCOnax7NztOrK9g\n4hPrk/7enPOd3U1HrAQ2AXcDF7ZlFwNntPNnA3cBG4E/oTneP/7ac9rX3QV8YK73Za76gua+Y1uB\nm9svjZuBE+d6f+bqczG0jQUfItPtC+CdwG3tdAWwz1zvz1z0BU2gfgb4PnAn8AdzvS/T7Ic/oxlZ\nbAV+BHwIOBf4V0N1LqcJ3duAk4fKp/y96cWGkqTO5vOJdUnSPGeISJI6M0QkSZ0ZIpKkzgwRSVJn\nhogkqTNDRHu1JAcn+XA7/2tJvtzDe3wwyR9N8TX3JDlkgvKLkvzOzLVOmh5DRHu7pTT31IId9xDa\npSRdfm+mekGWF3BpQTBEtLf7JPDqJDcDlwIHJflCkg1J/nS8Ujsy+L0kXwfem+TVSb6a5KYkNyR5\nbVvvn7YPALslyWDofQ5v629KcunQdlclub2dLhmqn6E6/6F92NJ1wK/00w1SN/P5BozSbLgQOKGq\nTk7ya8AXaZ478XfAN5O8taq+1dZ9oqreAZBkLXBuVf2wvcX+f6W5jcjvAb9RVVuSvGjofU6ieVDQ\nNmBT+1TFZ4BLgDcCjwB/k+TMqvrS+IuSnExzb6eTaO4JdTPw3V56QurAEJGe68Zq7vZKkltpbmQ3\nHiL/qy0/AHgr8IWhZ3AsaX9+E/jvST4P/MXQdr9WVb9oX7+e5kaBLwGur6qH2vKrgXcAXxp63duB\nv6yqrcDWJMPrpDlniEjPtXVo/mme+zvyy/bnIuDhqjp59MVV9eEkbwLOAG5NctIE232m3W6Y3DNN\nPD+iectzItrbPUbzbBGY5EOqqnlOyz1J3jteluTE9uerq+qmqroI+CnPfW7DqO8A70hySJLFwCqa\nJ+wN+zrwT5K8oH0uzD+aTBul2eJIRHu1qnooyTeT3A48wXOfrV07mYfmFvufSfIfaX6P1gC3A3+Q\n5DVtnbVVdXuSN46+bfvef5fko+wIjmuq6isjdW5pD43dBtxLEyrSvOGt4CVJnXk4S5LUmSEiSerM\nEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqbP/D/5voaz0V27eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f01dabb4630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(pcrt, recall)\n",
    "plt.xlabel('threshold')\n",
    "plt.ylabel('recall')\n",
    "plt.ylim([0.95, 1.0])\n",
    "plt.xlim([0.90, 1.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.929600122568\n"
     ]
    }
   ],
   "source": [
    "positive_prior = model.class_count_[1] / sum(model.class_count_)\n",
    "print(positive_prior)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['predict'] = data['predict_proba'] > positive_prior\n",
    "data.to_csv('cleaned_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>status</th>\n",
       "      <th>predict_proba</th>\n",
       "      <th>predict</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2733</th>\n",
       "      <td>были сегодня в биоконтроле тимошу тошнило бабу...</td>\n",
       "      <td>1</td>\n",
       "      <td>4.843399e-08</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5077</th>\n",
       "      <td>москва мо для противовирусной терапии котятам ...</td>\n",
       "      <td>1</td>\n",
       "      <td>9.214664e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8037</th>\n",
       "      <td>представьте что завтра вы можете спасти чью то...</td>\n",
       "      <td>1</td>\n",
       "      <td>2.919623e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13193</th>\n",
       "      <td>нужна же кровь девочке вот мне написали по ссы...</td>\n",
       "      <td>1</td>\n",
       "      <td>9.659473e-02</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15817</th>\n",
       "      <td>донорство крови что нужно знать</td>\n",
       "      <td>1</td>\n",
       "      <td>3.069744e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16705</th>\n",
       "      <td>две недели назад коллега написал смс что его б...</td>\n",
       "      <td>1</td>\n",
       "      <td>2.970216e-27</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18465</th>\n",
       "      <td>поможем ванечке одолеть рак</td>\n",
       "      <td>1</td>\n",
       "      <td>9.102617e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23854</th>\n",
       "      <td>аре требуется ещё кровь лежит в мск в гауз мнп...</td>\n",
       "      <td>1</td>\n",
       "      <td>5.516382e-08</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26131</th>\n",
       "      <td>срочно нужна машина на завтра чтобы в забрать ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.658520e-02</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26671</th>\n",
       "      <td>уважаемые участники группы очень просим финанс...</td>\n",
       "      <td>1</td>\n",
       "      <td>3.618974e-25</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27139</th>\n",
       "      <td>нижневартовск срочно нужна помощь в первом мик...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.047304e-04</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27922</th>\n",
       "      <td>вчера вечером терезе была сделана операция это...</td>\n",
       "      <td>1</td>\n",
       "      <td>3.258462e-06</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28223</th>\n",
       "      <td>передержка александра а нужна помощь транспорт...</td>\n",
       "      <td>1</td>\n",
       "      <td>9.499581e-04</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28338</th>\n",
       "      <td>уважаемые участники группы очень просим финанс...</td>\n",
       "      <td>1</td>\n",
       "      <td>2.125030e-21</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28560</th>\n",
       "      <td>на саму операцию денежных средств не нужно сде...</td>\n",
       "      <td>1</td>\n",
       "      <td>7.721355e-14</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29480</th>\n",
       "      <td>друзья случайно никто не едет в понедельник в ...</td>\n",
       "      <td>1</td>\n",
       "      <td>7.503318e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29790</th>\n",
       "      <td>иногда для того чтобы спасти человека не нужно...</td>\n",
       "      <td>1</td>\n",
       "      <td>5.018854e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29987</th>\n",
       "      <td>маруси нужен донор костного мозга</td>\n",
       "      <td>1</td>\n",
       "      <td>9.073567e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32039</th>\n",
       "      <td>возле офиса по адресу горский сидит кошка похо...</td>\n",
       "      <td>1</td>\n",
       "      <td>7.830650e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32141</th>\n",
       "      <td>в наших силах отменить страшный приговор максиму</td>\n",
       "      <td>1</td>\n",
       "      <td>8.979162e-01</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32559</th>\n",
       "      <td>моему сердцу нужна не кровь а ты</td>\n",
       "      <td>1</td>\n",
       "      <td>1.128774e-02</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33230</th>\n",
       "      <td>нужен донор коту кот кошка без разницы главное...</td>\n",
       "      <td>1</td>\n",
       "      <td>2.395527e-07</td>\n",
       "      <td>False</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  status  \\\n",
       "2733   были сегодня в биоконтроле тимошу тошнило бабу...       1   \n",
       "5077   москва мо для противовирусной терапии котятам ...       1   \n",
       "8037   представьте что завтра вы можете спасти чью то...       1   \n",
       "13193  нужна же кровь девочке вот мне написали по ссы...       1   \n",
       "15817                   донорство крови что нужно знать        1   \n",
       "16705  две недели назад коллега написал смс что его б...       1   \n",
       "18465                       поможем ванечке одолеть рак        1   \n",
       "23854  аре требуется ещё кровь лежит в мск в гауз мнп...       1   \n",
       "26131  срочно нужна машина на завтра чтобы в забрать ...       1   \n",
       "26671  уважаемые участники группы очень просим финанс...       1   \n",
       "27139  нижневартовск срочно нужна помощь в первом мик...       1   \n",
       "27922  вчера вечером терезе была сделана операция это...       1   \n",
       "28223  передержка александра а нужна помощь транспорт...       1   \n",
       "28338  уважаемые участники группы очень просим финанс...       1   \n",
       "28560  на саму операцию денежных средств не нужно сде...       1   \n",
       "29480  друзья случайно никто не едет в понедельник в ...       1   \n",
       "29790  иногда для того чтобы спасти человека не нужно...       1   \n",
       "29987                 маруси нужен донор костного мозга        1   \n",
       "32039  возле офиса по адресу горский сидит кошка похо...       1   \n",
       "32141  в наших силах отменить страшный приговор максиму        1   \n",
       "32559                  моему сердцу нужна не кровь а ты        1   \n",
       "33230  нужен донор коту кот кошка без разницы главное...       1   \n",
       "\n",
       "       predict_proba predict lang  \n",
       "2733    4.843399e-08   False   ru  \n",
       "5077    9.214664e-01   False   ru  \n",
       "8037    2.919623e-01   False   ru  \n",
       "13193   9.659473e-02   False   ru  \n",
       "15817   3.069744e-01   False   ru  \n",
       "16705   2.970216e-27   False   ru  \n",
       "18465   9.102617e-01   False   ru  \n",
       "23854   5.516382e-08   False   ru  \n",
       "26131   1.658520e-02   False   ru  \n",
       "26671   3.618974e-25   False   ru  \n",
       "27139   1.047304e-04   False   ru  \n",
       "27922   3.258462e-06   False   ru  \n",
       "28223   9.499581e-04   False   ru  \n",
       "28338   2.125030e-21   False   ru  \n",
       "28560   7.721355e-14   False   ru  \n",
       "29480   7.503318e-01   False   ru  \n",
       "29790   5.018854e-01   False   ru  \n",
       "29987   9.073567e-01   False   ru  \n",
       "32039   7.830650e-01   False   ru  \n",
       "32141   8.979162e-01   False   ru  \n",
       "32559   1.128774e-02   False   ru  \n",
       "33230   2.395527e-07   False   ru  "
      ]
     },
     "execution_count": 336,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_negative = data[~data['predict'] & (data['status'] == 1)]\n",
    "false_negative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>status</th>\n",
       "      <th>predict_proba</th>\n",
       "      <th>predict</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14394</th>\n",
       "      <td>нужна помощь нухотьчтотохорошее</td>\n",
       "      <td>0</td>\n",
       "      <td>0.956173</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20560</th>\n",
       "      <td>подпишите петицию не оставайтесь равнодушными</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998173</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20598</th>\n",
       "      <td>срочно нужна кровь срочно</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21814</th>\n",
       "      <td>в наличии</td>\n",
       "      <td>0</td>\n",
       "      <td>0.949510</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30201</th>\n",
       "      <td>в наличии</td>\n",
       "      <td>0</td>\n",
       "      <td>0.949510</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32347</th>\n",
       "      <td>ростовнадону помогите спасти жизнь необходимы ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32353</th>\n",
       "      <td>в наличии</td>\n",
       "      <td>0</td>\n",
       "      <td>0.949510</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32358</th>\n",
       "      <td>москва помогите спасти жизни срочно нужны доно...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32475</th>\n",
       "      <td>срочно требуется донор почки группа крови опла...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32565</th>\n",
       "      <td>нужно больше крови</td>\n",
       "      <td>0</td>\n",
       "      <td>0.990180</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32657</th>\n",
       "      <td>срочно до апреля года</td>\n",
       "      <td>0</td>\n",
       "      <td>0.972856</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32771</th>\n",
       "      <td>завтра с братом идем сдавать кровь кому нужно ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999967</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32803</th>\n",
       "      <td>стерлитамак информация для доноров желающих по...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999996</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32829</th>\n",
       "      <td>здравствуйте очень просим вас разместить от им...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32854</th>\n",
       "      <td>нужна срочная помощь</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33022</th>\n",
       "      <td>всем доброй вечер пожалуйста подскажите есть л...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33074</th>\n",
       "      <td>очень нужная информация</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999965</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33082</th>\n",
       "      <td>парализованной многодетной матери в калинингра...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.990349</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33090</th>\n",
       "      <td>в какой поликлиники можно здать кровь и что дл...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999315</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33167</th>\n",
       "      <td>срочно требуется донор положитьная часть печен...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.962826</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33218</th>\n",
       "      <td>максимальной репост</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999480</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33249</th>\n",
       "      <td>якутск помогите спасти жизнь срочно нужны доно...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.994782</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33275</th>\n",
       "      <td>друзья ребенку моих знакомых срочно нужна кров...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33313</th>\n",
       "      <td>с р о ч н о дочка зоя умирает от рака крови оч...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.986482</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33347</th>\n",
       "      <td>всем привет помогите все кто может</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33350</th>\n",
       "      <td>срочно нужен донор для телефона а именно нужен...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999983</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33469</th>\n",
       "      <td>нужен донор для моего планшета нужны кнопки</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999640</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33571</th>\n",
       "      <td>здравствуйте очень просим вас разместить от им...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33601</th>\n",
       "      <td>железнодорожная травма</td>\n",
       "      <td>0</td>\n",
       "      <td>0.974414</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33610</th>\n",
       "      <td>всем привет поступила информация из станции пе...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33627</th>\n",
       "      <td>срочно друзья в актау нужна ваша помощь</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33641</th>\n",
       "      <td>мне нужна кровь</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999939</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33698</th>\n",
       "      <td>нужны доноры срочно</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33804</th>\n",
       "      <td>сургут помогите спасти жизнь срочно нужны доно...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.995369</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33856</th>\n",
       "      <td>дзержинск помогите спасти жизнь срочно нужны д...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33868</th>\n",
       "      <td>помогите распространить информацию репост уваж...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33915</th>\n",
       "      <td>попросили разместить очень нужна ваша помощь у...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33919</th>\n",
       "      <td>подслушано опубликуйте пожалуйста сургут помог...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.967453</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34028</th>\n",
       "      <td>тбилиси помощь добрый вечер пожалуйста отзовит...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999981</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34030</th>\n",
       "      <td>тбилиси помощь добрый вечер пожалуйста отзовит...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34140</th>\n",
       "      <td>всем доброго утра</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999652</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34174</th>\n",
       "      <td>друзья всем большое спасибо кто не остался рав...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34225</th>\n",
       "      <td>присоединяйтесь к флешмобу апреля в помощь яки...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999964</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34298</th>\n",
       "      <td>срочно нужны деньги кто нибудь кто в курсе под...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999852</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34330</th>\n",
       "      <td>озерки поможем вместе спасти жизни срочно нужн...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34331</th>\n",
       "      <td>моя родственница замечательный врач мама двоих...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34356</th>\n",
       "      <td>срочно мальчик руслан лет находится в коме сро...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34435</th>\n",
       "      <td>всем нужна кровь</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34538</th>\n",
       "      <td>если ты давно хотел сдать кровь то вот тот зна...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999995</td>\n",
       "      <td>True</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  status  \\\n",
       "14394                   нужна помощь нухотьчтотохорошее        0   \n",
       "20560     подпишите петицию не оставайтесь равнодушными        0   \n",
       "20598                         срочно нужна кровь срочно        0   \n",
       "21814                                         в наличии        0   \n",
       "30201                                         в наличии        0   \n",
       "32347  ростовнадону помогите спасти жизнь необходимы ...       0   \n",
       "32353                                         в наличии        0   \n",
       "32358  москва помогите спасти жизни срочно нужны доно...       0   \n",
       "32475  срочно требуется донор почки группа крови опла...       0   \n",
       "32565                                нужно больше крови        0   \n",
       "32657                             срочно до апреля года        0   \n",
       "32771  завтра с братом идем сдавать кровь кому нужно ...       0   \n",
       "32803  стерлитамак информация для доноров желающих по...       0   \n",
       "32829  здравствуйте очень просим вас разместить от им...       0   \n",
       "32854                              нужна срочная помощь        0   \n",
       "33022  всем доброй вечер пожалуйста подскажите есть л...       0   \n",
       "33074                           очень нужная информация        0   \n",
       "33082  парализованной многодетной матери в калинингра...       0   \n",
       "33090  в какой поликлиники можно здать кровь и что дл...       0   \n",
       "33167  срочно требуется донор положитьная часть печен...       0   \n",
       "33218                               максимальной репост        0   \n",
       "33249  якутск помогите спасти жизнь срочно нужны доно...       0   \n",
       "33275  друзья ребенку моих знакомых срочно нужна кров...       0   \n",
       "33313  с р о ч н о дочка зоя умирает от рака крови оч...       0   \n",
       "33347                всем привет помогите все кто может        0   \n",
       "33350  срочно нужен донор для телефона а именно нужен...       0   \n",
       "33469       нужен донор для моего планшета нужны кнопки        0   \n",
       "33571  здравствуйте очень просим вас разместить от им...       0   \n",
       "33601                            железнодорожная травма        0   \n",
       "33610  всем привет поступила информация из станции пе...       0   \n",
       "33627           срочно друзья в актау нужна ваша помощь        0   \n",
       "33641                                   мне нужна кровь        0   \n",
       "33698                               нужны доноры срочно        0   \n",
       "33804  сургут помогите спасти жизнь срочно нужны доно...       0   \n",
       "33856  дзержинск помогите спасти жизнь срочно нужны д...       0   \n",
       "33868  помогите распространить информацию репост уваж...       0   \n",
       "33915  попросили разместить очень нужна ваша помощь у...       0   \n",
       "33919  подслушано опубликуйте пожалуйста сургут помог...       0   \n",
       "34028  тбилиси помощь добрый вечер пожалуйста отзовит...       0   \n",
       "34030  тбилиси помощь добрый вечер пожалуйста отзовит...       0   \n",
       "34140                                 всем доброго утра        0   \n",
       "34174  друзья всем большое спасибо кто не остался рав...       0   \n",
       "34225  присоединяйтесь к флешмобу апреля в помощь яки...       0   \n",
       "34298  срочно нужны деньги кто нибудь кто в курсе под...       0   \n",
       "34330  озерки поможем вместе спасти жизни срочно нужн...       0   \n",
       "34331  моя родственница замечательный врач мама двоих...       0   \n",
       "34356  срочно мальчик руслан лет находится в коме сро...       0   \n",
       "34435                                  всем нужна кровь        0   \n",
       "34538  если ты давно хотел сдать кровь то вот тот зна...       0   \n",
       "\n",
       "       predict_proba predict lang  \n",
       "14394       0.956173    True   ru  \n",
       "20560       0.998173    True   ru  \n",
       "20598       1.000000    True   ru  \n",
       "21814       0.949510    True   ru  \n",
       "30201       0.949510    True   ru  \n",
       "32347       1.000000    True   ru  \n",
       "32353       0.949510    True   ru  \n",
       "32358       1.000000    True   ru  \n",
       "32475       0.999998    True   ru  \n",
       "32565       0.990180    True   ru  \n",
       "32657       0.972856    True   ru  \n",
       "32771       0.999967    True   ru  \n",
       "32803       0.999996    True   ru  \n",
       "32829       1.000000    True   ru  \n",
       "32854       1.000000    True   ru  \n",
       "33022       1.000000    True   ru  \n",
       "33074       0.999965    True   ru  \n",
       "33082       0.990349    True   ru  \n",
       "33090       0.999315    True   ru  \n",
       "33167       0.962826    True   ru  \n",
       "33218       0.999480    True   ru  \n",
       "33249       0.994782    True   ru  \n",
       "33275       1.000000    True   ru  \n",
       "33313       0.986482    True   ru  \n",
       "33347       1.000000    True   ru  \n",
       "33350       0.999983    True   ru  \n",
       "33469       0.999640    True   ru  \n",
       "33571       1.000000    True   ru  \n",
       "33601       0.974414    True   ru  \n",
       "33610       1.000000    True   ru  \n",
       "33627       1.000000    True   ru  \n",
       "33641       0.999939    True   ru  \n",
       "33698       1.000000    True   ru  \n",
       "33804       0.995369    True   ru  \n",
       "33856       1.000000    True   ru  \n",
       "33868       1.000000    True   ru  \n",
       "33915       1.000000    True   ru  \n",
       "33919       0.967453    True   ru  \n",
       "34028       0.999981    True   ru  \n",
       "34030       1.000000    True   ru  \n",
       "34140       0.999652    True   ru  \n",
       "34174       1.000000    True   ru  \n",
       "34225       0.999964    True   ru  \n",
       "34298       0.999852    True   ru  \n",
       "34330       1.000000    True   ru  \n",
       "34331       1.000000    True   ru  \n",
       "34356       0.999995    True   ru  \n",
       "34435       0.999999    True   ru  \n",
       "34538       0.999995    True   ru  "
      ]
     },
     "execution_count": 337,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "false_positive = data[data['predict'] & (data['status'] == 0)]\n",
    "false_positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "PicklingError",
     "evalue": "Can't pickle <function VectorizerMixin.build_analyzer.<locals>.<lambda> at 0x7f01d99b6598>: it's not found as sklearn.feature_extraction.text.VectorizerMixin.build_analyzer.<locals>.<lambda>",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_global\u001b[0;34m(self, obj, name)\u001b[0m\n\u001b[1;32m    906\u001b[0m             \u001b[0mmodule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmodule_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 907\u001b[0;31m             \u001b[0mobj2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_getattribute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    908\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36m_getattribute\u001b[0;34m(obj, name)\u001b[0m\n\u001b[1;32m    264\u001b[0m             raise AttributeError(\"Can't get local attribute {!r} on {!r}\"\n\u001b[0;32m--> 265\u001b[0;31m                                  .format(name, obj))\n\u001b[0m\u001b[1;32m    266\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: Can't get local attribute 'VectorizerMixin.build_analyzer.<locals>.<lambda>' on <function VectorizerMixin.build_analyzer at 0x7f01eba05ea0>",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mPicklingError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-338-e6236f9f6408>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexternals\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvectorizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'model/bayes/vectorizer.pkl'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'model/bayes/model.pkl'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(value, filename, compress, cache_size, protocol)\u001b[0m\n\u001b[1;32m    401\u001b[0m         pickler = NumpyPickler(filename, compress=compress,\n\u001b[1;32m    402\u001b[0m                                cache_size=cache_size, protocol=protocol)\n\u001b[0;32m--> 403\u001b[0;31m         \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    404\u001b[0m         \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 408\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    409\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSTOP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    278\u001b[0m                         traceback.format_exc()))\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 520\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    521\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 623\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    624\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    278\u001b[0m                         traceback.format_exc()))\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    473\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    808\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 810\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    811\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    812\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    834\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    835\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 836\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    837\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    838\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    278\u001b[0m                         traceback.format_exc()))\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 520\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    521\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    622\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 623\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    624\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    278\u001b[0m                         traceback.format_exc()))\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    473\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    808\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 810\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    811\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    812\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    834\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    835\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 836\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    837\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    838\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    278\u001b[0m                         traceback.format_exc()))\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    473\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_global\u001b[0;34m(self, obj, name)\u001b[0m\n\u001b[1;32m    909\u001b[0m             raise PicklingError(\n\u001b[1;32m    910\u001b[0m                 \u001b[0;34m\"Can't pickle %r: it's not found as %s.%s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 911\u001b[0;31m                 (obj, module_name, name))\n\u001b[0m\u001b[1;32m    912\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    913\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mobj2\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mPicklingError\u001b[0m: Can't pickle <function VectorizerMixin.build_analyzer.<locals>.<lambda> at 0x7f01d99b6598>: it's not found as sklearn.feature_extraction.text.VectorizerMixin.build_analyzer.<locals>.<lambda>"
     ]
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "\n",
    "joblib.dump(vectorizer, 'model/bayes/vectorizer.pkl')\n",
    "joblib.dump(model, 'model/bayes/model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}