NLP_Gender-Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Machine Learning and Natural Language Processing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "os.chdir('C:/Users/Swati Arora/Data Science using Python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>gender</th>\n",
       "      <th>status</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vimal20011</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>A team of 5 working on various projects relate...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sheom</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': True, u'identity_verifie...</td>\n",
       "      <td>We are an IT solution and service provider com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ezbik</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>System administration is my work &amp; hobby.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>angelme</td>\n",
       "      <td>F</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>Good day! Thank you for taking some time to ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>snitch1</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>I build good relation with clients and deliver...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     username gender                                             status  \\\n",
       "0  vimal20011      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "1       sheom      M  {u'payment_verified': True, u'identity_verifie...   \n",
       "2       ezbik      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "3     angelme      F  {u'payment_verified': False, u'identity_verifi...   \n",
       "4     snitch1      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "\n",
       "                                         description  \n",
       "0  A team of 5 working on various projects relate...  \n",
       "1  We are an IT solution and service provider com...  \n",
       "2          System administration is my work & hobby.  \n",
       "3  Good day! Thank you for taking some time to ch...  \n",
       "4  I build good relation with clients and deliver...  "
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>status</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>nazrulmadina</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>I am one of Self-employed person having more t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sehrishwarraich</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>i am sehrish warraich.I do my job sincerelly a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>samadhinie</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>Since 2006, Web based solution provider (Web e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ebottabi</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>Founder of  a geolocation service developed on...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mrjimoy</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>Me? I am Jimmy, meaning honest, courageous, or...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          username                                             status  \\\n",
       "0     nazrulmadina  {u'payment_verified': False, u'identity_verifi...   \n",
       "1  sehrishwarraich  {u'payment_verified': False, u'identity_verifi...   \n",
       "2       samadhinie  {u'payment_verified': False, u'identity_verifi...   \n",
       "3         ebottabi  {u'payment_verified': False, u'identity_verifi...   \n",
       "4          mrjimoy  {u'payment_verified': False, u'identity_verifi...   \n",
       "\n",
       "                                         description  \n",
       "0  I am one of Self-employed person having more t...  \n",
       "1  i am sehrish warraich.I do my job sincerelly a...  \n",
       "2  Since 2006, Web based solution provider (Web e...  \n",
       "3  Founder of  a geolocation service developed on...  \n",
       "4  Me? I am Jimmy, meaning honest, courageous, or...  "
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Predicting Gender with Username\n",
    "Some potential features of usernames: whether it has capital letters, whether it has digits, number of characters, number of vowels, first and last letters, etc. See http://www.nltk.org/book/ch06.html for some related code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_features(name):\n",
    "    vowels = \"aAeEiIoOuU\"\n",
    "    return {\n",
    "        'last_char': name[-1],\n",
    "        'last_two': name[-2:],\n",
    "        #'last_three':name[-3:],\n",
    "        'first': name[0],\n",
    "        'first2': name[:1],\n",
    "        #'first3':name[:2],\n",
    "        'upper':any(x.isupper() for x in name),\n",
    "        'digit':any(x.isdigit() for x in name),\n",
    "        'length':sum(c.isalpha() for c in name),\n",
    "        'vowel': sum(name.count(c) for c in vowels)\n",
    "       \n",
    "    }\n",
    "\n",
    "# commented first 3 and last three features since it was decreasing the AUC and accuracy of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "username =train.username\n",
    "gender = train.gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_sets =[(extract_features(n),g) for n, g in zip(username,gender)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this function calculate the True Negative , True Positive, False Negative and False Positive values for model\n",
    "true_pos=true_neg=false_pos=false_neg=0\n",
    "def sens_spec(gen,pgen):\n",
    "    tp=tn=fp=fn=0\n",
    "    for g,pg in zip(gen, pgen):\n",
    "        if g==pg=='M':\n",
    "            tp = tp+1\n",
    "        elif g==pg=='F':\n",
    "            tn = tn+1\n",
    "        elif g=='M' and pg=='F':\n",
    "            fn=fn+1\n",
    "        elif g=='F' and pg=='M':\n",
    "            fp=fp+1\n",
    "    return(tp,tn,fp,fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9848\n",
      "accuracy: 0.9768\n",
      "accuracy: 0.9824\n",
      "accuracy: 0.9816\n",
      "accuracy: 0.9831865492393915\n",
      "CV mean accuracy: 0.981757309848\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feature_sets):\n",
    "    training = [feature_sets[i] for i in train_idx]\n",
    "    testing = [feature_sets[i] for i in test_idx]\n",
    "    classifier_dt = nltk.classify.DecisionTreeClassifier.train(feature_sets, entropy_cutoff=0,support_cutoff=0)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_dt, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predicting the gender suinf model\n",
    "name = ([extract_features(n) for n in username])\n",
    "pred_gender_train = [classifier_dt.classify(n) for n in name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calling the function and calcultaing Sensitivity and Specificity for AUC calculation\n",
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.08617753244916244\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Naive Bayes Classifier for Username"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.78\n",
      "accuracy: 0.7792\n",
      "accuracy: 0.776\n",
      "accuracy: 0.7888\n",
      "accuracy: 0.7918334667734187\n",
      "CV mean accuracy: 0.783166693355\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feature_sets):\n",
    "    training = [feature_sets[i] for i in train_idx]\n",
    "    testing = [feature_sets[i] for i in test_idx]\n",
    "    classifier_nb = nltk.NaiveBayesClassifier.train(training)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_nb, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                last_two = 'va'                F : M      =     10.4 : 1.0\n",
      "                last_two = 'nn'                F : M      =      7.6 : 1.0\n",
      "                last_two = '3d'                F : M      =      5.8 : 1.0\n",
      "                last_two = 'm1'                F : M      =      5.8 : 1.0\n",
      "                last_two = 'm3'                F : M      =      5.8 : 1.0\n",
      "                last_two = 'a6'                F : M      =      5.8 : 1.0\n",
      "                last_two = 'iv'                F : M      =      5.8 : 1.0\n",
      "                last_two = '30'                F : M      =      5.8 : 1.0\n",
      "                last_two = 'ju'                F : M      =      5.8 : 1.0\n",
      "                last_two = '47'                F : M      =      5.5 : 1.0\n",
      "                last_two = 'jo'                F : M      =      4.9 : 1.0\n",
      "                last_two = 'ek'                F : M      =      4.9 : 1.0\n",
      "                last_two = 'bd'                M : F      =      4.5 : 1.0\n",
      "                last_two = 'ev'                M : F      =      3.9 : 1.0\n",
      "                last_two = 'ss'                F : M      =      3.9 : 1.0\n",
      "                last_two = 'ol'                M : F      =      3.7 : 1.0\n",
      "                last_two = 'ic'                M : F      =      3.7 : 1.0\n",
      "                last_two = 'ov'                M : F      =      3.7 : 1.0\n",
      "                last_two = 'de'                M : F      =      3.7 : 1.0\n",
      "                last_two = 'vm'                F : M      =      3.5 : 1.0\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Most informative features\n",
    "print (classifier_nb.show_most_informative_features(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "username_test = test.username\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predicting the gender suinf model\n",
    "name = ([extract_features(n) for n in username])\n",
    "pred_gender_train = [classifier_nb.classify(n) for n in name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calling the function and calcultaing Sensitivity and Specificity for AUC calculation\n",
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7802253760196143\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_set = ([extract_features(n) for n in username_test])\n",
    "#zz = classifier.classify([extract_features(n) for n in username_test]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['F', 'M', 'M', 'M', 'M']"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_gender = [classifier_nb.classify(n) for n in test_set]\n",
    "pred_gender[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Storing the test predictions in a list named pred_gender1\n",
    "zz = pd.DataFrame({'prediction':pred_gender,'username':username_test })\n",
    "zz.to_csv('pred_gender1.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maximum Entropy Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.815\n",
      "             2          -0.35212        0.815\n",
      "             3          -0.34650        0.815\n",
      "             4          -0.34146        0.815\n",
      "         Final          -0.33703        0.815\n",
      "accuracy: 0.811048839071257\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.813\n",
      "             2          -0.35567        0.813\n",
      "             3          -0.34922        0.813\n",
      "             4          -0.34347        0.813\n",
      "         Final          -0.33846        0.814\n",
      "accuracy: 0.8118494795836669\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.812\n",
      "             2          -0.35764        0.812\n",
      "             3          -0.35156        0.812\n",
      "             4          -0.34610        0.812\n",
      "         Final          -0.34128        0.813\n",
      "accuracy: 0.8118494795836669\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.810\n",
      "             2          -0.36032        0.810\n",
      "             3          -0.35395        0.810\n",
      "             4          -0.34827        0.810\n",
      "         Final          -0.34332        0.811\n",
      "accuracy: 0.8126501200960768\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.815\n",
      "             2          -0.35274        0.815\n",
      "             3          -0.34695        0.815\n",
      "             4          -0.34175        0.815\n",
      "         Final          -0.33717        0.815\n",
      "accuracy: 0.8118494795836669\n",
      "CV mean accuracy: 0.811849479584\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feature_sets):\n",
    "    train = [feature_sets[i] for i in train_idx]\n",
    "    test = [feature_sets[i] for i in test_idx]\n",
    "    classifier2_me = nltk.classify.MaxentClassifier.train(train, trace=3, max_iter=5)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier2_me, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier2_me.classify(n) for n in name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calling the function and calcultaing Sensitivity and Specificity for AUC calculation\n",
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9965782720273738\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['M', 'M', 'M', 'M', 'M']"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_gender = [classifier2_me.classify(n) for n in test_set]\n",
    "pred_gender[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Storing the test predictions in a list named pred_gender1\n",
    "zz = pd.DataFrame({'prediction':pred_gender,'username':username_test })\n",
    "zz.to_csv('pred_gender4.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Predicting Gender with Description\n",
    "The updated notebook for lecture 11 might be of some help, which now includes demo code for making predictions with NLTK classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import string\n",
    "ps = PorterStemmer()\n",
    "from nltk.tokenize import word_tokenize\n",
    "def preprocess(text):\n",
    "    return [ps.stem(w) for w in word_tokenize(text.lower()) \n",
    "             if w not in string.punctuation and w not in stopwords.words('english')] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.head()\n",
    "description = train.description\n",
    "description_test = test.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "des_words =[(preprocess(des),g) for des,g in zip(description,gender)]\n",
    "#des_words[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2679"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(description_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "des_words_test =[preprocess(des) for des in description_test]\n",
    "#des_words_test[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# flatten the words into one list\n",
    "all_words = [w for words, c in des_words for w in words]\n",
    "all_words_test = [w for words in des_words_test for w in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#all_words_test[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calculating word frequency\n",
    "words_freq = nltk.FreqDist(all_words)\n",
    "words_freq_test = nltk.FreqDist(all_words_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data: Before: 36128 , after: 9953\n",
      "Test data: Before: 20142 , after: 5820\n"
     ]
    }
   ],
   "source": [
    "selected_words = [word for word, freq in words_freq.items() if freq>1]\n",
    "print('train data: Before:',len(words_freq), ', after:', len(selected_words))\n",
    "selected_words_test = [word for word, freq in words_freq_test.items() if freq>1]\n",
    "print('Test data: Before:',len(words_freq_test), ', after:', len(selected_words_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "des_words = [([w for w in words if w in selected_words], c) for words, c in des_words]\n",
    "des_words_test = [[w for w in words if w in selected_words] for words in des_words_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#des_words[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_features(words):\n",
    "    ''' simply using words counts'''\n",
    "    return nltk.FreqDist(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feat = [(extract_features(w),g) for w,g in des_words]\n",
    "feat_test = [extract_features(w) for w in des_words_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.5744\n",
      "accuracy: 0.5432\n",
      "accuracy: 0.5744\n",
      "accuracy: 0.5752\n",
      "accuracy: 0.5508406725380304\n",
      "CV mean accuracy: 0.563608134508\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feat):\n",
    "    training = [feat[i] for i in train_idx]\n",
    "    testing = [feat[i] for i in test_idx]\n",
    "    classifier_nb = nltk.NaiveBayesClassifier.train(training)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_nb, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z =[preprocess(des) for des in description]\n",
    "z = [([w for w in words if w in selected_words]) for words in z]\n",
    "feat_train = [extract_features(w) for w in z]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier_nb.classify(n) for n in feat_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.07182934468520777\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                   femal = 1                   F : M      =     21.5 : 1.0\n",
      "                     mom = 1                   F : M      =     18.0 : 1.0\n",
      "                lifestyl = 1                   F : M      =     15.7 : 1.0\n",
      "                 inquiri = 1                   F : M      =     15.7 : 1.0\n",
      "                    take = 3                   F : M      =     15.7 : 1.0\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print (classifier_nb.show_most_informative_features(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Maximum Entropy Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.815\n",
      "             2          -0.40123        0.803\n",
      "             3          -0.34830        0.789\n",
      "             4          -0.33270        0.774\n",
      "         Final          -0.32500        0.768\n",
      "accuracy: 0.7312\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.810\n",
      "             2          -0.41520        0.789\n",
      "             3          -0.36496        0.771\n",
      "             4          -0.35007        0.754\n",
      "         Final          -0.34343        0.750\n",
      "accuracy: 0.748\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.814\n",
      "             2          -0.35900        0.803\n",
      "             3          -0.31551        0.794\n",
      "             4          -0.30249        0.785\n",
      "         Final          -0.29605        0.780\n",
      "accuracy: 0.768\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.814\n",
      "             2          -0.37149        0.804\n",
      "             3          -0.31923        0.797\n",
      "             4          -0.30302        0.790\n",
      "         Final          -0.29476        0.785\n",
      "accuracy: 0.7512\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.812\n",
      "             2          -0.31540        0.810\n",
      "             3          -0.28328        0.804\n",
      "             4          -0.27114        0.799\n",
      "         Final          -0.26483        0.795\n",
      "accuracy: 0.7862289831865492\n",
      "CV mean accuracy: 0.756925796637\n"
     ]
    }
   ],
   "source": [
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feat):\n",
    "    train = [feat[i] for i in train_idx]\n",
    "    test = [feat[i] for i in test_idx]\n",
    "    classifier2_me = nltk.classify.MaxentClassifier.train(train, trace=3, max_iter=5)       \n",
    "    accu.append( nltk.classify.util.accuracy(classifier2_me, test) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier2_me.classify(n) for n in feat_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#classifier = nltk.classifier2_me.MaxentClassifier.train(feat, trace=3, max_iter=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9153240201262268\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_test = [classifier2_me.classify(n) for n in feat_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#nltk.classify.util.accuracy(classifier, feat_test)\n",
    "# support your predictions are stored in a list named pred_gender1\n",
    "zz = pd.DataFrame({'prediction':pred_test,'Username':username_test })\n",
    "zz.to_csv('pred_gender3.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.9784\n",
      "accuracy: 0.9856\n",
      "accuracy: 0.9784\n",
      "accuracy: 0.9824\n",
      "accuracy: 0.9839871897518014\n",
      "CV mean accuracy: 0.98175743795\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(feat):\n",
    "    training = [feature_sets[i] for i in train_idx]\n",
    "    testing = [feature_sets[i] for i in test_idx]\n",
    "    classifier_dt = nltk.classify.DecisionTreeClassifier.train(feature_sets, entropy_cutoff=0,support_cutoff=0)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_dt, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier_dt.classify(n) for n in feat_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_test = [classifier_dt.classify(n) for n in feat_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#nltk.classify.util.accuracy(classifier, feat_test)\n",
    "# support your predictions are stored in a list named pred_gender1\n",
    "zz = pd.DataFrame({'prediction':pred_test,'Username':username_test })\n",
    "zz.to_csv('pred_gender2.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Predicting Gender with Username, Description, and Status\n",
    "If you need to merge multiple dict-format features into one, check the following question: https://stackoverflow.com/questions/38987/how-to-merge-two-dictionaries-in-a-single-expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>gender</th>\n",
       "      <th>status</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vimal20011</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>A team of 5 working on various projects relate...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sheom</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': True, u'identity_verifie...</td>\n",
       "      <td>We are an IT solution and service provider com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ezbik</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>System administration is my work &amp; hobby.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>angelme</td>\n",
       "      <td>F</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>Good day! Thank you for taking some time to ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>snitch1</td>\n",
       "      <td>M</td>\n",
       "      <td>{u'payment_verified': False, u'identity_verifi...</td>\n",
       "      <td>I build good relation with clients and deliver...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     username gender                                             status  \\\n",
       "0  vimal20011      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "1       sheom      M  {u'payment_verified': True, u'identity_verifie...   \n",
       "2       ezbik      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "3     angelme      F  {u'payment_verified': False, u'identity_verifi...   \n",
       "4     snitch1      M  {u'payment_verified': False, u'identity_verifi...   \n",
       "\n",
       "                                         description  \n",
       "0  A team of 5 working on various projects relate...  \n",
       "1  We are an IT solution and service provider com...  \n",
       "2          System administration is my work & hobby.  \n",
       "3  Good day! Thank you for taking some time to ch...  \n",
       "4  I build good relation with clients and deliver...  "
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parse Json format status as dictionary\n",
    "from ast import literal_eval\n",
    "status = train['status'].apply(literal_eval)\n",
    "status_test = test['status'].apply(literal_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spliting the dictionary format status as multiple columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for train data\n",
    "status_new = pd.DataFrame()\n",
    "for l in status.iteritems():\n",
    "    row = (l[1])\n",
    "    df =pd.DataFrame(list(row.items()))\n",
    "    df['row']=l[0]\n",
    "    #print(df)\n",
    "    status_new = status_new.append(df.pivot(index='row',columns=0, values=1))\n",
    "    #print(status_new)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for test data\n",
    "status_test_new = pd.DataFrame()\n",
    "for l in status_test.iteritems():\n",
    "    row = (l[1])\n",
    "    df =pd.DataFrame(list(row.items()))\n",
    "    df['row']=l[0]\n",
    "    #print(df)\n",
    "    status_test_new = status_test_new.append(df.pivot(index='row',columns=0, values=1))\n",
    "    #print(status_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deposit_made</th>\n",
       "      <th>email_verified</th>\n",
       "      <th>facebook_connected</th>\n",
       "      <th>identity_verified</th>\n",
       "      <th>payment_verified</th>\n",
       "      <th>phone_verified</th>\n",
       "      <th>profile_complete</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "0    deposit_made  email_verified  facebook_connected  identity_verified  \\\n",
       "row                                                                        \n",
       "0            True            True               False              False   \n",
       "1            True            True                True              False   \n",
       "2           False            True               False              False   \n",
       "3            True            True               False              False   \n",
       "4           False            True               False              False   \n",
       "\n",
       "0    payment_verified  phone_verified  profile_complete  \n",
       "row                                                      \n",
       "0               False           False              True  \n",
       "1                True           False              True  \n",
       "2               False           False              True  \n",
       "3               False            True              True  \n",
       "4               False           False              True  "
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df = pd.DataFrame(list(a_dictionary.items()), columns = ['column1', 'column2'])\n",
    "status_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'better': 2,\n",
       "          'data': 1,\n",
       "          'entri': 1,\n",
       "          'experienc': 1,\n",
       "          'job': 2,\n",
       "          'mission': 1,\n",
       "          'next': 1,\n",
       "          'previou': 1,\n",
       "          'submit': 1,\n",
       "          'time': 1})"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# description\n",
    "feat_test[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'digit': False,\n",
       " 'first': 's',\n",
       " 'first2': 's',\n",
       " 'last_char': 'm',\n",
       " 'last_two': 'om',\n",
       " 'length': 5,\n",
       " 'upper': False,\n",
       " 'vowel': 2}"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Username\n",
    "feature_sets[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Converting the status to dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "st = status_new.to_dict(orient='records')\n",
    "st_test = status_test_new.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'deposit_made': True,\n",
       " 'email_verified': True,\n",
       " 'facebook_connected': True,\n",
       " 'identity_verified': False,\n",
       " 'payment_verified': True,\n",
       " 'phone_verified': False,\n",
       " 'profile_complete': True}"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "st[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge Username, Description and Status dictionary in single expression using zip and unzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for train data\n",
    "comb_features = []\n",
    "for user,des,sta in zip(feature_sets,feat,st):\n",
    "    #print(user)\n",
    "    #print(des)\n",
    "    z = {**user[0], **des[0]}\n",
    "    x = {**z,**sta}\n",
    "    comb_features.append(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for test data\n",
    "comb_features_test = []\n",
    "for user,des,sta in zip(test_set,feat_test,st_test):\n",
    "    #print(user)\n",
    "    #print(des)\n",
    "    z = {**user, **des}\n",
    "    x = {**z,**sta}\n",
    "    comb_features_test.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6249"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(comb_features)\n",
    "#len(feature_sets)\n",
    "#len(feat)\n",
    "#comb_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comb_features_train = [(w,g) for w,g in zip(comb_features,gender)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comb_features_train_auc = [w for w in comb_features]\n",
    "comb_features_test = [w for w in comb_features_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.5776\n",
      "accuracy: 0.5528\n",
      "accuracy: 0.588\n",
      "accuracy: 0.588\n",
      "accuracy: 0.5876701361088871\n",
      "CV mean accuracy: 0.578814027222\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(comb_features_train):\n",
    "    training = [comb_features_train[i] for i in train_idx]\n",
    "    testing = [comb_features_train[i] for i in test_idx]\n",
    "    classifier_nb = nltk.NaiveBayesClassifier.train(training)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_nb, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier_nb.classify(n) for n in comb_features_train_auc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.06387382714885191\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                   femal = 1                   F : M      =     21.5 : 1.0\n",
      "                     mom = 1                   F : M      =     18.0 : 1.0\n",
      "                lifestyl = 1                   F : M      =     15.7 : 1.0\n",
      "                 inquiri = 1                   F : M      =     15.7 : 1.0\n",
      "                    take = 3                   F : M      =     15.7 : 1.0\n",
      "                   woman = 1                   F : M      =     14.6 : 1.0\n",
      "                   encod = 1                   F : M      =     14.6 : 1.0\n",
      "                last_two = 'va'                F : M      =     13.1 : 1.0\n",
      "                  well.i = 1                   F : M      =     12.9 : 1.0\n",
      "                accommod = 1                   F : M      =     12.9 : 1.0\n",
      "                     cat = 1                   F : M      =     12.9 : 1.0\n",
      "                  museum = 1                   F : M      =     12.9 : 1.0\n",
      "            well-written = 1                   F : M      =     12.9 : 1.0\n",
      "                 booklet = 1                   F : M      =     12.9 : 1.0\n",
      "             autorespond = 1                   F : M      =     12.9 : 1.0\n",
      "                 context = 1                   F : M      =     12.9 : 1.0\n",
      "                   corp. = 1                   F : M      =     12.9 : 1.0\n",
      "                   topic = 2                   F : M      =     12.9 : 1.0\n",
      "                   first = 2                   F : M      =     12.7 : 1.0\n",
      "                   linux = 1                   M : F      =     11.3 : 1.0\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print (classifier_nb.show_most_informative_features(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maximum Entropy Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.812\n",
      "             2          -0.28269        0.814\n",
      "             3          -0.23671        0.814\n",
      "             4          -0.22565        0.813\n",
      "         Final          -0.22114        0.812\n",
      "accuracy: 0.8144\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.814\n",
      "             2          -0.33672        0.815\n",
      "             3          -0.28288        0.810\n",
      "             4          -0.26800        0.800\n",
      "         Final          -0.26143        0.795\n",
      "accuracy: 0.7896\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.809\n",
      "             2          -0.28255        0.810\n",
      "             3          -0.23737        0.810\n",
      "             4          -0.22717        0.810\n",
      "         Final          -0.22309        0.809\n",
      "accuracy: 0.8296\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.812\n",
      "             2          -0.25784        0.813\n",
      "             3          -0.22288        0.813\n",
      "             4          -0.21581        0.813\n",
      "         Final          -0.21303        0.813\n",
      "accuracy: 0.8168\n",
      "  ==> Training (5 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -0.69315        0.818\n",
      "             2          -0.25390        0.818\n",
      "             3          -0.22220        0.819\n",
      "             4          -0.21364        0.819\n",
      "         Final          -0.20986        0.818\n",
      "accuracy: 0.7910328262610088\n",
      "CV mean accuracy: 0.808286565252\n"
     ]
    }
   ],
   "source": [
    "\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(comb_features_train):\n",
    "    train = [comb_features_train[i] for i in train_idx]\n",
    "    test = [comb_features_train[i] for i in test_idx]\n",
    "    classifier2_me = nltk.classify.MaxentClassifier.train(train, trace=3, max_iter=5)       \n",
    "    accu.append( nltk.classify.util.accuracy(classifier2_me, test) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier2_me.classify(n) for n in comb_features_train_auc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9899570936866425\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_test = [classifier2_me.classify(n) for n in comb_features_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# support your predictions are stored in a list named pred_gender5\n",
    "zz = pd.DataFrame({'prediction':pred_test,'Username':username_test })\n",
    "zz.to_csv('pred_gender5.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.98\n",
      "accuracy: 0.9848\n",
      "accuracy: 0.9824\n",
      "accuracy: 0.9824\n",
      "accuracy: 0.9791833466773419\n",
      "CV mean accuracy: 0.981756669335\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "k_fold = KFold(n_splits=5, shuffle=True)\n",
    "accu = []\n",
    "for train_idx, test_idx in k_fold.split(comb_features_train):\n",
    "    training = [feature_sets[i] for i in train_idx]\n",
    "    testing = [feature_sets[i] for i in test_idx]\n",
    "    classifier_dt = nltk.classify.DecisionTreeClassifier.train(feature_sets, entropy_cutoff=0,support_cutoff=0)   \n",
    "    accu.append( nltk.classify.util.accuracy(classifier_dt, testing) )\n",
    "    print('accuracy:', accu[len(accu)-1])    \n",
    "print('CV mean accuracy:', np.mean(accu)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_gender_train = [classifier_dt.classify(n) for n in comb_features_train_auc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "true_pos,true_neg,false_pos,false_neg = sens_spec(gender,pred_gender_train)\n",
    "TPR = true_pos/(true_pos+false_neg)\n",
    "TNR = true_neg/( true_neg+false_pos)\n",
    "FPR = false_pos/(false_pos+true_neg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10816937553464505\n"
     ]
    }
   ],
   "source": [
    "#AUC\n",
    "roc_auc =TPR *(1-TNR)\n",
    "print(roc_auc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Username: I have extracted the features giving the best model while removing ones which\n",
    "reduces the model performance. The result of three models are as follows:\n",
    "<br><br>\n",
    "- Decision Tree Clasifier\n",
    "-- Accuracy: 0.98 / AUC : 0.086 \n",
    "- Naive Bayes Classifier\n",
    "--Accuracy: 0.78 / AUC : 0.785\n",
    "- Maximum Entropy Classifier\n",
    "--Accuracy: 0.82 / AUC: 0.99\n",
    "\n",
    "<br><br>\n",
    "For Description: stemming the sentence into words and removed the stopwords and punctuations\n",
    "Created the disctionary by calculating the word frequency and then implementing the models:\n",
    "\n",
    "- Naive Bayes Clasifier\n",
    "-- Accuracy: 0.56 / AUC : 0.08\n",
    "- Maximum Entropy Clasifier\n",
    "-- Accuracy: 0.76 / AUC : 0.81\n",
    "- Decision Tree Clasifier\n",
    "-- Accuracy: 0.98 / AUC : 1.0\n",
    "\n",
    "<br><br>\n",
    "For predicting Gender using username, description and Status: I have used same format for username and description as above.\n",
    "I splitted the status as multiple column of value True and False, and then converted it to dictionary format.\n",
    "Merged all the features as single unit and applied the models to predict Gender:\n",
    "\n",
    "- Naive Bayes Classifier\n",
    "--Accuracy: 0.58 / AUC : 0.05\n",
    "- Maximum Entropy Clasifier\n",
    "-- Accuracy: 0.8 / AUC : 0.86\n",
    "- Decision Tree Clasifier\n",
    "-- Accuracy: 0.98 / AUC : 0.1\n",
    "<br><br>\n",
    "For few models even if the accuracy is more the AUC is less. SO, I have evaluated the model based on both Accuracy and AUC values. The best 5 model according to me are saved.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}