Email Categorization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Email Categorization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Goal: Predict A and B (email categories) based on values C and D (subject and description). Essentially, I looked at emails and tried to classify them into categories. I used some serious regular expressions and cleaning of the subject line, then used Glove word embeddings and a neural net and achieved about 80% accuracy (unfortunately the net takes a long time to run). I also used tf-idf and achieved about 64% accuracy. For improvement, the model needs a LOT more emails to train on--some categories had under 50 emails in them total! \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#import xgboost as xgb (this doesn't work but can fix later, neural nets are cooler anyway)\n",
    "import numpy as np\n",
    "from textblob import TextBlob\n",
    "import string\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer\n",
    "from sklearn import decomposition, ensemble\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras import layers, models, optimizers\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report\n",
    "import wordsegment\n",
    "import os\n",
    "import sys\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.layers import Dense, Input, GlobalMaxPooling1D\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding, Dense, Activation, Flatten\n",
    "from keras.models import Model\n",
    "from keras.initializers import Constant\n",
    "import re\n",
    "import nltk\n",
    "import pycountry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [],
   "source": [
    "# will need to change path later\n",
    "data = pd.read_excel(\"HRTM_CASE.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.dropna(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {},
   "outputs": [],
   "source": [
    "Subject = []\n",
    "Description = []\n",
    "Subject.append(\"memes's are hello\")\n",
    "Description.append(\"we're tryna is ur eating isntworking and I cant believe that it wouldn't do that\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data = pd.DataFrame({'Subject': Subject, 'Description': Description})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use Cantor pairing function to find each unique category for emails to go in\n",
    "data[\"Category\"] = 1/2 * (data.KeyCategory + data.KeyServiceOffering) * (data.KeyCategory + data.KeyServiceOffering + 1) + data.KeyCategory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label encode these categories\n",
    "data[\"Category\"] = data[\"Category\"].astype('category').cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_original = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0     2269\n",
       "12.0    2002\n",
       "2.0     1961\n",
       "4.0     1735\n",
       "8.0     1188\n",
       "5.0      784\n",
       "1.0      501\n",
       "10.0     314\n",
       "13.0     157\n",
       "7.0      121\n",
       "11.0      52\n",
       "3.0       34\n",
       "Name: KeyCategory, dtype: int64"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"KeyCategory\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Cleaning/Basic Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get rid of all emails with reply or FW in the subject line (we are just gonna do what we can, those are too hard)\n",
    "indices = []\n",
    "\n",
    "for i, row in data['Subject'].items():\n",
    "    if re.match('^[A-Za-z]{2}:\\s*', data[\"Subject\"][i]):\n",
    "        indices.append(i)\n",
    "# match = re.search(pattern, string)\n",
    "# if match:\n",
    "#     process(match)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop(indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9173"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Description'][11603];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set everything to lower case\n",
    "data['Description'] = data['Description'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "data['Subject'] = data['Subject'].apply(lambda x: \" \".join(x.lower() for x in x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove emails and special formatting stuff (must do this before we remove punctuation, or it will go away)\n",
    "# remove emails\n",
    "data['Description'] = data['Description'].str.replace('\\S*@\\S*\\s?',' ')\n",
    "data['Subject'] = data['Subject'].str.replace('\\S*@\\S*\\s?',' ')\n",
    "# remove urls, www. or http:// format\n",
    "data['Description'] = data['Description'].str.replace('^https?:\\/\\/.*[\\r\\n]*', ' ')\n",
    "data['Description'] = data['Description'].str.replace('\\s\\S*www.\\S*\\s', ' ')\n",
    "data['Subject'] = data['Subject'].str.replace('^https?:\\/\\/.*[\\r\\n]*', ' ')\n",
    "data['Subject'] = data['Subject'].str.replace('\\s\\S*www.\\S*\\s', ' ')\n",
    "\n",
    "# remove everything in from, sent, and to fields (will do subject in next box)\n",
    "# this one's only for description\n",
    "data['Description'] = data['Description'].str.replace('from:\\s.*?sent:\\s', 'sent: ')\n",
    "data['Description'] = data['Description'].str.replace('sent:\\s.*?to:\\s', 'to: ')\n",
    "data['Description'] = data['Description'].str.replace('to:\\s.*?subject:\\s', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove the subject from the description (slow but we gotta do it) \n",
    "# the regex just removes a fw: or a re: from the subject line, since\n",
    "# we don't have to match that part\n",
    "# for i, row in data.iterrows():\n",
    "#     text = re.sub('\\S+:\\s*','', row['Subject'])\n",
    "#     if row['Description'].replace(text, '') != row['Description']:\n",
    "#         data['Description'][i] = row['Description'].replace(text, ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # count number of dates in description\n",
    "# def description_dates(x):\n",
    "#     date1 = re.findall('\\s*(19|20)*\\d\\d[- /.](0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])\\s*', x[\"Description\"])\n",
    "#     date2 = re.findall('\\s*(0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])[- /.](19|20)*\\d\\d\\s*', x[\"Description\"])\n",
    "#     return (len(date1) + len(date2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data[\"DescriptionDates\"] = data.apply(description_dates, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # count number of dates in subject\n",
    "# def subject_dates(x):\n",
    "#     date1 = re.findall('\\s*(19|20)*\\d\\d[- /.](0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])\\s*', x.Subject)\n",
    "#     date2 = re.findall('\\s*(0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])[- /.](19|20)*\\d\\d\\s*', x.Subject)\n",
    "#     return (len(date1) + len(date2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data[\"SubjectDates\"] = data.apply(subject_dates, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace any numerical dates with the word \"date\" so that we don't lose dates in the model \n",
    "data['Description'] = data[\"Description\"].str.replace('\\s*(19|20)*\\d\\d[- /.](0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])\\s*', ' date ')\n",
    "data['Description'] = data[\"Description\"].str.replace('\\s*(0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])[- /.](19|20)*\\d\\d\\s*', ' date ')\n",
    "data['Subject'] = data[\"Subject\"].str.replace('\\s*(19|20)*\\d\\d[- /.](0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])\\s*', ' date ')\n",
    "data['Subject'] = data[\"Subject\"].str.replace('\\s*(0*[1-9]|1[012])[- /.](0*[1-9]|[12][0-9]|3[01])[- /.](19|20)*\\d\\d\\s*', ' date ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove \"___ floor\" strings, then remove all words/terms with numbers in them\n",
    "data['Description'] = data['Description'].str.replace(\"\\s[a-zA-Z0-9]*\\sfloor\", \" \")\n",
    "data['Description'] = data['Description'].str.replace(\"\\\\w*[0-9]+\\\\w*\\\\s*\", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove all non alphabetical characters (punctuation, underscores, dashes, apostrophes, numbers, etc)\n",
    "data['Description'] = data['Description'].str.replace(\"[^a-zA-Z]\", \" \")\n",
    "data['Subject'] = data['Subject'].str.replace(\"[^a-zA-Z]\", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove excess spaces (important for what we're doing next)\n",
    "data['Description'] = data[\"Description\"].str.replace( '\\s+', ' ')\n",
    "data['Subject'] = data[\"Subject\"].str.replace( '\\s+', ' ')\n",
    "# remove trailing whitespace\n",
    "data['Description'] = data['Description'].str.strip()\n",
    "data['Subject'] = data['Subject'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [],
   "source": [
    "# regex to remove roads and streets\n",
    "data['Description'] = data['Description'].str.replace(\"\\s[a-zA-Z]*\\scity\", \" \")\n",
    "data['Description'] = data['Description'].str.replace(\"\\s[a-zA-Z]*\\sroad\", \" \")\n",
    "data['Description'] = data['Description'].str.replace(\"\\s[a-zA-Z]*\\sstreet\", \" \")\n",
    "# remove building number\n",
    "data['Description'] = data[\"Description\"].str.replace('building no', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-425-5cde922696a3>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-425-5cde922696a3>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    6744     team request confirm good standing performance...\u001b[0m\n\u001b[1;37m                ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "6744     team request confirm good standing performance...\n",
    "6796     good day sarah ramirez avega managed care woul...\n",
    "7558"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Description'][7558]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove all the confidentiality messages and all that crap (don't have to worry about stopwords, \n",
    "# but it's worth doing this before we do the other stuff since we're looking for such specific patterns)\n",
    "# also look at some stuff below the header that we can remove (there are a lot of offices, so this isn't the fastest)\n",
    "\n",
    "#another message\n",
    "data['Description'] = data[\"Description\"].str.replace('email contains confidential information', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('sole use of the intended recipient', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('if you are not the intended recipient', ' ')\n",
    "data['Description'] = data['Description'].str.replace('please contact the sender delete this email', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('maintain the confidentiality of what you may have read', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('save paper think before you print', ' ')\n",
    "\n",
    "# and another\n",
    "data['Description'] = data[\"Description\"].str.replace('i am writing this letter to inform you of fadv messaging requirement', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('please be informed that moving forward all messages will be encrypted', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('compliance with the ra known as the data privacy act', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('please do not hesitate to contact us if you need further information', ' ')\n",
    "\n",
    "# and another message\n",
    "data['Description'] = data[\"Description\"].str.replace('information contained in this message', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('intended only for the recipient', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('may be a confidential attorney client communication or may otherwise be privileged and confidential and protected from disclosure', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('reader of this message is not the intended recipient', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('employee or agent responsible for delivering this message to the intended recipient', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('please be aware that any dissemination or copying of this communication is strictly prohibited', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('please immediately notify us by replying to the message and deleting it from your computer', ' ')\n",
    "\n",
    "# kpmg india message\n",
    "data['Description'] = data[\"Description\"].str.replace('allows reasonable personal use of the e mail system', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('views and opinions expressed in these communications', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('do not necessarily represent those of kpmg', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('disclaimer the information in this e mail is confidential and may be legally privileged', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('it is intended solely for the addressee', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('access to this e mail by anyone else is unauthorized', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('if you have received this communication in error', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('please address with the subject heading received in error send to then delete the e mail and destroy any copies of it', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('if you are not the intended recipient', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('any disclosure copying distribution', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('any action taken or omitted to be taken in reliance on it is prohibited and may be unlawful', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('any opinions or advice contained in this e mail are subject to the terms and conditions', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('expressed in the governing kpmg client engagement letter', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('opinions conclusions and other information in this e mail and any attachments that do not relate to the official business of the firm', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('neither given nor endorsed by it', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('kpmg cannot guarantee that e mail communications are secure or error free', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('nformation could be intercepted corrupted amended lost destroyed arrive late or incomplete or contain viruses', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('kpmg an indian partnership and a member firm of kpmg international cooperative', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('kpmg international a swiss entity that serves as a coordinating entity for a network of independent firms operating under the kpmg name', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('kpmg international cooperative ldquo kpmg international rdquo provides no services to clients', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('each member firm of kpmg international cooperative ldquo', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('kpmg international rdquo is a legally distinct and separate entity and each describes itself as such', ' ')\n",
    "\n",
    "# US confidentiality message\n",
    "data[\"Description\"] = data[\"Description\"].str.replace('this email was intended', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace('s p global market intelligence', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace('information contained in this message', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' intended only for the recipient', '')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' confidential attorney client communication', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' or may otherwise be', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' privileged and confidential and protected from disclosure', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' if the reader of this message is not the intended recipient', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' employee or agent responsible for delivering this message to the intended recipient', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' please be aware that any dissemination or copying of this communication is strictly prohibited', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' if you have received this communication in error please immediately notify us by replying to the message and deleting it from your computer', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' s p global inc reserves the right subject to applicable local law', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' monitor review and process the content of any electronic message or information', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' sent to or from s p global inc e mail addresses', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' without informing the sender or recipient of the message', ' ')\n",
    "data[\"Description\"] = data[\"Description\"].str.replace(' by sending electronic message or information to s p global inc e mail addresses', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('you as the sender are consenting to s p global inc', ' ')\n",
    "data['Description'] = data[\"Description\"].str.replace('processing any of your personal data therein', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ^note that there may be more automated messages like this... different offices might have different messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove excess whitespace and space once again\n",
    "data['Description'] = data[\"Description\"].str.replace( '\\s+', ' ')\n",
    "data['Subject'] = data[\"Subject\"].str.replace( '\\s+', ' ')\n",
    "data['Description'] = data['Description'].str.strip()\n",
    "data['Subject'] = data['Subject'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove countries, cities, and other subcountry names from data description, as well as phrases\n",
    "geography = pd.read_csv(\"https://raw.githubusercontent.com/datasets/world-cities/master/data/world-cities.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set everything to lower case, and get rid of special characters (since we did that to the original data)\n",
    "names = pd.Series(geography['name'].unique())\n",
    "countries = pd.Series(geography['country'].unique())\n",
    "subcountries = pd.Series(geography['subcountry'].unique())\n",
    "\n",
    "names = names.apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "countries = countries.apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "subcountries = subcountries.apply(lambda x: \" \".join(x.lower() for x in str(x).split()))\n",
    "\n",
    "# remove all non alphabetical characters (punctuation, underscores, dashes, apostrophes, numbers, etc)\n",
    "names = names.str.replace(\"[^a-zA-Z]\", \" \")\n",
    "countries = countries.str.replace(\"[^a-zA-Z]\", \" \")\n",
    "subcountries = subcountries.str.replace(\"[^a-zA-Z]\", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NEED TO ELIMINATE DOUBLE SPACE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use a list of phrases to remove, then do a str.replace for all of them\n",
    "remove = pd.concat([names, countries, subcountries], ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove = remove.str.replace( '\\s+', ' ')\n",
    "remove = remove.str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove stopwords and other super common words, as well as single word countries (may have to do a for loop for the double word countries)\n",
    "# words that need to be removed: fw: re: to: from: sent: to: \"hr operations\", and any one letter things typos\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "stop = set(stopwords.words('english'))\n",
    "stop.update((\"fw\",\"re\",\"subject\",\"to\",\"from\",\"sent\", \"spg\", \"spgmi\", \"inc\", \n",
    "             \"spgi\", \"www\", \"com\", \"spgmi\", \"â\", \"global\", \"please\", \"may\", \"â\", \"hi\", \"hello\", \"thanks\", \"ã\"\n",
    "             \"thank\", \"https\", \"spgr\", \"th\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"m\", \"n\", \"o\",\n",
    "             \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\", \"w\", \"x\", \"y\", \"z\", \"dear\", \"sir\", \"madam\", \"building\", \"floor\", \"twitter\",\n",
    "             \"linkedin\", \"facebook\", \"google\", \"youtube\", \"you tube\", \"kpmg\", \"kindly\", \"tel\", \"regards\"))\n",
    "stop.update(remove)\n",
    ",stop.update\n",
    "data['Description'] = data['Description'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n",
    "data['Subject'] = data['Subject'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get a list of all the words used less than 10 times, and a list for words used 10 or more times \n",
    "# (a word should be in one list or the other)\n",
    "vc = pd.Series(' '.join(data['Description'] + data['Subject']).split()).value_counts()\n",
    "common = vc[vc >= 10].index\n",
    "uncommon = vc[vc < 10].index\n",
    "words = common.append(uncommon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use word segmentation on all words used less than 10 times (they are often two words stuck together!)\n",
    "# don't for the other terms\n",
    "\n",
    "from wordsegment import load,segment\n",
    "load()\n",
    "\n",
    "# create an array to store correct (split) strings of uncommon words\n",
    "corrections = []\n",
    "\n",
    "# fill this array with a for loop\n",
    "for i in range(len(common)):\n",
    "    corrections.append(str.split(common[i]))\n",
    "for j in range(len(uncommon)):\n",
    "    corrections.append(segment(uncommon[j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to dictionary\n",
    "dictionary = dict(zip(words, corrections))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply dictionary to our data\n",
    "data['Description'] = data['Description'].apply(lambda x: \" \".join(\" \".join(dictionary[x]) for x in x.split() if x in dictionary))\n",
    "data['Subject'] = data['Subject'].apply(lambda x: \" \".join(\" \".join(dictionary[x]) for x in x.split() if x in dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Use the same process with spell check. Check spelling on words used under 20 times.\n",
    "# # (a word should be in one list or the other). Can tweak this and play with it!\n",
    "# vc = pd.Series(' '.join(data['Description'] + data['Subject']).split()).value_counts()\n",
    "# common = vc[vc >= 10].index\n",
    "# uncommon = vc[vc < 10].index\n",
    "# words = common.append(uncommon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # re-initialize the corrections array to make another dictionary (should be faster for textblob, \n",
    "# # because of how the algorithm works, it looks for distance between a correctly spelled word)\n",
    "# # or is just doing it the hard way faster?\n",
    "\n",
    "# # note: this has to be run overnight, can then pickle your datafile for next time\n",
    "# corrections = []\n",
    "\n",
    "# # fill this array with a for loop\n",
    "# for i in range(len(common)):\n",
    "#     corrections.append(common[i])\n",
    "# for j in range(len(uncommon)):\n",
    "#     if len(uncommon[j]) < 6:\n",
    "#         corrections.append(uncommon[j])\n",
    "#     else:\n",
    "#         corrections.append(str(TextBlob(uncommon[j]).correct()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # convert to dictionary\n",
    "# dictionary = dict(zip(words, corrections))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 427,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # apply dictionary to our data (slight difference in syntax)\n",
    "# data['Description'] = data['Description'].apply(lambda x: \" \".join(dictionary[x] for x in x.split() if x in dictionary))\n",
    "# data['Subject'] = data['Subject'].apply(lambda x: \" \".join(dictionary[x] for x in x.split() if x in dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove stopwords and other super common words one more time\n",
    "from nltk.corpus import stopwords\n",
    "stop = set(stopwords.words('english'))\n",
    "stop.update((\"fw\",\"re\",\"subject\",\"to\",\"from\",\"sent\", \"hr operations\", \"spg\", \"s&p\", \"s &p\", \"s& p\", \"spgmi\", \"inc\", \n",
    "             \"spgi\", \"www\", \"com\", \"spgmi\", \"â\", \"global\", \"please\", \"may\", \"â\", \"hi\", \"hello\", \"thanks\", \"ã\"\n",
    "             \"thank\", \"https\", \"spgr\", \"th\", \"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"m\", \"n\", \"o\",\n",
    "             \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\", \"w\", \"x\", \"y\", \"z\", \"dear\", \"sir\", \"madam\", \"building\", \"floor\", \"twitter\",\n",
    "             \"linkedin\", \"facebook\", \"google\", \"youtube\", \"you tube\", \"kpmg\", \"kindly\", \"tel\", \"regards\"))\n",
    "stop.update(remove)\n",
    "stop.update\n",
    "data['Description'] = data['Description'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n",
    "data['Subject'] = data['Subject'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmatize all the words (for whatever reason Bryan's worked better so I'm using Bryan's)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "def lemmatize_text(text):\n",
    "    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]\n",
    "\n",
    "data['Description'] = data.apply(lambda x: lemmatize_text(x['Description']), axis = 1)\n",
    "data['Description'] = data.apply(lambda x: \" \".join(x[\"Description\"]), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get rid of least frequent terms? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [],
   "source": [
    "vc = pd.Series(' '.join(data['Description'] + data['Subject']).split()).value_counts()\n",
    "uncommon = vc[vc < 2].index\n",
    "\n",
    "data['Description'] = data['Description'].apply(lambda x: \" \".join(x for x in x.split() if x not in uncommon))\n",
    "data['Subject'] = data['Subject'].apply(lambda x: \" \".join(x for x in x.split() if x not in uncommon))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [],
   "source": [
    "# look for more things that we need to do!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'team noticed unable enter time manually oct sunday nightshift attached screenshot error try enter time manually let know required info contact detail name vivek thakkar email address ein manager vishal radia department workplace service vivek thakkar'"
      ]
     },
     "execution_count": 434,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Description'][152]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max length of column Subject: 99\n",
      "\n",
      "Max length of column Description: 2314\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for c in data:\n",
    "    if data[c].dtype == 'object':\n",
    "        print('Max length of column %s: %s\\n' %  (c, data[c].map(len).max()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "927      hr ops treat email level manager approval toge...\n",
       "3949     team currently working team workday timesheets...\n",
       "6744     team request confirm good standing performance...\n",
       "6796     good day sarah ramirez avega managed care woul...\n",
       "7558     background check hr team first advantage condu...\n",
       "7917     chandramouli sneha represent employment screen...\n",
       "8474     represent employment screening practice ofkpmg...\n",
       "8822     ex employee needed get previous work experienc...\n",
       "9175     info consultant greeting verification work exp...\n",
       "9693     hr team hired received following email regardi...\n",
       "9869     concern represent employment screening practic...\n",
       "10164    represent employment screening practice curren...\n",
       "10530    team raising service desk request provide expe...\n",
       "10683    hrd team greeting first advantage private limi...\n",
       "10828    team greeting day represent employment screeni...\n",
       "11603    team greeting day represent employment screeni...\n",
       "Name: Description, dtype: object"
      ]
     },
     "execution_count": 436,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Description'].loc[data[\"Description\"].str.len() > 2000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Keras with Glove!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_NB_WORDS = 14000\n",
    "MAX_SEQUENCE_LENGTH = 100\n",
    "EMBEDDING_DIM = 100\n",
    "VALIDATION_SPLIT = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Text'] = data['Subject'] + data['Description']\n",
    "texts = data['Text']\n",
    "labels = data['KeyCategory']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15767 unique tokens.\n",
      "Shape of data tensor: (9173, 100)\n",
      "Shape of label tensor: (9173, 14)\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "data_num = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "labels = to_categorical(np.asarray(labels))\n",
    "print('Shape of data tensor:', data_num.shape)\n",
    "print('Shape of label tensor:', labels.shape)\n",
    "\n",
    "# split the data into a training set and a validation set\n",
    "\n",
    "indices = np.arange(data_num.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data_num = data_num[indices]\n",
    "labels = labels[indices]\n",
    "nb_validation_samples = int(VALIDATION_SPLIT * data_num.shape[0])\n",
    "\n",
    "x_train = data_num[:-nb_validation_samples]\n",
    "y_train = labels[:-nb_validation_samples]\n",
    "x_val = data_num[-nb_validation_samples:]\n",
    "y_val = labels[-nb_validation_samples:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400000, 100)"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "glove_input_file = 'glove.6B/glove.6B.100d.txt'\n",
    "word2vec_output_file = 'glove.6B/glove.6B.100d.txt.word2vec'\n",
    "glove2word2vec(glove_input_file, word2vec_output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors # load the Stanford GloVe model\n",
    "filename = 'glove.6B/glove.6B.100d.txt.word2vec'\n",
    "model = KeyedVectors.load_word2vec_format(filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if word in model.index2word:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = model[word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Embedding\n",
    "\n",
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 7339 samples, validate on 1834 samples\n",
      "Epoch 1/12\n",
      "7339/7339 [==============================] - 260s 35ms/step - loss: 2.2562 - acc: 0.1928 - val_loss: 2.0577 - val_acc: 0.2437\n",
      "Epoch 2/12\n",
      "7339/7339 [==============================] - 258s 35ms/step - loss: 2.0473 - acc: 0.2376 - val_loss: 2.1041 - val_acc: 0.1816\n",
      "Epoch 3/12\n",
      "7339/7339 [==============================] - 260s 35ms/step - loss: 1.9928 - acc: 0.2665 - val_loss: 2.0308 - val_acc: 0.2557\n",
      "Epoch 4/12\n",
      "7339/7339 [==============================] - 260s 35ms/step - loss: 1.9375 - acc: 0.2803 - val_loss: 1.9337 - val_acc: 0.3015\n",
      "Epoch 5/12\n",
      "3584/7339 [=============>................] - ETA: 2:10 - loss: 1.9189 - acc: 0.3147"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-444-e5d76b3064c2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;31m# happy learning! (gotta increase this for SURE)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m model.fit(x_train, y_train, validation_data=(x_val, y_val),\n\u001b[1;32m---> 21\u001b[1;33m           epochs=12, batch_size=512)\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1040\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1042\u001b[1;33m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1043\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1044\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\keras\\engine\\training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[1;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m                 \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2659\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2660\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2661\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2662\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2663\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2629\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2630\u001b[0m                                 session)\n\u001b[1;32m-> 2631\u001b[1;33m         \u001b[0mfetched\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2632\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2633\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m   1449\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1450\u001b[0m           return tf_session.TF_SessionRunCallable(\n\u001b[1;32m-> 1451\u001b[1;33m               self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[0;32m   1452\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1453\u001b[0m           return tf_session.TF_DeprecatedSessionRunCallable(\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "data_format = 'channels_first'\n",
    "x = Conv1D(128, 5, activation='relu', data_format = 'channels_first')(embedded_sequences)\n",
    "x = MaxPooling1D(5)(x)\n",
    "x = Conv1D(128, 5, activation='relu', data_format = 'channels_first')(x)\n",
    "x = MaxPooling1D(5)(x)\n",
    "x = Conv1D(128, 5, activation='relu', data_format = 'channels_first')(x)\n",
    "x = MaxPooling1D(35)(x)  # global max pooling\n",
    "x = Flatten()(x)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "preds = Dense(len(labels[0]), activation='softmax')(x)\n",
    "\n",
    "model = Model(sequence_input, preds)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "\n",
    "# happy learning! (gotta increase this for SURE)\n",
    "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=12, batch_size=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using tf-idf to do a basic model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "# my split into training and testing datasets\n",
    "# train_x, test_x, train_y, test_y = model_selection.train_test_split(data[['Subject', 'Description']], data['Category'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "throw the subject and description together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['Text'] = data['Subject'] + data['Description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "msg_train, msg_test, label_train, label_test = \\\n",
    "model_selection.train_test_split(data[\"Text\"], data['KeyCategory'], test_size=0.2)\n",
    "bow_transformer = CountVectorizer(max_features=3000).fit(msg_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages_bow = bow_transformer.transform(msg_train)\n",
    "tfidf_transformer = TfidfTransformer().fit(messages_bow)\n",
    "messages_tfidf = tfidf_transformer.transform(messages_bow)\n",
    "# %time testmodel = GradientBoostingClassifier(learning_rate=0.01,max_depth=20, n_estimators=100,verbose=3).fit(messages_tfidf, label_train)\n",
    "# testmodel = RandomForestClassifier(n_estimators=100,verbose=1,n_jobs=2).fit(messages_tfidf, label_train)\n",
    "testmodel = LinearSVC().fit(messages_tfidf, label_train)\n",
    "\n",
    "messages_bow = bow_transformer.transform(msg_test)\n",
    "messages_tfidf = tfidf_transformer.transform(messages_bow)\n",
    "\n",
    "all_predictions = testmodel.predict(messages_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        1.0       0.30      0.25      0.27        79\n",
      "        2.0       0.53      0.54      0.54       334\n",
      "        3.0       1.00      0.29      0.44         7\n",
      "        4.0       0.68      0.73      0.70       281\n",
      "        5.0       0.72      0.65      0.69       107\n",
      "        6.0       0.73      0.80      0.76       390\n",
      "        7.0       0.53      0.32      0.40        25\n",
      "        8.0       0.81      0.75      0.78       112\n",
      "       10.0       0.38      0.20      0.26        71\n",
      "       11.0       0.60      0.43      0.50         7\n",
      "       12.0       0.65      0.70      0.67       388\n",
      "       13.0       0.77      0.29      0.43        34\n",
      "\n",
      "avg / total       0.64      0.64      0.64      1835\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print (classification_report(label_test, all_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}