02 Topic Modelling - Final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic modelling, with manual filtering of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import gensim\n",
    "import numpy as np\n",
    "import spacy\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "from gensim.models import CoherenceModel, LdaModel, LsiModel, HdpModel\n",
    "from gensim.models.wrappers import LdaMallet\n",
    "from gensim.corpora import Dictionary\n",
    "import pyLDAvis.gensim\n",
    "\n",
    "import os, re, operator, warnings\n",
    "warnings.filterwarnings('ignore')  # Let's not pay heed to them right now\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['barbara', 'snowden', 'iowa', 'emanuel', 'goldman', 'bring', 'america', 'like', 'chrysler', 'jerry', 'voter', 'political', 'american', 'president', 'poll', 'independent', 'democratic', 'campaign', 'progressive', 'want', 'socialist', 'thing', 'think', 'vote', 'party', 'debate', 'primary', 'nomination', 'presidential', 'caucus', 'history', 'candidate', 'bernie', 'sander', 'hillary', 'clinton', 'barack', 'obama', 'donald', 'trump', 'republican', 'democrat', 'republican', 'democrat', 'romney']\n"
     ]
    }
   ],
   "source": [
    "my_stop_words = [u'say', u'\\'s', u'Mr', u'be', u'said', u'says', u'saying']\n",
    "for stopword in my_stop_words:\n",
    "    lexeme = nlp.vocab[stopword]\n",
    "    lexeme.is_stop = True\n",
    "\n",
    "filtered = ['barbara','snowden','iowa','emanuel','goldman','bring','america','like','chrysler','jerry','voter','political','american','president','poll','independent','democratic','campaign','progressive','want','socialist','thing','think','vote','party', 'debate', 'primary', 'nomination', 'presidential', 'caucus', 'history', 'candidate','bernie','sanders','hillary','clinton','barack','obama','donald','trump','republican','democrat','republicans','democrats','romney']\n",
    "filtered = [nlp(w)[0].lemma_ for w in filtered]\n",
    "print(filtered)\n",
    "def is_filtered(w):\n",
    "    return str(w) in filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_parties = pickle.load(open('text_dumps.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = text_parties['republican'] + text_parties['democrat']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68c67b5291e6444992c9b5ff0f9ae7ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1001), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d944f77c8ad456cbccefbe5200afe3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1047), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for party in ['democrat','republican']:\n",
    "    text = text_parties[party]\n",
    "    text = text.lower()\n",
    "    text.split('\\n')[:10]\n",
    "    ### Pre-processing data!\n",
    "\n",
    "    doc = nlp(text)\n",
    "\n",
    "    texts_nlp, article_nlp, skl_texts = [], [], []\n",
    "\n",
    "    for phrase in tqdm(text.split('\\n')):\n",
    "        phrase_nlp = nlp(phrase)\n",
    "        for w in phrase_nlp:\n",
    "            if w.text != '\\n' and not w.is_stop and not w.is_punct and not w.like_num and w.is_alpha:\n",
    "                if not is_filtered(w.lemma_) and len(w.lemma_) > 2:\n",
    "                    article_nlp.append(w.lemma_)\n",
    "            # if it's a new line, it means we're onto our next document\n",
    "        skl_texts.append(' '.join(article_nlp))\n",
    "        texts_nlp.append(article_nlp)\n",
    "        article_nlp = []\n",
    "\n",
    "    texts = texts_nlp\n",
    "\n",
    "    with open('tokenized_dump_%s.pkl' %party,'wb') as f:\n",
    "        pickle.dump(texts,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram = gensim.models.Phrases(texts)\n",
    "\n",
    "texts = [bigram[line] for line in texts]\n",
    "\n",
    "texts[20][0:100]\n",
    "\n",
    "dictionary = Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "\n",
    "corpus[1][0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSI\n",
    "\n",
    "LSI stands for Latent Semantic Indeixing - it is a popular information retreival method which works by decomposing the original matrix of words to maintain key topics. Gensim's implementation uses an SVD."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsimodel = LsiModel(corpus=corpus, num_topics=10, id2word=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.269*\"people\" + 0.218*\"need\" + 0.188*\"support\" + 0.136*\"right\" + 0.123*\"good\" + 0.121*\"new\" + 0.115*\"election\" + 0.115*\"state\" + 0.114*\"high\" + 0.113*\"change\"'),\n",
       " (1,\n",
       "  '0.336*\"people\" + 0.178*\"kid\" + 0.162*\"gay\" + 0.159*\"sex_marriage\" + 0.141*\"property\" + 0.138*\"movement\" + 0.136*\"gay_right\" + 0.136*\"gay_sex\" + -0.133*\"israel\" + -0.130*\"need\"'),\n",
       " (2,\n",
       "  '-0.613*\"state\" + -0.171*\"split\" + -0.146*\"unemployment_rate\" + -0.130*\"national\" + -0.128*\"unemployment\" + 0.127*\"people\" + -0.119*\"average\" + -0.102*\"large\" + -0.097*\"unicameral\" + -0.095*\"tax\"'),\n",
       " (3,\n",
       "  '0.542*\"state\" + 0.171*\"split\" + 0.145*\"unemployment_rate\" + -0.134*\"money\" + -0.129*\"year\" + -0.121*\"supporter\" + -0.121*\"favorability\" + -0.121*\"general_election\" + 0.121*\"unemployment\" + -0.113*\"tax\"'),\n",
       " (4,\n",
       "  '-0.220*\"government\" + -0.202*\"right\" + -0.200*\"tax\" + -0.154*\"spouse\" + -0.147*\"federal\" + -0.140*\"work\" + -0.114*\"post\" + 0.111*\"general_election\" + 0.111*\"favorability\" + -0.110*\"country\"'),\n",
       " (5,\n",
       "  '0.334*\"right\" + 0.307*\"spouse\" + 0.197*\"support\" + -0.158*\"people\" + 0.157*\"federal\" + 0.155*\"deny\" + 0.154*\"homosexual\" + 0.154*\"property\" + 0.154*\"joint\" + 0.124*\"tax\"'),\n",
       " (6,\n",
       "  '-0.238*\"talk\" + -0.194*\"fact\" + -0.176*\"feel\" + -0.159*\"medium\" + -0.145*\"subject\" + -0.142*\"find\" + -0.137*\"topic\" + -0.126*\"outside\" + -0.124*\"conversation\" + -0.124*\"reality\"'),\n",
       " (7,\n",
       "  '-0.234*\"win\" + 0.194*\"tax\" + -0.178*\"point\" + -0.151*\"turn\" + -0.148*\"election\" + -0.144*\"right\" + -0.134*\"gop\" + -0.114*\"war\" + -0.110*\"work\" + -0.110*\"way\"'),\n",
       " (8,\n",
       "  '0.282*\"war\" + 0.235*\"fight\" + 0.152*\"resign\" + 0.135*\"way\" + -0.134*\"win\" + 0.133*\"force\" + -0.126*\"people\" + 0.112*\"tea\" + 0.111*\"email\" + 0.107*\"picture\"'),\n",
       " (9,\n",
       "  '-0.237*\"win\" + -0.192*\"supporter\" + 0.163*\"war\" + 0.150*\"investment\" + 0.146*\"wealthy\" + 0.141*\"feel\" + -0.135*\"time\" + 0.133*\"thank\" + 0.132*\"lot\" + 0.131*\"low\"')]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsimodel.show_topics(num_topics=15)  # Showing only the top 5 topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " '0.336*\"people\" + 0.178*\"kid\" + 0.162*\"gay\" + 0.159*\"sex_marriage\" + 0.141*\"property\" + 0.138*\"movement\" + 0.136*\"gay_right\" + 0.136*\"gay_sex\" + -0.133*\"israel\" + -0.130*\"need\"'),\n",
    " \n",
    " *\"unemployment_rate\" + -0.130*\"national\" + -0.128*\"unemployment\"\n",
    " \n",
    " tax\n",
    " \n",
    " 5*\"unemployment_rate\" + -0.134*\"money\" \n",
    " \n",
    "  0.121*\"unemployment\" + -0.113*\"tax\"'),\n",
    "  \n",
    "  general_election\n",
    "  \n",
    "  homosexual\n",
    "  \n",
    "  \"tax\"\n",
    "  \n",
    "  282*\"war\" \n",
    "  \n",
    "  \" + 0.163*\"war\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HDP\n",
    "\n",
    "HDP, the Hierarchical Dirichlet process is an unsupervised topic model which figures out the number of topics on it's own."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdpmodel = HdpModel(corpus=corpus, id2word=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.001*sandy + 0.001*negate + 0.001*shove + 0.001*share + 0.001*testimony + 0.001*need + 0.001*man + 0.001*upvot + 0.001*slice + 0.001*west + 0.001*running + 0.001*grandchild + 0.001*worship + 0.001*respect + 0.001*quinnipiac + 0.001*adviser + 0.001*past + 0.001*reference + 0.001*win + 0.001*fil'),\n",
       " (1,\n",
       "  '0.002*people + 0.001*forfend + 0.001*wambach + 0.001*gay_right + 0.001*tucker + 0.001*reagan + 0.001*save + 0.001*shapiro + 0.001*ban + 0.001*regardless + 0.001*fdr + 0.001*cousin + 0.001*native + 0.001*institution + 0.001*post + 0.001*eachother + 0.001*flavor + 0.001*sexism + 0.001*relationship + 0.001*rule'),\n",
       " (2,\n",
       "  '0.002*upset + 0.002*shop + 0.002*begin + 0.001*andrew + 0.001*bernard + 0.001*jeanne + 0.001*few + 0.001*art + 0.001*thin + 0.001*disability + 0.001*lapd + 0.001*explain + 0.001*journalist + 0.001*paycheck + 0.001*bialek + 0.001*stick + 0.001*away + 0.001*hobble + 0.001*absence + 0.001*shell'),\n",
       " (3,\n",
       "  '0.002*missing + 0.002*community + 0.001*supporter + 0.001*promotion + 0.001*upvot + 0.001*enlightenment + 0.001*nationally + 0.001*overt + 0.001*scared + 0.001*compensation + 0.001*weekend + 0.001*necessary + 0.001*pakistani + 0.001*sector + 0.001*smithereen + 0.001*denial + 0.001*shelve + 0.001*gentleman + 0.001*wishy + 0.001*brother'),\n",
       " (4,\n",
       "  '0.003*state + 0.002*reckless + 0.002*whoop + 0.001*aoc + 0.001*degree + 0.001*liberty + 0.001*wish + 0.001*aldous + 0.001*quality + 0.001*texan + 0.001*orc + 0.001*demand + 0.001*money + 0.001*huckabee + 0.001*bigot + 0.001*hostile + 0.001*cheap + 0.001*lady + 0.001*horrible + 0.001*false'),\n",
       " (5,\n",
       "  '0.002*entitled + 0.002*buster + 0.002*warp + 0.002*restart + 0.001*bereavement + 0.001*commitment + 0.001*massachusett + 0.001*colleague + 0.001*rake + 0.001*tipping + 0.001*stalinist + 0.001*daniel + 0.001*varied + 0.001*korea + 0.001*custodial + 0.001*sue + 0.001*chairman + 0.001*extremist + 0.001*enlightenment + 0.001*retreat'),\n",
       " (6,\n",
       "  '0.002*tolerant + 0.001*medal + 0.001*twin + 0.001*tax + 0.001*painful + 0.001*powerful + 0.001*popular + 0.001*whoopi + 0.001*stasi + 0.001*paycheck + 0.001*relevant + 0.001*divide + 0.001*multifacet + 0.001*debt + 0.001*flip + 0.001*prepare + 0.001*islamophobe + 0.001*overconfidence + 0.001*pro_life + 0.001*plea'),\n",
       " (7,\n",
       "  '0.002*shall + 0.002*indoctrinate + 0.002*jury + 0.001*care + 0.001*discuss + 0.001*detrimental + 0.001*stormy + 0.001*voting + 0.001*massive + 0.001*gesture + 0.001*embrace + 0.001*flexibility + 0.001*language + 0.001*unfavorable + 0.001*jon + 0.001*unamerican + 0.001*year + 0.001*lavish + 0.001*counter + 0.001*deal'),\n",
       " (8,\n",
       "  '0.003*fume + 0.002*wellstone + 0.001*moment + 0.001*betray + 0.001*universal + 0.001*alienate + 0.001*bachmann + 0.001*tracking + 0.001*grumpy + 0.001*sink + 0.001*ideological + 0.001*unamerican + 0.001*forget + 0.001*november + 0.001*jong + 0.001*august + 0.001*lessig + 0.001*push + 0.001*killer + 0.001*monolithic'),\n",
       " (9,\n",
       "  '0.002*government + 0.002*ruin + 0.002*wrong + 0.002*ban + 0.001*msm + 0.001*regard + 0.001*backstory + 0.001*poor + 0.001*shitstorm + 0.001*welp + 0.001*firewall + 0.001*food + 0.001*mrw + 0.001*thief + 0.001*manhattan + 0.001*gingrich + 0.001*major + 0.001*demographic + 0.001*matter + 0.001*dolore')]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hdpmodel.show_topics(num_topics=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sexism\n",
    "tax\n",
    "pro_life\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA\n",
    "\n",
    "LDA, or Latent Dirichlet Allocation is arguably the most famous topic modelling algorithm out there. Out here we create a simple topic model with 10 topics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldamodel = LdaModel(corpus=corpus, num_topics=10, id2word=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.025*\"support\" + 0.023*\"defeat\" + 0.022*\"predict\" + 0.022*\"whoop\" + 0.022*\"foreign\" + 0.022*\"tactic\" + 0.022*\"hopeful\" + 0.019*\"opponent\" + 0.018*\"belief\" + 0.017*\"lead\"'),\n",
       " (1,\n",
       "  '0.028*\"fund\" + 0.021*\"dollar\" + 0.020*\"ideal\" + 0.020*\"sach\" + 0.020*\"hedge\" + 0.020*\"man\" + 0.019*\"dnc\" + 0.019*\"ban\" + 0.018*\"conservative\" + 0.018*\"rich\"'),\n",
       " (2,\n",
       "  '0.020*\"key\" + 0.016*\"know\" + 0.012*\"need\" + 0.011*\"win\" + 0.011*\"run\" + 0.011*\"left\" + 0.011*\"national\" + 0.010*\"wall_street\" + 0.010*\"play\" + 0.010*\"chair\"'),\n",
       " (3,\n",
       "  '0.024*\"point\" + 0.022*\"half\" + 0.022*\"awkward\" + 0.022*\"breach\" + 0.022*\"increasingly\" + 0.022*\"frontrunner\" + 0.022*\"husband\" + 0.022*\"prove\" + 0.020*\"few\" + 0.015*\"finally\"'),\n",
       " (4,\n",
       "  '0.029*\"trade\" + 0.022*\"fire\" + 0.017*\"support\" + 0.015*\"plan\" + 0.015*\"deal\" + 0.015*\"release\" + 0.015*\"goal\" + 0.015*\"day\" + 0.015*\"policy\" + 0.015*\"cost\"'),\n",
       " (5,\n",
       "  '0.057*\"wall_street\" + 0.019*\"stand\" + 0.019*\"run\" + 0.019*\"stupid\" + 0.019*\"joe_biden\" + 0.019*\"officially\" + 0.019*\"hall\" + 0.019*\"town\" + 0.019*\"announce\" + 0.019*\"new_hampshire\"'),\n",
       " (6,\n",
       "  '0.027*\"plan\" + 0.027*\"gun_control\" + 0.026*\"actually\" + 0.026*\"attack\" + 0.026*\"wall_street\" + 0.025*\"rahm\" + 0.025*\"corrupt\" + 0.025*\"chicago\" + 0.025*\"clear\" + 0.019*\"catch\"'),\n",
       " (7,\n",
       "  '0.033*\"public\" + 0.032*\"back\" + 0.029*\"union\" + 0.025*\"employee\" + 0.020*\"win\" + 0.017*\"need\" + 0.017*\"comment\" + 0.017*\"endorsement\" + 0.016*\"fivethirtyeight\" + 0.016*\"financial\"'),\n",
       " (8,\n",
       "  '0.028*\"time\" + 0.026*\"kasich\" + 0.026*\"endorse\" + 0.025*\"john\" + 0.025*\"new_york\" + 0.025*\"wedding\" + 0.020*\"claim\" + 0.019*\"victory\" + 0.016*\"election\" + 0.005*\"win\"'),\n",
       " (9,\n",
       "  '0.025*\"war\" + 0.016*\"gun\" + 0.016*\"ask\" + 0.016*\"issue\" + 0.016*\"reason\" + 0.016*\"explain\" + 0.016*\"wall_street\" + 0.016*\"tie\" + 0.015*\"stop\" + 0.015*\"voting\"')]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ldamodel.show_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tax\n",
    "gun control\n",
    "gun\n",
    "war"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}