LDA-Cosine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Work\\Anaconda\\envs\\py35\\lib\\site-packages\\gensim\\utils.py:855: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "# imports:\n",
    "import time;\n",
    "import urllib.request\n",
    "import csv\n",
    "import os\n",
    "import itertools\n",
    "import logging\n",
    "import json\n",
    "import numpy as np\n",
    "import gensim\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlencode\n",
    "from gensim.utils import smart_open, simple_preprocess\n",
    "from gensim.corpora.wikicorpus import _extract_pages, filter_wiki\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "from xml.etree.cElementTree import iterparse\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)\n",
    "logging.root.level = logging.INFO\n",
    "model_name=\"LDA-Cosine\"\n",
    "id2word_wiki=gensim.corpora.Dictionary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking folder : MatMusTec\n"
     ]
    }
   ],
   "source": [
    "# List Categories:\n",
    "category_list = [\"Mathematics\",\"Technology\",\"Music\"]\n",
    "#category_list = [\"Mathematics\",\"Technology\",\"Music\",\"History\",\"Geography\",\"Arts\",\"Health\",\"Nature\",\"Religion\",\"Literature\"]\n",
    "#category_list = [\"1977 introductions\", \"Language\" ,\"Arts\", \"Asia\", \"Asthma\", \"Automobiles\", \"BRICS nation\", \"Belief\", \"Climate\", \"Crime\", \"Culture\", \"Dance\", \"Deserts\", \"Dolls\", \"Earth\", \"Engines\", \"Fishing\", \"Folklore\", \"Games\", \"Geography\", \"Glass\", \"Government agencies\", \"Health\", \"History\", \"Humans\", \"Hygiene\", \"India\", \"Industry\", \"Internet\", \"Law\", \"Life\", \"Literature\", \"Mathematics\", \"Matter\", \"Millionaires\", \"Music\", \"Nature\", \"Nothing\", \"Parties\", \"Peace\", \"Politics\", \"Religion\", \"Sexology\", \"Society\", \"Songs\", \"Space\", \"Technology\", \"Television\", \"Transport\", \"Water sports\"]\n",
    "test_folder =  \"./test/\"\n",
    "root_folder = \"./Simplex/\"\n",
    "list.sort(category_list)\n",
    "folder_name=''.join([x[0]+x[1]+x[2] for x in category_list])\n",
    "root_folder='./'+folder_name+'/'\n",
    "\n",
    "print(\"Checking folder : \"+folder_name)\n",
    "if not os.path.exists(root_folder):\n",
    "    os.mkdir(root_folder)\n",
    "if not os.path.exists(root_folder+model_name):\n",
    "    os.mkdir(root_folder+model_name)\n",
    "\n",
    "    \n",
    "\n",
    "wiki_bow_path = root_folder+'wiki_bow.mm'\n",
    "wiki_dict_path = root_folder+'wiki_dict.dict'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scanning CSV\n"
     ]
    }
   ],
   "source": [
    "# Download Page Ids:\n",
    "#https://petscan.wmflabs.org/?language=en&project=wikipedia&depth=1&format=csv&categories=mathematics&doit=Do it!\n",
    "print (\"Scanning CSV\")\n",
    "for cat in category_list:\n",
    "    if not os.path.exists(root_folder+cat+\".csv\"):\n",
    "        url=\"https://petscan.wmflabs.org/?language=en&project=wikipedia&depth=1&format=csv&doit=Do%20it!&categories=\"+cat\n",
    "        urllib.request.urlretrieve(url, root_folder+cat+\".csv\")\n",
    "        print(\"Downloading \",cat+\".csv\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done downloading\n"
     ]
    }
   ],
   "source": [
    "# CSV to XML Download data:\n",
    "\n",
    "def get_data(ids,output_file):\n",
    "    url=\"https://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&pageids=\"+ids\n",
    "    req = urllib.request.urlopen(url)\n",
    "    if req.getcode() == 200:\n",
    "        soup = BeautifulSoup(req.read(), 'html.parser')\n",
    "        s = soup.find_all('page')\n",
    "        for si in s:\n",
    "            output_file.write(str(si))\n",
    "\n",
    "def batch_train(file):\n",
    "    output_file = open(file.replace(\".csv\",\".xml\"), 'a', encoding=\"utf8\")\n",
    "    output_file.write(\"<pages>\")\n",
    "                    \n",
    "    csvReader = csv.reader(open(file,'r'))\n",
    "    totalRecords = sum(1 for row in csv.reader(open(file,'r',encoding=\"UTF-8\")) )\n",
    "    print (file.replace(\".csv\",\".xml\"),totalRecords)\n",
    "    start = 0\n",
    "    end = start + 50\n",
    "   \n",
    "    while (start <= totalRecords):\n",
    "        pageIds = \"\"\n",
    "        for row in itertools.islice(csv.reader(open(file,'r',encoding=\"UTF-8\")),start,end):\n",
    "            pageIds = pageIds + row[2] + \"|\"\n",
    "        \n",
    "        get_data(pageIds,output_file)\n",
    "        start = end + 1\n",
    "        end = start + 50\n",
    "        if end> totalRecords:\n",
    "            end=totalRecords\n",
    "            \n",
    "    output_file.write(\"</pages>\")\n",
    "    \n",
    "\n",
    "for path, subdirs, files in os.walk(root_folder):\n",
    "    for file in files:\n",
    "        if file.endswith('.csv'):\n",
    "            if not os.path.exists(root_folder+file.replace(\".csv\",\".xml\")):\n",
    "                print(\"Downloading \",root_folder+file.replace(\".csv\",\".xml\"))\n",
    "                batch_train(path + file)\n",
    "\n",
    "print (\"Done downloading\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train Model\n",
    "def head(stream, n=10):\n",
    "    \"\"\"Convenience fnc: return the first `n` elements of the stream, as plain list.\"\"\"\n",
    "    return list(itertools.islice(stream, n))\n",
    "\n",
    "def my_extract_pages(f):\n",
    "    elems = (elem for _, elem in iterparse(f, events=(\"end\",)))\n",
    "    page_tag = \"rev\"\n",
    "    for elem in elems:\n",
    "        if elem.tag == page_tag and elem.text != None:\n",
    "            text = elem.text\n",
    "            yield text\n",
    "            elem.clear()\n",
    "\n",
    "def tokenize(text):\n",
    "    return [token for token in simple_preprocess(text) if token not in STOPWORDS]\n",
    "\n",
    "def iter_wiki(dump_file):\n",
    "    ignore_namespaces = 'Wikipedia Category File Portal Template MediaWiki User Help Book Draft'.split()\n",
    "    for text in my_extract_pages(smart_open(dump_file)):\n",
    "        text = filter_wiki(text)\n",
    "        tokens = tokenize(text)\n",
    "        if len(tokens) < 50:\n",
    "            continue  # ignore short articles and various meta-articles\n",
    "        yield tokens\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : loading Dictionary object from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_dict.dict\n",
      "INFO : loaded ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_dict.dict\n",
      "INFO : loaded corpus index from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_bow.mm.index\n",
      "INFO : initializing corpus reader from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_bow.mm\n",
      "INFO : accepted corpus with 1968 documents, 37814 features, 419342 non-zero entries\n",
      "INFO : using asymmetric alpha [0.20349776650601445, 0.15460680268266819, 0.12465746387131386, 0.10442834231175278, 0.089848016434062955, 0.078840302634700543, 0.070235422324486096, 0.063324036444942805, 0.05765099744942407, 0.052910849340634232]\n",
      "INFO : using symmetric eta at 2.6445231924683978e-05\n",
      "INFO : using serial LDA version on this node\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : running online LDA training, 10 topics, 10 passes over the supplied corpus of 1968 documents, updating model once every 1968 documents, evaluating perplexity every 1968 documents, iterating 50x with a convergence threshold of 0.001000\n",
      "INFO : -11.625 per-word bound, 3157.8 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 0, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.005*\"radar\" + 0.003*\"energy\" + 0.002*\"nuclear\" + 0.002*\"observatory\" + 0.002*\"electric\" + 0.002*\"sensor\" + 0.002*\"gas\" + 0.002*\"medium\" + 0.002*\"communication\" + 0.002*\"machine\"\n",
      "INFO : topic #8 (0.058): 0.003*\"software\" + 0.003*\"video\" + 0.003*\"engineering\" + 0.003*\"radio\" + 0.003*\"internet\" + 0.003*\"mobile\" + 0.003*\"content\" + 0.002*\"users\" + 0.002*\"user\" + 0.002*\"air\"\n",
      "INFO : topic #2 (0.125): 0.005*\"mobile\" + 0.004*\"engineering\" + 0.004*\"digital\" + 0.003*\"software\" + 0.003*\"user\" + 0.002*\"video\" + 0.002*\"technical\" + 0.002*\"devices\" + 0.002*\"observatory\" + 0.002*\"users\"\n",
      "INFO : topic #1 (0.155): 0.006*\"canon\" + 0.003*\"technical\" + 0.003*\"mobile\" + 0.003*\"technological\" + 0.003*\"technologies\" + 0.003*\"digital\" + 0.002*\"software\" + 0.002*\"gas\" + 0.002*\"engineering\" + 0.002*\"communication\"\n",
      "INFO : topic #0 (0.203): 0.004*\"mobile\" + 0.003*\"digital\" + 0.003*\"web\" + 0.003*\"technologies\" + 0.003*\"user\" + 0.002*\"engineering\" + 0.002*\"materials\" + 0.002*\"technological\" + 0.002*\"yes\" + 0.002*\"internet\"\n",
      "INFO : topic diff=4.491186, rho=1.000000\n",
      "INFO : -9.429 per-word bound, 689.4 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 1, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.015*\"radar\" + 0.005*\"sensor\" + 0.004*\"electric\" + 0.003*\"air\" + 0.003*\"energy\" + 0.003*\"slide\" + 0.002*\"medium\" + 0.002*\"internal\" + 0.002*\"combustion\" + 0.002*\"military\"\n",
      "INFO : topic #8 (0.058): 0.004*\"video\" + 0.004*\"radio\" + 0.003*\"content\" + 0.003*\"software\" + 0.003*\"mobile\" + 0.003*\"user\" + 0.002*\"users\" + 0.002*\"lte\" + 0.002*\"internet\" + 0.002*\"device\"\n",
      "INFO : topic #2 (0.125): 0.007*\"mobile\" + 0.005*\"engineering\" + 0.004*\"software\" + 0.003*\"digital\" + 0.003*\"user\" + 0.002*\"library\" + 0.002*\"devices\" + 0.002*\"device\" + 0.002*\"users\" + 0.002*\"applications\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.005*\"technological\" + 0.005*\"technical\" + 0.003*\"gas\" + 0.003*\"technologies\" + 0.002*\"paper\" + 0.002*\"communication\" + 0.002*\"revolution\" + 0.002*\"file\" + 0.002*\"engineering\"\n",
      "INFO : topic #0 (0.203): 0.005*\"mobile\" + 0.004*\"web\" + 0.004*\"user\" + 0.003*\"technologies\" + 0.003*\"digital\" + 0.003*\"message\" + 0.003*\"iso\" + 0.002*\"materials\" + 0.002*\"mm\" + 0.002*\"product\"\n",
      "INFO : topic diff=0.818718, rho=0.577350\n",
      "INFO : -9.186 per-word bound, 582.5 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 2, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.018*\"radar\" + 0.006*\"sensor\" + 0.004*\"electric\" + 0.004*\"air\" + 0.004*\"slide\" + 0.003*\"internal\" + 0.003*\"military\" + 0.003*\"combustion\" + 0.003*\"medium\" + 0.003*\"energy\"\n",
      "INFO : topic #8 (0.058): 0.005*\"video\" + 0.004*\"radio\" + 0.003*\"content\" + 0.003*\"software\" + 0.003*\"lte\" + 0.003*\"mobile\" + 0.003*\"user\" + 0.003*\"device\" + 0.003*\"signal\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.009*\"mobile\" + 0.005*\"engineering\" + 0.005*\"software\" + 0.003*\"digital\" + 0.003*\"user\" + 0.003*\"library\" + 0.002*\"users\" + 0.002*\"device\" + 0.002*\"devices\" + 0.002*\"applications\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.005*\"technical\" + 0.004*\"gas\" + 0.003*\"technologies\" + 0.003*\"revolution\" + 0.003*\"industrial\" + 0.003*\"paper\" + 0.002*\"communication\" + 0.002*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.005*\"mobile\" + 0.004*\"user\" + 0.004*\"web\" + 0.004*\"message\" + 0.003*\"technologies\" + 0.003*\"iso\" + 0.003*\"digital\" + 0.002*\"materials\" + 0.002*\"users\" + 0.002*\"messages\"\n",
      "INFO : topic diff=0.654460, rho=0.500000\n",
      "INFO : -9.072 per-word bound, 538.3 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 3, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.020*\"radar\" + 0.007*\"sensor\" + 0.005*\"electric\" + 0.005*\"air\" + 0.004*\"slide\" + 0.004*\"military\" + 0.003*\"combustion\" + 0.003*\"internal\" + 0.003*\"aircraft\" + 0.003*\"medium\"\n",
      "INFO : topic #8 (0.058): 0.006*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"content\" + 0.003*\"mobile\" + 0.003*\"software\" + 0.003*\"signal\" + 0.003*\"user\" + 0.003*\"device\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.010*\"mobile\" + 0.005*\"software\" + 0.005*\"engineering\" + 0.003*\"digital\" + 0.003*\"user\" + 0.003*\"library\" + 0.003*\"users\" + 0.003*\"device\" + 0.002*\"applications\" + 0.002*\"devices\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.005*\"technical\" + 0.004*\"gas\" + 0.004*\"industrial\" + 0.003*\"revolution\" + 0.003*\"technologies\" + 0.003*\"paper\" + 0.002*\"bc\" + 0.002*\"machine\"\n",
      "INFO : topic #0 (0.203): 0.006*\"mobile\" + 0.005*\"user\" + 0.004*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"technologies\" + 0.003*\"messages\" + 0.002*\"users\" + 0.002*\"digital\" + 0.002*\"product\"\n",
      "INFO : topic diff=0.507571, rho=0.447214\n",
      "INFO : -9.010 per-word bound, 515.5 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 4, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.022*\"radar\" + 0.007*\"sensor\" + 0.005*\"air\" + 0.005*\"electric\" + 0.004*\"slide\" + 0.004*\"military\" + 0.004*\"aircraft\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.003*\"medium\"\n",
      "INFO : topic #8 (0.058): 0.006*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"content\" + 0.004*\"mobile\" + 0.003*\"software\" + 0.003*\"signal\" + 0.003*\"device\" + 0.003*\"invented\" + 0.003*\"user\"\n",
      "INFO : topic #2 (0.125): 0.011*\"mobile\" + 0.006*\"software\" + 0.005*\"engineering\" + 0.003*\"digital\" + 0.003*\"library\" + 0.003*\"user\" + 0.003*\"users\" + 0.003*\"device\" + 0.003*\"applications\" + 0.002*\"architecture\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.006*\"technical\" + 0.004*\"gas\" + 0.004*\"industrial\" + 0.004*\"revolution\" + 0.003*\"paper\" + 0.003*\"technologies\" + 0.003*\"machine\" + 0.002*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.006*\"mobile\" + 0.005*\"user\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"technologies\" + 0.003*\"messages\" + 0.003*\"users\" + 0.003*\"product\" + 0.002*\"digital\"\n",
      "INFO : topic diff=0.396165, rho=0.408248\n",
      "INFO : -8.970 per-word bound, 501.6 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 5, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.022*\"radar\" + 0.008*\"sensor\" + 0.006*\"air\" + 0.006*\"electric\" + 0.004*\"slide\" + 0.004*\"aircraft\" + 0.004*\"military\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.003*\"fuel\"\n",
      "INFO : topic #8 (0.058): 0.006*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"mobile\" + 0.004*\"content\" + 0.004*\"signal\" + 0.003*\"software\" + 0.003*\"device\" + 0.003*\"invented\" + 0.003*\"audio\"\n",
      "INFO : topic #2 (0.125): 0.011*\"mobile\" + 0.006*\"software\" + 0.005*\"engineering\" + 0.003*\"library\" + 0.003*\"digital\" + 0.003*\"users\" + 0.003*\"user\" + 0.003*\"applications\" + 0.003*\"device\" + 0.003*\"metadata\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.006*\"technical\" + 0.005*\"gas\" + 0.005*\"industrial\" + 0.004*\"revolution\" + 0.003*\"paper\" + 0.003*\"machine\" + 0.003*\"technologies\" + 0.003*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.006*\"mobile\" + 0.006*\"user\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"technologies\" + 0.003*\"users\" + 0.003*\"messages\" + 0.003*\"product\" + 0.002*\"specification\"\n",
      "INFO : topic diff=0.312289, rho=0.377964\n",
      "INFO : -8.943 per-word bound, 492.3 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 6, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.023*\"radar\" + 0.008*\"sensor\" + 0.006*\"air\" + 0.006*\"electric\" + 0.005*\"aircraft\" + 0.005*\"slide\" + 0.005*\"military\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.003*\"car\"\n",
      "INFO : topic #8 (0.058): 0.007*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"mobile\" + 0.004*\"signal\" + 0.004*\"content\" + 0.003*\"software\" + 0.003*\"device\" + 0.003*\"audio\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.012*\"mobile\" + 0.006*\"software\" + 0.004*\"engineering\" + 0.003*\"library\" + 0.003*\"users\" + 0.003*\"digital\" + 0.003*\"companies\" + 0.003*\"metadata\" + 0.003*\"products\" + 0.003*\"user\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.006*\"technical\" + 0.005*\"industrial\" + 0.005*\"gas\" + 0.004*\"revolution\" + 0.003*\"paper\" + 0.003*\"machine\" + 0.003*\"bc\" + 0.003*\"iron\"\n",
      "INFO : topic #0 (0.203): 0.006*\"user\" + 0.006*\"mobile\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"users\" + 0.003*\"technologies\" + 0.003*\"messages\" + 0.003*\"product\" + 0.002*\"specification\"\n",
      "INFO : topic diff=0.248609, rho=0.353553\n",
      "INFO : -8.924 per-word bound, 485.6 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 7, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.023*\"radar\" + 0.008*\"sensor\" + 0.007*\"air\" + 0.006*\"electric\" + 0.005*\"aircraft\" + 0.005*\"military\" + 0.005*\"slide\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.003*\"car\"\n",
      "INFO : topic #8 (0.058): 0.007*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"mobile\" + 0.004*\"signal\" + 0.004*\"content\" + 0.004*\"device\" + 0.003*\"software\" + 0.003*\"audio\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.012*\"mobile\" + 0.007*\"software\" + 0.004*\"engineering\" + 0.003*\"library\" + 0.003*\"users\" + 0.003*\"companies\" + 0.003*\"metadata\" + 0.003*\"digital\" + 0.003*\"products\" + 0.003*\"business\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.005*\"technical\" + 0.005*\"industrial\" + 0.005*\"gas\" + 0.004*\"revolution\" + 0.004*\"paper\" + 0.003*\"machine\" + 0.003*\"iron\" + 0.003*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.006*\"user\" + 0.006*\"mobile\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"users\" + 0.003*\"messages\" + 0.003*\"product\" + 0.003*\"technologies\" + 0.003*\"application\"\n",
      "INFO : topic diff=0.200314, rho=0.333333\n",
      "INFO : -8.908 per-word bound, 480.5 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 8, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.024*\"radar\" + 0.008*\"sensor\" + 0.007*\"air\" + 0.006*\"electric\" + 0.006*\"aircraft\" + 0.005*\"military\" + 0.005*\"slide\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.003*\"car\"\n",
      "INFO : topic #8 (0.058): 0.007*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"signal\" + 0.004*\"mobile\" + 0.004*\"content\" + 0.004*\"device\" + 0.003*\"audio\" + 0.003*\"software\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.013*\"mobile\" + 0.007*\"software\" + 0.004*\"engineering\" + 0.003*\"library\" + 0.003*\"companies\" + 0.003*\"users\" + 0.003*\"business\" + 0.003*\"metadata\" + 0.003*\"products\" + 0.003*\"digital\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.005*\"technical\" + 0.005*\"industrial\" + 0.005*\"gas\" + 0.004*\"revolution\" + 0.004*\"paper\" + 0.003*\"machine\" + 0.003*\"iron\" + 0.003*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.007*\"user\" + 0.006*\"mobile\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"users\" + 0.003*\"messages\" + 0.003*\"product\" + 0.003*\"technologies\" + 0.003*\"application\"\n",
      "INFO : topic diff=0.163598, rho=0.316228\n",
      "INFO : -8.896 per-word bound, 476.5 perplexity estimate based on a held-out corpus of 1968 documents with 804398 words\n",
      "INFO : PROGRESS: pass 9, at document #1968/1968\n",
      "INFO : topic #9 (0.053): 0.024*\"radar\" + 0.008*\"sensor\" + 0.007*\"air\" + 0.007*\"electric\" + 0.006*\"aircraft\" + 0.005*\"military\" + 0.005*\"slide\" + 0.004*\"combustion\" + 0.004*\"internal\" + 0.004*\"car\"\n",
      "INFO : topic #8 (0.058): 0.007*\"video\" + 0.005*\"radio\" + 0.004*\"lte\" + 0.004*\"signal\" + 0.004*\"mobile\" + 0.004*\"device\" + 0.004*\"audio\" + 0.004*\"content\" + 0.003*\"software\" + 0.003*\"invented\"\n",
      "INFO : topic #2 (0.125): 0.013*\"mobile\" + 0.007*\"software\" + 0.004*\"engineering\" + 0.003*\"companies\" + 0.003*\"library\" + 0.003*\"business\" + 0.003*\"users\" + 0.003*\"metadata\" + 0.003*\"products\" + 0.003*\"million\"\n",
      "INFO : topic #1 (0.155): 0.008*\"canon\" + 0.006*\"technological\" + 0.005*\"industrial\" + 0.005*\"technical\" + 0.005*\"gas\" + 0.004*\"revolution\" + 0.004*\"paper\" + 0.004*\"machine\" + 0.003*\"iron\" + 0.003*\"bc\"\n",
      "INFO : topic #0 (0.203): 0.007*\"user\" + 0.006*\"mobile\" + 0.005*\"message\" + 0.004*\"web\" + 0.003*\"iso\" + 0.003*\"users\" + 0.003*\"messages\" + 0.003*\"product\" + 0.003*\"technical\" + 0.003*\"technologies\"\n",
      "INFO : topic diff=0.135464, rho=0.301511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4min 12s\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Model\")\n",
    "class WikiCorpus(object):\n",
    "    def __init__(self, dump_file, dictionary, clip_docs=None):\n",
    "        \"\"\"\n",
    "        Parse the first `clip_docs` Wikipedia documents from file `dump_file`.\n",
    "        Yield each document in turn, as a list of tokens (unicode strings).\n",
    "        \n",
    "        \"\"\"\n",
    "        self.dump_file = dump_file\n",
    "        self.dictionary = dictionary\n",
    "        self.clip_docs = clip_docs\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for tokens in itertools.islice(iter_wiki(self.dump_file), self.clip_docs):\n",
    "            yield self.dictionary.doc2bow(tokens)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.clip_docs\n",
    "\n",
    "\n",
    "if not (os.path.exists(wiki_bow_path) and  os.path.exists(wiki_dict_path)):\n",
    "    for path, subdirs, files in os.walk(root_folder):\n",
    "        del subdirs[:]\n",
    "        for file in files:\n",
    "            if file.endswith('.xml'):\n",
    "                doc_path = path + file\n",
    "                stream = iter_wiki(doc_path)\n",
    "                doc_stream = (tokens for tokens in iter_wiki(doc_path))\n",
    "                id2word_wiki.merge_with(gensim.corpora.Dictionary(doc_stream))\n",
    "\n",
    "    id2word_wiki.filter_extremes(no_below=10, no_above=0.1)\n",
    "    \n",
    "    # create a stream of bag-of-words vectors\n",
    "    wiki_corpus = WikiCorpus(doc_path, id2word_wiki)\n",
    "    \n",
    "    id2word_wiki.save(wiki_dict_path) \n",
    "    gensim.corpora.MmCorpus.serialize(wiki_bow_path, wiki_corpus)\n",
    "\n",
    "id2word_wiki = gensim.corpora.Dictionary.load(wiki_dict_path)\n",
    "mm_corpus = gensim.corpora.MmCorpus(wiki_bow_path)\n",
    "clipped_corpus = gensim.utils.ClippedCorpus(mm_corpus, 4000) \n",
    "lda_model = gensim.models.LdaModel(clipped_corpus, num_topics=len(category_list), id2word=id2word_wiki, passes=10, alpha='asymmetric')\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calcuating centroid\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Arts.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Geography.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Health.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/History.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Literature.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Mathematics.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Music.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Nature.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Religion.xml\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Technology.xml\n",
      "Centroid calculation done\n"
     ]
    }
   ],
   "source": [
    "def calculate_centroid(topic_docs):\n",
    "    test_doc = [tokens for tokens in iter_wiki(topic_docs)]\n",
    "    part = [lda_model[id2word_wiki.doc2bow(tokens)] for tokens in test_doc]\n",
    "    \n",
    "    topic_dic={}\n",
    "    \n",
    "    for i in range(len(category_list)):\n",
    "        topic_dic[i]=0\n",
    "        \n",
    "    for doc in part:\n",
    "        for p in doc:\n",
    "            topic_dic[p[0]] += p[1]\n",
    "    \n",
    "    centroid = [(x, topic_dic[x]/len(part)) for x in range(len(category_list))]\n",
    "    return centroid\n",
    "\n",
    "print(\"Calcuating centroid\")\n",
    "\n",
    "centroids_dict={}\n",
    "for path, subdirs, files in os.walk(root_folder):\n",
    "    del subdirs[:]\n",
    "    for file in files:\n",
    "        if file.endswith('.xml'):\n",
    "            doc_path = path + file\n",
    "            centroid = calculate_centroid(doc_path)\n",
    "            centroids_dict[file.replace(\".xml\",\"\")]=centroid\n",
    "\n",
    "print(\"Centroid calculation done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def draw_graph(x_label,y,file,text_data,topic):\n",
    "    x = np.arange(len(x_label))  # the x locations for the groups\n",
    "    width = 0.1       # the width of the bars\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    fig.set_figheight(6)\n",
    "    fig.set_figwidth(8)\n",
    "    rects = ax.bar(x, y, width, color='blue')\n",
    "    ax.set_ylabel('Probabilities')\n",
    "    ax.set_xlabel('Categories')\n",
    "    ax.set_title('Topic Distribution of: ' + topic)\n",
    "    ax.title.set_position([.5, 1.2])\n",
    "    ax.set_ylim([0,1])\n",
    "    \n",
    "    def autolabel(rects):\n",
    "        \"\"\"\n",
    "        Attach a text label above each bar displaying its height\n",
    "        \"\"\"\n",
    "        for rect in rects:\n",
    "            height = rect.get_height()\n",
    "            ax.text(rect.get_x() - rect.get_width()*2, 1.05 * height,round(height,2))\n",
    "    \n",
    "\n",
    "    autolabel(rects)\n",
    "    plt.tight_layout(pad=6)\n",
    "    plt.xticks(x,category_list,rotation=90)\n",
    "    \n",
    "    plt.savefig(file.replace(\".xml\",\"_\")+str(len(category_list))+'.png')\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing results\n",
      " Done \n"
     ]
    }
   ],
   "source": [
    "# Test \n",
    "def get_part(testFile):\n",
    "    test_doc = [tokens for tokens in iter_wiki(testFile)]\n",
    "    part = [lda_model[id2word_wiki.doc2bow(tokens)] for tokens in test_doc]\n",
    "    return part\n",
    "\n",
    "results=\"\"\n",
    "print(\"Testing results\")\n",
    "for path, subdirs, files in os.walk(test_folder):\n",
    "    for file in files:\n",
    "        if file.endswith('.xml'):\n",
    "            doc_path = test_folder + file\n",
    "            path=get_part(doc_path)\n",
    "            results+=file.replace(\".xml\",\"\")+\"\\n\\n\"\n",
    "            graph_data=[]\n",
    "            text_data=\"\"\n",
    "            for topic in category_list:\n",
    "                \n",
    "                cos_dis=np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip([centroids_dict[topic]], path)])\n",
    "                text_data +=topic+\":\"+str(cos_dis)+\"\\n\"\n",
    "                \n",
    "                graph_data.append(np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip([centroids_dict[topic]], path)]))\n",
    "            results+=text_data+\"\\n\\n\"\n",
    "                \n",
    "            draw_graph(list(centroids_dict.keys()),graph_data,root_folder+model_name+'/'+file,text_data,file)\n",
    "output_file = open(root_folder+model_name+\"/\"+model_name+str(len(category_list))+\".txt\", 'w', encoding=\"utf8\")\n",
    "output_file.write(results)\n",
    "output_file.close()\n",
    "print(\" Done \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}