Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tuple"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(('a','b'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'b']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(('a','b'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def f():\n",
    "    n=0\n",
    "    def g():\n",
    "        nonlocal n\n",
    "        n=n+1\n",
    "        print(n)\n",
    "    return g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=f()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "a()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "a()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "a()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tup=('a','b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vending_machine(snacks):\n",
    "    \"\"\"Cycles through sequence of snacks.\n",
    "\n",
    "    >>> vender = vending_machine(('chips', 'chocolate', 'popcorn'))\n",
    "    >>> vender()\n",
    "    'chips'\n",
    "    >>> vender()\n",
    "    'chocolate'\n",
    "    >>> vender()\n",
    "    'popcorn'\n",
    "    >>> vender()\n",
    "    'chips'\n",
    "    >>> other = vending_machine(('brownie',)) #注意，这里的'brownie'后面必须有逗号，不然tuple连接不起来\n",
    "    >>> other()\n",
    "    'brownie'\n",
    "    >>> vender()\n",
    "    'chocolate'\n",
    "    \"\"\"\n",
    "    i=-1\n",
    "    def vend():\n",
    "        nonlocal i\n",
    "        i=(i+1)%len(snacks)\n",
    "        return snacks[i]\n",
    "    return vend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vender = vending_machine(('chips', 'chocolate', 'popcorn'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chips'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vender()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chocolate'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vender()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'popcorn'"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vender()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chips'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vender()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('a', 'a', 'b')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "('a',) + tup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('a', 'b')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "()+tup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('a', 'b', 'c', 'b')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "('a','b','c')+('b',)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=('chips', 'chocolate', 'popcorn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('chips', 'chocolate')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=[1,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a.append([3,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, [3, 4]]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "b=list(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b is a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b == a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a[0]=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 2, [3, 4]]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "a[2][0]=6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 2, [6, 4]]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, [6, 4]]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "a.extend([7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 2, [6, 4], 7]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=a+[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 2, [6, 4], 7, 8]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, [6, 4]]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "b[2][1]=a[2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, [6, [[...], 7, 8]]]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 2, [6, [[...], 7, 8]], 7, 8]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=[1,2,3,1,3,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.pop(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def f(a):\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 1, 3, 1]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(f(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list(a)[2] is a[2]\n"
     ]
    }
   ],
   "source": [
    "a=[1,2,[3,4]]\n",
    "def f(m,s,t):\n",
    "    if m is a:print('list(a) is a')\n",
    "    if s is a[2]:print('list(a)[2] is a[2]')\n",
    "    if t is a[1:]:print('a[1:] is a[1:]')\n",
    "f(list(a),list(a)[2],a[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accumulate(lst):\n",
    "    sum_so_far=0\n",
    "    for i,item in enumerate(lst):\n",
    "        if isinstance(item,list):\n",
    "            sum_so_far=sum_so_far+accumulate(item)\n",
    "        else:\n",
    "            sum_so_far+=item\n",
    "            lst[i]=sum_so_far\n",
    "    return sum_so_far"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ls=[1,2,[3,4],5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accumulate(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, [3, 7], 15]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "21%10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def has_seven(k):\n",
    "    if k%10==7:\n",
    "        return True\n",
    "    elif k<10:\n",
    "        return False\n",
    "    else:\n",
    "        return has_seven(k//10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_pingpong_tracker():\n",
    "    index,current,add=1,0,True\n",
    "    def pingpong_tracker():\n",
    "        nonlocal index,current,add\n",
    "        if add:\n",
    "            current+=1\n",
    "        else:\n",
    "            current-=1\n",
    "        if has_seven(index) or index%7==0:\n",
    "            add=not add\n",
    "        index+=1\n",
    "        return current\n",
    "    return pingpong_tracker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x=make_pingpong_tracker()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5, 6, 7, 6, 5]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x() for _ in range(9)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coding: utf-8\n",
    "import numpy as np\n",
    "import pymongo\n",
    "from pymongo import MongoClient\n",
    "import re\n",
    "#import jieba\n",
    "import gensim\n",
    "import codecs\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "import time \n",
    "\n",
    "# jieba.load_userdict('med_cnDictionary.txt')\n",
    "\n",
    "#client = pymongo.MongoClient('mongodb://sysu:sysu**2016@222.200.166.144:27017')\n",
    "#db = client['wanfang_med']\n",
    "#coll = db['wanfang_year_10k']\n",
    "\n",
    "def token(texts):\n",
    "    pa = re.compile('(\\W)*')\n",
    "    stopwords = codes.open('', 'r', 'encoding=utf-8').readlines()\n",
    "    token_ = []\n",
    "    for doc in texts:\n",
    "        doc = doc.strip()\n",
    "        if len(doc) == 0:   continue\n",
    "        doc = re.sub(pa, '', doc)\n",
    "        words = list(jieba.cut(doc, cut_all=False))\n",
    "        tem = []\n",
    "        for word in words:\n",
    "            if word not in stopwords:\n",
    "                tem.append(word)\n",
    "        token.append(tem)\n",
    "    return token_\n",
    "            \n",
    "def readDb(n=10):\n",
    "    texts = [] \n",
    "    for each in coll.find()[0:n]:\n",
    "        if each['abstract_cn'] == None:\n",
    "            continue\n",
    "        texts.append(each['abstract_cn'].encode('utf-8'))\n",
    "    return texts\n",
    "\n",
    "def lda_model(corpus, k=3):\n",
    "    lda = LdaModel(corpus, num_topics=k)\n",
    "    topics = lda.show_topics()\n",
    "    print('lda', topics)\n",
    "   \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['MapReduce', '迭代', '并行计算', 'iMapReduceHadoopMapReduce', '式', '集中体现', '科研人员', 'Kmeans', '框架', '计算'], ['SAT', '简称', '求解', '现状及', 'NPC', '涌现', '健壮性', '迫切需要', '梳理', '成果'], ['编列', '课程', '资源', '学习', '智能化', '文献', '进化', '在线', '智慧', '学习者'], ['互评', '同伴', 'Moodle30', 'Workshop', 'Moodle', '不知', '操作方法', '顺利开展', '理解', '社群'], ['指令', 'GeoGebra', '翻译', '错误', '503063', '选项', '其官', '已达', '470', '预备'], ['InventoriLogic', 'iLogic', '编写程序', 'ET', '产品设计', 'BN', 'V', '机械', '编写', '介绍'], ['稳定', '恒稳', '饱和度', '线性系统', '有界', '椭球', '既保证', '保守', '判据', '演示'], ['约束', '进化', '优化', '可行', '处理', '领城', '罚函数法', '法多', '法', '混合法'], ['数据中心', '计算技术', '内存', '增长速度', '超乎', '保持良好', '海量', '想象', '剧增', '技术'], ['子句', '评论', '情感', '副词', '特殊符号', '抓取', '倾向', '产品', '识别', '官网'], ['空闲', '工件', '调度', '车间', '制造', '工期', '与己', '云', '工序', '加工'], ['AhoCorasick', '状态机', '存储状态', 'BitmappedAC', '压缩', '经典', '阶段', '极', '大幅度', '转移'], ['外存储器', '存控', '处理器', '片', '公平性', '调度', '现代', '作', '吞吐量', '动态随机'], ['GDSS', '凸', '加法', '乘法', '群体', '矩阵', 'AHP', 'Hadamard', '决策', '判断']]\n"
     ]
    }
   ],
   "source": [
    "corpus = [['MapReduce', '迭代', '并行计算', 'iMapReduceHadoopMapReduce', '式', '集中体现', '科研人员', 'Kmeans', '框架', '计算'], ['SAT', '简称', '求解', '现状及', 'NPC', '涌现', '健壮性', '迫切需要', '梳理', '成果'], ['编列', '课程', '资源', '学习', '智能化', '文献', '进化', '在线', '智慧', '学习者'], ['互评', '同伴', 'Moodle30', 'Workshop', 'Moodle', '不知', '操作方法', '顺利开展', '理解', '社群'], ['指令', 'GeoGebra', '翻译', '错误', '503063', '选项', '其官', '已达', '470', '预备'], ['InventoriLogic', 'iLogic', '编写程序', 'ET', '产品设计', 'BN', 'V', '机械', '编写', '介绍'], \n",
    "['稳定', '恒稳', '饱和度', '线性系统', '有界', '椭球', '既保证', '保守', '判据', '演示'], ['约束', '进化', '优化', '可行', '处理', '领城', '罚函数法', '法多', '法', '混合法'], ['数据中心', '计算技术', '内存', '增长速度', '超乎', '保持良好', '海量', '想象', '剧增', '技术'],\n",
    "['子句', '评论', '情感', '副词', '特殊符号', '抓取', '倾向', '产品', '识别', '官网'],\n",
    "['空闲', '工件', '调度', '车间', '制造', '工期', '与己', '云', '工序', '加工'],\n",
    "['AhoCorasick', '状态机', '存储状态', 'BitmappedAC', '压缩', '经典', '阶段', '极', '大幅度', '转移'],\n",
    "['外存储器', '存控', '处理器', '片', '公平性', '调度', '现代', '作', '吞吐量', '动态随机'],\n",
    "['GDSS', '凸', '加法', '乘法', '群体', '矩阵', 'AHP', 'Hadamard', '决策', '判断']]\n",
    "print(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['MapReduce',\n",
       "  '迭代',\n",
       "  '并行计算',\n",
       "  'iMapReduceHadoopMapReduce',\n",
       "  '式',\n",
       "  '集中体现',\n",
       "  '科研人员',\n",
       "  'Kmeans',\n",
       "  '框架',\n",
       "  '计算'],\n",
       " ['SAT', '简称', '求解', '现状及', 'NPC', '涌现', '健壮性', '迫切需要', '梳理', '成果']]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import logging, gensim\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load id->word mapping (the dictionary), one of the results of step 2 above\n",
    "id2word = gensim.corpora.Dictionary.load_from_text('wiki_en_wordids.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-120-f70a1e02b1d8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mLdaModel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_topics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\models\\ldamodel.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics, callbacks, dtype)\u001b[0m\n\u001b[0;32m    310\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid2word\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    311\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"no word id mapping provided; initializing from corpus, assuming identity\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 312\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid2word\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdict_from_corpus\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    313\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_terms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid2word\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    314\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid2word\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py\u001b[0m in \u001b[0;36mdict_from_corpus\u001b[1;34m(corpus)\u001b[0m\n\u001b[0;32m    570\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    571\u001b[0m     \"\"\"\n\u001b[1;32m--> 572\u001b[1;33m     \u001b[0mnum_terms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mget_max_id\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    573\u001b[0m     \u001b[0mid2word\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFakeDict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_terms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    574\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mid2word\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py\u001b[0m in \u001b[0;36mget_max_id\u001b[1;34m(corpus)\u001b[0m\n\u001b[0;32m    510\u001b[0m     \u001b[0mmaxid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    511\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mdocument\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 512\u001b[1;33m         \u001b[0mmaxid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmaxid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfieldid\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfieldid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdocument\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# [-1] to avoid exceptions from max(empty)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    513\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mmaxid\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    514\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    510\u001b[0m     \u001b[0mmaxid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    511\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mdocument\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcorpus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 512\u001b[1;33m         \u001b[0mmaxid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmaxid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfieldid\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfieldid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdocument\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# [-1] to avoid exceptions from max(empty)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    513\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mmaxid\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    514\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: too many values to unpack (expected 2)"
     ]
    }
   ],
   "source": [
    "import logging, gensim\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    ">>> # load id->word mapping (the dictionary), one of the results of step 2 above\n",
    ">>> id2word = gensim.corpora.Dictionary.load_from_text('wiki_en_wordids.txt')\n",
    ">>> # load corpus iterator\n",
    ">>> mm = gensim.corpora.MmCorpus('wiki_en_tfidf.mm')\n",
    ">>> # mm = gensim.corpora.MmCorpus('wiki_en_tfidf.mm.bz2') # use this if you compressed the TFIDF output\n",
    "\n",
    ">>> print(mm)\n",
    "MmCorpus(3931787 documents, 100000 features, 756379027 non-zero entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}