keyword_expansion_introduction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\liaok\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "C:\\Users\\liaok\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from bailarn.keyword_expansion import KeywordExpansion\n",
    "\n",
    "from bailarn.keyword_expansion import utils\n",
    "import deepcut\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train new model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ke = KeywordExpansion(new_model=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load file name "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_list = utils.get_file_name_list('../../Downloads/token_pos/tokenized_post/','*.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "314058"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(file_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and tranfrom file to sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                                          | 0/50 [00:00<?, ?it/s]\r",
      " 48%|██████████████████████████████████████████████                                                  | 24/50 [00:00<00:00, 236.30it/s]\r",
      " 92%|████████████████████████████████████████████████████████████████████████████████████████▎       | 46/50 [00:00<00:00, 226.11it/s]\r",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:00<00:00, 224.66it/s]\n"
     ]
    }
   ],
   "source": [
    "sentences = utils.pantip_json_to_sentences(file_list[0:50])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : collecting all words and their counts\n",
      "INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "INFO : collected 2569 word types from a corpus of 643833 raw words and 443 sentences\n",
      "INFO : Loading a fresh vocabulary\n",
      "INFO : min_count=5 retains 2143 unique words (83% of original 2569, drops 426)\n",
      "INFO : min_count=5 leaves 642736 word corpus (99% of original 643833, drops 1097)\n",
      "INFO : deleting the raw counts dictionary of 2569 items\n",
      "INFO : sample=0.001 downsamples 79 most-common words\n",
      "INFO : downsampling leaves estimated 480052 word corpus (74.7% of prior 642736)\n",
      "INFO : estimated required memory for 2143 words and 300 dimensions: 6214700 bytes\n",
      "INFO : resetting layer weights\n",
      "INFO : training model with 4 workers on 2143 vocabulary and 300 features, using sg=1 hs=0 sample=0.001 negative=5 window=5\n",
      "INFO : EPOCH 1 - PROGRESS: at 17.83% examples, 293370 words/s, in_qsize 7, out_qsize 0\n",
      "INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "INFO : EPOCH - 1 : training on 643833 raw words (480057 effective words) took 1.6s, 299104 effective words/s\n",
      "INFO : EPOCH 2 - PROGRESS: at 15.12% examples, 249570 words/s, in_qsize 7, out_qsize 0\n",
      "INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "INFO : EPOCH - 2 : training on 643833 raw words (480023 effective words) took 1.9s, 259127 effective words/s\n",
      "INFO : EPOCH 3 - PROGRESS: at 13.54% examples, 219740 words/s, in_qsize 7, out_qsize 0\n",
      "INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "INFO : EPOCH - 3 : training on 643833 raw words (480425 effective words) took 1.9s, 254335 effective words/s\n",
      "INFO : EPOCH 4 - PROGRESS: at 17.16% examples, 284728 words/s, in_qsize 7, out_qsize 0\n",
      "INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "INFO : EPOCH - 4 : training on 643833 raw words (480300 effective words) took 1.8s, 269469 effective words/s\n",
      "INFO : EPOCH 5 - PROGRESS: at 16.70% examples, 273180 words/s, in_qsize 7, out_qsize 0\n",
      "INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "INFO : EPOCH - 5 : training on 643833 raw words (480059 effective words) took 1.7s, 282860 effective words/s\n",
      "INFO : training on a 3219165 raw words (2400864 effective words) took 8.9s, 270650 effective words/s\n"
     ]
    }
   ],
   "source": [
    "ke.train(sentence=sentences[0:],size=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('เนต', 0.8959490060806274),\n",
       " ('script', 0.8333032727241516),\n",
       " ('street', 0.8165960907936096),\n",
       " ('เหอะ', 0.8126247525215149),\n",
       " ('sync', 0.8103899955749512),\n",
       " ('ถี่', 0.7986263632774353),\n",
       " ('ทั่ว', 0.7970977425575256),\n",
       " ('คูปอง', 0.7718362808227539),\n",
       " ('11', 0.7709002494812012),\n",
       " ('มอก', 0.7592698931694031)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ke.predict('เร็ว')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = ['เนต']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.18181818181818182, 'precision': 0.1, 'recall': 1.0}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ke.evaluate('เร็ว', exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : saving Word2Vec object under test, separately None\n",
      "INFO : not storing attribute vectors_norm\n",
      "INFO : not storing attribute cum_table\n",
      "INFO : saved test\n"
     ]
    }
   ],
   "source": [
    "ke.save('test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use default model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : loading Word2Vec object from C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower\n",
      "INFO : Model saved using code from earlier Gensim Version. Re-loading old model in a compatible way.\n",
      "INFO : loading Word2Vec object from C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower\n",
      "INFO : loading wv recursively from C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower.wv.* with mmap=None\n",
      "INFO : loading syn0 from C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower.wv.syn0.npy with mmap=None\n",
      "INFO : setting ignored attribute syn0norm to None\n",
      "INFO : loading syn1neg from C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower.syn1neg.npy with mmap=None\n",
      "INFO : setting ignored attribute cum_table to None\n",
      "INFO : loaded C:\\Users\\liaok\\Documents\\Thai_NLP_platform\\bailarn\\keyword_expansion\\./models/phone_lower\n"
     ]
    }
   ],
   "source": [
    "ke = KeywordExpansion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('ช้า', 0.8411610126495361),\n",
       " ('ไว', 0.8156427145004272),\n",
       " ('แรง', 0.7749983072280884),\n",
       " ('น้อย', 0.7714924216270447),\n",
       " ('สูง', 0.7665342688560486),\n",
       " ('เต็ม', 0.7643394470214844),\n",
       " ('ลื่น', 0.7565712928771973),\n",
       " ('ง่าย', 0.7555305361747742),\n",
       " ('สรุป', 0.7497644424438477),\n",
       " ('เร็วnet', 0.7487413287162781)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ke.predict('เร็ว')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ke = KeywordExpansion(model_path='./bailarn/keyword_expansion/models/mobileandwiki.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('ios', 0.743586540222168),\n",
       " ('android', 0.72163987159729),\n",
       " ('Android', 0.7210320234298706),\n",
       " ('เวอร์ชั่น', 0.7165907025337219),\n",
       " ('อัพ', 0.7055657505989075),\n",
       " ('เปลี่ยนาก', 0.7004285454750061),\n",
       " ('คับ', 0.697682797908783),\n",
       " ('iOS', 0.6968648433685303),\n",
       " ('แรมก้เหลือ', 0.6965107321739197),\n",
       " ('แอป', 0.6941495537757874)]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ke.predict('แอนดรอย')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ธงจะ', 0.7304272055625916),\n",
       " ('โอเอสเอง', 0.7292593717575073),\n",
       " ('และ<b', 0.7142135500907898),\n",
       " ('ศูนย์ปรกติ', 0.7088775038719177),\n",
       " ('ดิเนี้ย', 0.7068774700164795),\n",
       " ('กวนตีu', 0.7015659213066101),\n",
       " ('ซัมซุงโน๊ตสี่', 0.6988876461982727),\n",
       " ('อัสชุส', 0.6914786696434021),\n",
       " ('อุ๊ต๊ะ\\\\sแม่จ้าว', 0.6869146823883057),\n",
       " ('8bg', 0.6861496567726135)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ke.predict('ซุมซุง')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}