CBoW.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(13)\n",
    "\n",
    "import keras.backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Lambda\n",
    "from keras.utils.data_utils import get_file\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = get_file('alice.txt', origin='http://www.gutenberg.org/files/11/11-0.txt')\n",
    "corpus = open(path).readlines()[:300]\n",
    "corpus = [sentence for sentence in corpus if sentence.count(' ') >= 2]\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(corpus)\n",
    "corpus = tokenizer.texts_to_sequences(corpus)\n",
    "nb_samples = sum(len(s) for s in corpus)\n",
    "V = len(tokenizer.word_index) + 1\n",
    "dim = 100\n",
    "window_size = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(corpus, window_size, V):\n",
    "    maxlen = window_size*2\n",
    "    for words in corpus:\n",
    "        L = len(words)\n",
    "        for index, word in enumerate(words):\n",
    "            contexts = []\n",
    "            labels   = []            \n",
    "            s = index - window_size\n",
    "            e = index + window_size + 1\n",
    "            \n",
    "            contexts.append([words[i] for i in range(s, e) if 0 <= i < L and i != index])\n",
    "            labels.append(word)\n",
    "\n",
    "            x = sequence.pad_sequences(contexts, maxlen=maxlen)\n",
    "            y = np_utils.to_categorical(labels, V)\n",
    "            yield (x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbow = Sequential()\n",
    "cbow.add(Embedding(input_dim=V, output_dim=dim, input_length=window_size*2))\n",
    "cbow.add(Lambda(lambda x: K.mean(x, axis=1), output_shape=(dim,)))\n",
    "cbow.add(Dense(V, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cbow.compile(loss='categorical_crossentropy', optimizer='adadelta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 17897.1324615\n",
      "1 16611.4819975\n",
      "2 16054.5737875\n",
      "3 15975.4586413\n",
      "4 16078.7989137\n",
      "5 16168.2691295\n",
      "6 16193.6849765\n",
      "7 16191.0186955\n",
      "8 16183.2132714\n",
      "9 16174.5272493\n"
     ]
    }
   ],
   "source": [
    "for ite in range(10):\n",
    "    loss = 0.\n",
    "    for x, y in generate_data(corpus, window_size, V):\n",
    "        loss += cbow.train_on_batch(x, y)\n",
    "\n",
    "    print(ite, loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = open('vectors.txt' ,'w')\n",
    "f.write('{} {}\\n'.format(V-1, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = cbow.get_weights()[0]\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    str_vec = ' '.join(map(str, list(vectors[i, :])))\n",
    "    f.write('{} {}\\n'.format(word, str_vec))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v = gensim.models.KeyedVectors.load_word2vec_format('./vectors.txt', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('a', 0.8029883503913879),\n",
       " ('her', 0.6420163512229919),\n",
       " ('i', 0.6390146613121033),\n",
       " ('it', 0.6209530830383301),\n",
       " ('this', 0.6207232475280762),\n",
       " ('my', 0.6043308973312378),\n",
       " ('you', 0.5718163847923279),\n",
       " ('to', 0.5632857084274292),\n",
       " ('about', 0.5545670986175537),\n",
       " ('down', 0.5478866696357727)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['the'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('said', 0.522140383720398),\n",
       " ('she', 0.4846324920654297),\n",
       " ('thought', 0.4466657042503357),\n",
       " ('you', 0.4399128556251526),\n",
       " ('say', 0.4043766260147095),\n",
       " ('now', 0.40042510628700256),\n",
       " ('her', 0.3967324495315552),\n",
       " ('that', 0.38552236557006836),\n",
       " ('because', 0.37965667247772217),\n",
       " ('dark', 0.3769078254699707)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v.most_similar(positive=['alice'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}