abstractive-summarization/6.copynet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ctexts.json','r') as fopen:\n",
    "    ctexts = json.load(fopen)\n",
    "    \n",
    "with open('headlines.json','r') as fopen:\n",
    "    headlines = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import collections\n",
    "from tensorflow.python.util import nest\n",
    "from tensorflow.contrib.framework.python.framework import tensor_util\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "class CopyNetWrapperState(\n",
    "    collections.namedtuple(\"CopyNetWrapperState\", (\"cell_state\", \"last_ids\", \"prob_c\"))):\n",
    "    def clone(self, **kwargs):\n",
    "        def with_same_shape(old, new):\n",
    "            \"\"\"Check and set new tensor's shape.\"\"\"\n",
    "            if isinstance(old, tf.Tensor) and isinstance(new, tf.Tensor):\n",
    "                return tensor_util.with_same_shape(old, new)\n",
    "            return new\n",
    "\n",
    "        return nest.map_structure(\n",
    "            with_same_shape,\n",
    "            self,\n",
    "            super(CopyNetWrapperState, self)._replace(**kwargs))\n",
    "\n",
    "class CopyNetWrapper(tf.contrib.rnn.RNNCell):\n",
    "    '''\n",
    "    A copynet RNN cell wrapper\n",
    "    '''\n",
    "    def __init__(self, cell, source_extend_tokens, max_oovs, encode_output, output_layer, vocab_size, name=None):\n",
    "        '''\n",
    "        Args:\n",
    "            - cell: the decoder cell\n",
    "            - source_extend_tokens: input tokens with oov word ids\n",
    "            - max_oovs: max number of oov words in each batch\n",
    "            - encode_output: the output of encoder cell\n",
    "            - output_layer: the layer used to map decoder output to vocab distribution\n",
    "            - vocab_size: this is target vocab size\n",
    "        '''\n",
    "        super(CopyNetWrapper, self).__init__(name=name)\n",
    "        self.cell = cell\n",
    "        self.source_extend_tokens = source_extend_tokens\n",
    "        self.encode_output = encode_output\n",
    "        self.max_oovs = max_oovs\n",
    "        self.output_layer = output_layer\n",
    "        self._output_size = vocab_size + max_oovs\n",
    "        self.copy_layer = Dense(self.cell.output_size,activation=tf.tanh,use_bias=False,name=\"Copy_Weigth\")\n",
    "\n",
    "    def __call__(self, inputs, state):\n",
    "        last_ids = state.last_ids\n",
    "        prob_c = state.prob_c\n",
    "        cell_state = state.cell_state\n",
    "        \n",
    "        '''\n",
    "            - Selective read\n",
    "                At first, my implement is based on the paper, which looks like belowing:\n",
    "                    mask = tf.cast(tf.equal(tf.expand_dims(last_ids,1),self.source_extend_tokens), tf.float32)\n",
    "                    pt = mask * prob_c\n",
    "                    pt_sum = tf.reduce_sum(pt, axis=1)\n",
    "                    pt = tf.where(tf.less(pt_sum, 1e-7), pt, pt / tf.expand_dims(pt_sum, axis=1))\n",
    "                    selective_read = tf.einsum(\"ijk,ij->ik\",self.encode_output, pt)\n",
    "                It looks OK for me, but I got NAN loss after one training step. I tried tf.Print to debug, but cannot find \n",
    "                the problem. Then I tried tfdbg to detect the source of NAN and found it refer to the tf.where code piece, \n",
    "                which is 'pt / tf.expand_dims(pt_sum, axis=1)'. \n",
    "                I am not sure why this code will got NANs. Maybe pt and pt_sum are not of the same shape, due to the broadcasting,\n",
    "                we will got NANs, and these NANs will be used to calculate the gradients, and cause NANs values.\n",
    "                Take a close look at the paper, we will find that p(y_t,c|.) is same for all the same input tokens. And selective_read\n",
    "                is sum of p_t * h_t, in which p_t is 1/K *(p(x_t,c|.)), K is sum of p(x_t,c|.). So p_t is equal to:\n",
    "                p_t = 1 / (number of times x_t shows in input tokens)\n",
    "        '''\n",
    "        # get selective read\n",
    "        # batch * input length\n",
    "        mask = tf.cast(tf.equal(tf.expand_dims(last_ids,1),self.source_extend_tokens), tf.float32)\n",
    "        mask_sum = tf.reduce_sum(mask, axis=1)\n",
    "        mask = tf.where(tf.less(mask_sum, 1e-7), mask, mask / tf.expand_dims(mask_sum, axis=1))\n",
    "        pt = mask * prob_c\n",
    "        selective_read = tf.einsum(\"ijk,ij->ik\",self.encode_output, pt)\n",
    "\n",
    "        inputs = tf.concat([inputs, selective_read], axis=-1)\n",
    "        outputs, cell_state = self.cell(inputs, cell_state)\n",
    "\n",
    "        # this is generate mode\n",
    "        vocab_dist = self.output_layer(outputs)\n",
    "\n",
    "        # this is copy mode\n",
    "        # batch * length * output size\n",
    "        copy_score = self.copy_layer(self.encode_output)\n",
    "        # batch * length\n",
    "        copy_score = tf.einsum(\"ijk,ik->ij\",copy_score,outputs)\n",
    "\n",
    "        vocab_size = tf.shape(vocab_dist)[-1]\n",
    "        extended_vsize = vocab_size + self.max_oovs \n",
    "\n",
    "        batch_size = tf.shape(vocab_dist)[0]\n",
    "        extra_zeros = tf.zeros((batch_size, self.max_oovs))\n",
    "        # batch * extend vocab size\n",
    "        vocab_dists_extended = tf.concat(axis=-1, values=[vocab_dist, extra_zeros])\n",
    "\n",
    "        # this part is same as that of point generator, but using einsum is much simpler.\n",
    "        source_mask = tf.one_hot(self.source_extend_tokens,extended_vsize)\n",
    "        attn_dists_projected = tf.einsum(\"ijn,ij->in\", source_mask, copy_score)\n",
    "\n",
    "        final_dist = vocab_dists_extended + attn_dists_projected\n",
    "\n",
    "        # This is greeding search, need to test with beam search\n",
    "        last_ids = tf.argmax(final_dist, axis=-1, output_type=tf.int32)\n",
    "\n",
    "        # this is used to calculate p(y_t,c|.)\n",
    "        # safe softmax\n",
    "        final_dist_max = tf.expand_dims(tf.reduce_max(final_dist,axis=1), axis=1)\n",
    "        final_dist_exp = tf.reduce_sum(tf.exp(final_dist - final_dist_max),axis=1)\n",
    "        p_c = tf.exp(attn_dists_projected - final_dist_max) / tf.expand_dims(final_dist_exp, axis=1)\n",
    "        p_c = tf.einsum(\"ijn,in->ij\",source_mask,p_c)\n",
    "\n",
    "        state = CopyNetWrapperState(cell_state=cell_state, last_ids=last_ids, prob_c=p_c)\n",
    "        return final_dist, state\n",
    "\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        \"\"\"size(s) of state(s) used by this cell.\n",
    "            It can be represented by an Integer, a TensorShape or a tuple of Integers\n",
    "            or TensorShapes.\n",
    "        \"\"\"\n",
    "        return CopyNetWrapperState(cell_state=self.cell.state_size, last_ids=tf.TensorShape([]),\n",
    "            prob_c = self.source_extend_tokens.shape[-1].value)\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        \"\"\"Integer or TensorShape: size of outputs produced by this cell.\"\"\"\n",
    "        return self._output_size\n",
    "\n",
    "    def zero_state(self, batch_size, dtype):\n",
    "        with tf.name_scope(type(self).__name__ + \"ZeroState\", values=[batch_size]):\n",
    "            cell_state = self.cell.zero_state(batch_size, dtype)\n",
    "            last_ids = tf.zeros([batch_size], tf.int32) - 1\n",
    "            prob_c = tf.zeros([batch_size, tf.shape(self.encode_output)[1]], tf.float32)\n",
    "            return CopyNetWrapperState(cell_state=cell_state, last_ids=last_ids, prob_c=prob_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "def topic_modelling(string, n = 500):\n",
    "    vectorizer = TfidfVectorizer()\n",
    "    tf = vectorizer.fit_transform([string])\n",
    "    tf_features = vectorizer.get_feature_names()\n",
    "    compose = TruncatedSVD(1).fit(tf)\n",
    "    return ' '.join([tf_features[i] for i in compose.components_[0].argsort()[: -n - 1 : -1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 1.54 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "h, c = [], []\n",
    "for i in range(len(ctexts)):\n",
    "    try:\n",
    "        c.append(ctexts[i])\n",
    "        h.append(headlines[i])\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 50942\n",
      "Most common words [('the', 97951), ('dot', 79175), ('comma', 74246), ('and', 42544), ('to', 41157), ('of', 38549)]\n",
      "Sample data [4, 11504, 7, 13450, 659, 12, 506, 8180, 11, 4096] ['the', 'daman', 'and', 'diu', 'administration', 'on', 'wednesday', 'withdrew', 'a', 'circular']\n"
     ]
    }
   ],
   "source": [
    "concat_from = ' '.join(c).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab to size: 8534\n",
      "Most common words [('to', 1388), ('in', 1196), ('comma', 876), ('s', 785), ('for', 733), ('of', 596)]\n",
      "Sample data [2797, 14, 2798, 2799, 656, 2800, 5, 1642, 657, 2087] ['daman', 'and', 'diu', 'revokes', 'mandatory', 'rakshabandhan', 'in', 'offices', 'order', 'malaika']\n"
     ]
    }
   ],
   "source": [
    "concat_to = ' '.join(h).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab to size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'daman and diu revokes mandatory rakshabandhan in offices order EOS'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(h)):\n",
    "    h[i] = h[i] + ' EOS'\n",
    "h[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic, UNK=3):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k, UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(c, h, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent2idx(sent, vocab, UNK=UNK):\n",
    "    tokens = sent.split()\n",
    "    oovs = []\n",
    "    extend_tokens = []\n",
    "    tokenized = []\n",
    "    for token in tokens:\n",
    "        if token not in vocab:\n",
    "            tokenized.append(UNK)\n",
    "            if token not in oovs:\n",
    "                oovs.append(token)\n",
    "            extend_tokens.append(len(vocab) + oovs.index(token))\n",
    "        else:\n",
    "            extend_tokens.append(vocab[token])\n",
    "            tokenized.append(vocab[token])\n",
    "    return tokenized, extend_tokens, oovs\n",
    "\n",
    "def target2idx(sent, oovs, vocab,UNK=UNK):\n",
    "    tokens = sent.split()\n",
    "    tokenized = []\n",
    "    for token in tokens:\n",
    "        if token not in vocab:\n",
    "            if token not in oovs:\n",
    "                tokenized.append(UNK)\n",
    "            else:\n",
    "                tokenized.append(len(vocab) + oovs.index(token))\n",
    "        else:\n",
    "            tokenized.append(vocab[token])\n",
    "    return tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Summarization:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 from_dict_size):\n",
    "        \n",
    "        def lstm_cell(reuse=False):\n",
    "            return tf.nn.rnn_cell.GRUCell(size_layer, reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        self.source_oov_words = tf.placeholder(tf.int32, shape=[])\n",
    "        self.source_extend_tokens = tf.placeholder(tf.int32, shape=[None, None])\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        condition = tf.less(decoder_input, from_dict_size)\n",
    "        self.decoder_input = self.Y\n",
    "        self.decoder_input_length = self.Y_seq_len\n",
    "        self.predict_count = tf.reduce_sum(self.decoder_input_length)\n",
    "        \n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        encoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        self.encoder_out, self.encoder_state = tf.nn.dynamic_rnn(cell = encoder_cells, \n",
    "                                                                 inputs = encoder_embedded, \n",
    "                                                                 sequence_length = self.X_seq_len,\n",
    "                                                                 dtype = tf.float32)\n",
    "        self.decode_initial_state = self.encoder_state[-1]\n",
    "        initial_state = [self.decode_initial_state for i in range(num_layers)]\n",
    "        self.initial_state = tuple(initial_state)\n",
    "        decoder_cell = attention(self.encoder_out, self.X_seq_len)\n",
    "        dense_layer = tf.layers.Dense(embedded_size)\n",
    "        \n",
    "        initial_state = decoder_cell.zero_state(batch_size, tf.float32).clone(cell_state=self.initial_state)\n",
    "        \n",
    "        self.decode_cell = CopyNetWrapper(decoder_cell, self.source_extend_tokens, 50, \n",
    "                                          self.encoder_out, dense_layer, embedded_size)\n",
    "        \n",
    "        dense_layer = tf.layers.Dense(from_dict_size)\n",
    "        \n",
    "        self.initial_state = self.decode_cell.zero_state(batch_size,\n",
    "                                                         tf.float32).clone(cell_state=initial_state)\n",
    "        \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                sequence_length = self.decoder_input_length,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = self.decode_cell,\n",
    "                helper = training_helper,\n",
    "                initial_state = self.initial_state,\n",
    "                output_layer = dense_layer)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = encoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "        predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = self.decode_cell,\n",
    "                helper = predicting_helper,\n",
    "                initial_state = self.initial_state,\n",
    "                output_layer = dense_layer)\n",
    "        \n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.X_seq_len))\n",
    "        \n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.sample_id\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        \n",
    "        self.optimizer = tf.train.AdamOptimizer().minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "batch_size = 8\n",
    "epoch = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Summarization(size_layer, num_layers, embedded_size, len(dictionary_from))\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def batching(X, Y):\n",
    "    s_, es_, oovs_, target_ = [], [], [], []\n",
    "    for x, y in zip(X, Y):\n",
    "        s,es,oovs = sent2idx(x, dictionary_from)\n",
    "        target = target2idx(y, oovs,dictionary_from)\n",
    "        s_.append(s)\n",
    "        es_.append(es)\n",
    "        oovs_.append(oovs)\n",
    "        target_.append(target)\n",
    "    s_ = pad_sequences(s_,padding='post')\n",
    "    es_ = pad_sequences(es_,padding='post')\n",
    "    target_ = pad_sequences(target_,padding='post')\n",
    "    maxlen = max([len(o) for o in oovs_])\n",
    "    return s_, es_, target_, maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:51<00:00,  1.41it/s, accuracy=0.069, cost=6.68] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:32<00:00,  3.26it/s, accuracy=0.112, cost=7.37] \n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, avg loss: 8.006333, avg accuracy: 0.093200\n",
      "epoch: 0, avg loss test: 7.498233, avg accuracy test: 0.111772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:39<00:00,  1.43it/s, accuracy=0.111, cost=6.88] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:31<00:00,  2.83it/s, accuracy=0.0938, cost=7.13]\n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 6.790594, avg accuracy: 0.109877\n",
      "epoch: 1, avg loss test: 7.670779, avg accuracy test: 0.111236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:42<00:00,  1.40it/s, accuracy=0.182, cost=4.48] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:30<00:00,  5.15it/s, accuracy=0.134, cost=8.54] \n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 5.148010, avg accuracy: 0.138735\n",
      "epoch: 2, avg loss test: 8.594195, avg accuracy test: 0.116735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:38<00:00,  1.58it/s, accuracy=0.302, cost=3.48]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:31<00:00,  5.22it/s, accuracy=0.0568, cost=10.2]\n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 4.152503, avg accuracy: 0.205743\n",
      "epoch: 3, avg loss test: 9.923908, avg accuracy test: 0.117548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:43<00:00,  2.01s/it, accuracy=0.225, cost=8.03]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:32<00:00,  4.73it/s, accuracy=0.102, cost=8.72] \n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 3.535306, avg accuracy: 0.263569\n",
      "epoch: 4, avg loss test: 10.764884, avg accuracy test: 0.114450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:40<00:00,  1.54it/s, accuracy=0.205, cost=4.01]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:31<00:00,  4.45it/s, accuracy=0.12, cost=15.8]  \n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 3.246131, avg accuracy: 0.304056\n",
      "epoch: 5, avg loss test: 11.884373, avg accuracy test: 0.106891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:39<00:00,  1.08s/it, accuracy=0.317, cost=2.73]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:32<00:00,  4.45it/s, accuracy=0.0964, cost=9.4] \n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 2.938670, avg accuracy: 0.339051\n",
      "epoch: 6, avg loss test: 10.258962, avg accuracy test: 0.116750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:40<00:00,  1.79it/s, accuracy=0.396, cost=2.72]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:31<00:00,  1.02it/s, accuracy=0.0833, cost=10.7]\n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 2.457255, avg accuracy: 0.409344\n",
      "epoch: 7, avg loss test: 11.130594, avg accuracy test: 0.118689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:47<00:00,  1.50it/s, accuracy=0.571, cost=1.9] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:32<00:00,  5.30it/s, accuracy=0.0706, cost=12.3]\n",
      "train minibatch loop:   0%|          | 0/440 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 2.118696, avg accuracy: 0.469681\n",
      "epoch: 8, avg loss test: 12.793685, avg accuracy test: 0.110155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [06:44<00:00,  1.40it/s, accuracy=0.452, cost=2]   \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:31<00:00,  3.35it/s, accuracy=0.069, cost=16.2] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 1.810338, avg accuracy: 0.531653\n",
      "epoch: 9, avg loss test: 12.994831, avg accuracy test: 0.111562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "\n",
    "for EPOCH in range(10):\n",
    "    lasttime = time.time()\n",
    "    total_loss, total_accuracy, total_loss_test, total_accuracy_test = 0, 0, 0, 0\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    test_X, test_Y = shuffle(test_X, test_Y)\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size), desc='train minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(train_X))\n",
    "        batch_x, batch_es, batch_y, maxlen = batching(train_X[k: index],\n",
    "                                                     train_Y[k: index])\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                 model.source_extend_tokens:batch_es,\n",
    "                                                model.Y:batch_y,\n",
    "                                                model.source_oov_words:maxlen})\n",
    "        total_loss += loss\n",
    "        total_accuracy += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc='test minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(test_X))\n",
    "        batch_x, batch_es, batch_y, maxlen = batching(test_X[k: index],\n",
    "                                                     test_Y[k: index])\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                 model.source_extend_tokens:batch_es,\n",
    "                                                model.Y:batch_y,\n",
    "                                                model.source_oov_words:maxlen})\n",
    "        total_loss_test += loss\n",
    "        total_accuracy_test += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    total_loss /= (len(train_X) / batch_size)\n",
    "    total_accuracy /= (len(train_X) / batch_size)\n",
    "    total_loss_test /= (len(test_X) / batch_size)\n",
    "    total_accuracy_test /= (len(test_X) / batch_size)\n",
    "        \n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(EPOCH, total_loss, total_accuracy))\n",
    "    print('epoch: %d, avg loss test: %f, avg accuracy test: %f'%(EPOCH, total_loss_test, total_accuracy_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, batch_es, batch_y, maxlen = batching(test_X[:1], test_Y[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['india', 'india', 'in', 'indian', 'in', 'EOS']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[rev_dictionary_from[i] for i in sess.run(model.predicting_ids, feed_dict = {model.X: batch_x,\n",
    "                                             model.Y: batch_y,\n",
    "                                             model.source_extend_tokens:batch_es,\n",
    "                                             model.source_oov_words:maxlen})[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}