lab-12-6-seq-to-seq-with-attention-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lab-12-6 sequence to sequence with attention (Keras + eager version)\n",
    "\n",
    "### simple neural machine translation training\n",
    "\n",
    "* sequence to sequence\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)\n",
    "* [Effective Approaches to Attention-based Neural Machine Translation](https://arxiv.org/abs/1508.04025)\n",
    "* [Neural Machine Translation with Attention from Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "# Import TensorFlow >= 1.10 and enable eager execution\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "from matplotlib import font_manager, rc\n",
    "\n",
    "rc('font', family='AppleGothic') #for mac\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from pprint import pprint\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "targets = [['나는', '배가', '고프다'],\n",
    "           ['텐서플로우는', '매우', '어렵다'],\n",
    "           ['텐서플로우는', '딥러닝을', '위한', '프레임워크이다'],\n",
    "           ['텐서플로우는', '매우', '빠르게', '변화한다']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0,\n",
      " 'I': 1,\n",
      " 'a': 2,\n",
      " 'changing': 3,\n",
      " 'deep': 4,\n",
      " 'difficult': 5,\n",
      " 'fast': 6,\n",
      " 'feel': 7,\n",
      " 'for': 8,\n",
      " 'framework': 9,\n",
      " 'hungry': 10,\n",
      " 'is': 11,\n",
      " 'learning': 12,\n",
      " 'tensorflow': 13,\n",
      " 'very': 14}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for sources\n",
    "s_vocab = list(set(sum(sources, [])))\n",
    "s_vocab.sort()\n",
    "s_vocab = ['<pad>'] + s_vocab\n",
    "source2idx = {word : idx for idx, word in enumerate(s_vocab)}\n",
    "idx2source = {idx : word for idx, word in enumerate(s_vocab)}\n",
    "\n",
    "pprint(source2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<bos>': 1,\n",
      " '<eos>': 2,\n",
      " '<pad>': 0,\n",
      " '고프다': 3,\n",
      " '나는': 4,\n",
      " '딥러닝을': 5,\n",
      " '매우': 6,\n",
      " '배가': 7,\n",
      " '변화한다': 8,\n",
      " '빠르게': 9,\n",
      " '어렵다': 10,\n",
      " '위한': 11,\n",
      " '텐서플로우는': 12,\n",
      " '프레임워크이다': 13}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for targets\n",
    "t_vocab = list(set(sum(targets, [])))\n",
    "t_vocab.sort()\n",
    "t_vocab = ['<pad>', '<bos>', '<eos>'] + t_vocab\n",
    "target2idx = {word : idx for idx, word in enumerate(t_vocab)}\n",
    "idx2target = {idx : word for idx, word in enumerate(t_vocab)}\n",
    "\n",
    "pprint(target2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sequences, max_len, dic, mode = 'source'):\n",
    "    assert mode in ['source', 'target'], 'source와 target 중에 선택해주세요.'\n",
    "    \n",
    "    if mode == 'source':\n",
    "        # preprocessing for source (encoder)\n",
    "        s_input = list(map(lambda sentence : [dic.get(token) for token in sentence], sequences))\n",
    "        s_len = list(map(lambda sentence : len(sentence), s_input))\n",
    "        s_input = pad_sequences(sequences = s_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        return s_len, s_input\n",
    "    \n",
    "    elif mode == 'target':\n",
    "        # preprocessing for target (decoder)\n",
    "        # input\n",
    "        t_input = list(map(lambda sentence : ['<bos>'] + sentence + ['<eos>'], sequences))\n",
    "        t_input = list(map(lambda sentence : [dic.get(token) for token in sentence], t_input))\n",
    "        t_len = list(map(lambda sentence : len(sentence), t_input))\n",
    "        t_input = pad_sequences(sequences = t_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        # output\n",
    "        t_output = list(map(lambda sentence : sentence + ['<eos>'], sequences))\n",
    "        t_output = list(map(lambda sentence : [dic.get(token) for token in sentence], t_output))\n",
    "        t_output = pad_sequences(sequences = t_output, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        return t_len, t_input, t_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 7, 5] [[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for source\n",
    "s_max_len = 10\n",
    "s_len, s_input = preprocess(sequences = sources,\n",
    "                            max_len = s_max_len, dic = source2idx, mode = 'source')\n",
    "print(s_len, s_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 6, 6] [[ 1  4  7  3  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  6 10  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  5 11 13  2  0  0  0  0  0  0]\n",
      " [ 1 12  6  9  8  2  0  0  0  0  0  0]] [[ 4  7  3  2  0  0  0  0  0  0  0  0]\n",
      " [12  6 10  2  0  0  0  0  0  0  0  0]\n",
      " [12  5 11 13  2  0  0  0  0  0  0  0]\n",
      " [12  6  9  8  2  0  0  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for target\n",
    "t_max_len = 12\n",
    "t_len, t_input, t_output = preprocess(sequences = targets,\n",
    "                                      max_len = t_max_len, dic = target2idx, mode = 'target')\n",
    "print(t_len, t_input, t_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hyper-param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper-parameters\n",
    "epochs = 100\n",
    "batch_size = 4\n",
    "learning_rate = .005\n",
    "total_step = epochs / batch_size\n",
    "buffer_size = 100\n",
    "n_batch = buffer_size//batch_size\n",
    "embedding_dim = 32\n",
    "units = 128\n",
    "\n",
    "# input\n",
    "data = tf.data.Dataset.from_tensor_slices((s_len, s_input, t_len, t_input, t_output))\n",
    "data = data.shuffle(buffer_size = buffer_size)\n",
    "data = data.batch(batch_size = batch_size)\n",
    "# s_mb_len, s_mb_input, t_mb_len, t_mb_input, t_mb_output = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
    "  # the code automatically does that.\n",
    "    if tf.test.is_gpu_available():\n",
    "        return tf.keras.layers.CuDNNGRU(units, \n",
    "                                        return_sequences=True, \n",
    "                                        return_state=True, \n",
    "                                        recurrent_initializer='glorot_uniform')\n",
    "    else:\n",
    "        return tf.keras.layers.GRU(units, \n",
    "                                   return_sequences=True, \n",
    "                                   return_state=True, \n",
    "                                   recurrent_activation='sigmoid', \n",
    "                                   recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "#         print(\"state: {}\".format(state.shape))\n",
    "#         print(\"output: {}\".format(state.shape))\n",
    "              \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        # * `score = FC(tanh(FC(EO) + FC(H)))`\n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "                \n",
    "        #* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, 1)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        # * `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        # * `embedding output` = The input to the decoder X is passed through an embedding layer.\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        # * `merged vector = concat(embedding output, context vector)`\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(len(source2idx), embedding_dim, units, batch_size)\n",
    "decoder = Decoder(len(target2idx), embedding_dim, units, batch_size)\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)\n",
    "\n",
    "# creating optimizer\n",
    "optimizer = tf.train.AdamOptimizer()\n",
    "\n",
    "# creating check point (Object-based saving)\n",
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                encoder=encoder,\n",
    "                                decoder=decoder)\n",
    "\n",
    "# create writer for tensorboard\n",
    "summary_writer = tf.contrib.summary.create_file_writer(logdir=checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Loss 0.0396 Batch Loss 0.9905\n",
      "Epoch 10 Loss 0.0381 Batch Loss 0.9515\n",
      "Epoch 20 Loss 0.0349 Batch Loss 0.8719\n",
      "Epoch 30 Loss 0.0328 Batch Loss 0.8209\n",
      "Epoch 40 Loss 0.0297 Batch Loss 0.7431\n",
      "Epoch 50 Loss 0.0223 Batch Loss 0.5575\n",
      "Epoch 60 Loss 0.0174 Batch Loss 0.4351\n",
      "Epoch 70 Loss 0.0124 Batch Loss 0.3088\n",
      "Epoch 80 Loss 0.0076 Batch Loss 0.1900\n",
      "Epoch 90 Loss 0.0043 Batch Loss 0.1071\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 100\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for i, (s_len, s_input, t_len, t_input, t_output) in enumerate(data):\n",
    "        loss = 0\n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(s_input, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([target2idx['<bos>']] * batch_size, 1)\n",
    "            \n",
    "            #Teacher Forcing: feeding the target as the next input\n",
    "            for t in range(1, t_input.shape[1]):\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(t_input[:, t], predictions)\n",
    "            \n",
    "                dec_input = tf.expand_dims(t_input[:, t], 1) #using teacher forcing\n",
    "                \n",
    "        batch_loss = (loss / int(t_input.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradient = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradient, variables))\n",
    "        \n",
    "    if epoch % 10 == 0:\n",
    "        #save model every 10 epoch\n",
    "        print('Epoch {} Loss {:.4f} Batch Loss {:.4f}'.format(epoch,\n",
    "                                            total_loss / n_batch,\n",
    "                                            batch_loss.numpy()))\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "#     sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang['<bos>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += idx2target[predicted_id] + ' '\n",
    "\n",
    "        if idx2target.get(predicted_id) == '<eos>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot\n",
    "\n",
    "# result, sentence, attention_plot = evaluate(sentence, encoder, decoder, source2idx, target2idx,\n",
    "#                                             s_max_len, t_max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Shapes (128, 14) and (32, 14) are incompatible",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-eebf1c567676>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#restore checkpoint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlatest_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/util.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, save_path)\u001b[0m\n\u001b[1;32m   1808\u001b[0m           \u001b[0mcalled\u001b[0m \u001b[0mwhen\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1809\u001b[0m     \"\"\"\n\u001b[0;32m-> 1810\u001b[0;31m     \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_saver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msave_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1811\u001b[0m     \u001b[0;31m# Create the save counter now so it gets initialized with other variables\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1812\u001b[0m     \u001b[0;31m# when graph building. Creating it earlier would lead to double\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/util.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, save_path)\u001b[0m\n\u001b[1;32m   1475\u001b[0m         saveable_object_cache=self._saveable_object_cache)\n\u001b[1;32m   1476\u001b[0m     base._CheckpointPosition(  # pylint: disable=protected-access\n\u001b[0;32m-> 1477\u001b[0;31m         checkpoint=checkpoint, proto_id=0).restore(self._root_checkpointable)\n\u001b[0m\u001b[1;32m   1478\u001b[0m     load_status = CheckpointLoadStatus(\n\u001b[1;32m   1479\u001b[0m         \u001b[0mcheckpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, checkpointable)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# This object's correspondence with a checkpointed object is new, so\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;31m# process deferred restorations for it and its dependencies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m         \u001b[0mrestore_ops\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheckpointable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restore_from_checkpoint_position\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_checkpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew_restore_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrestore_ops\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36m_restore_from_checkpoint_position\u001b[0;34m(self, checkpoint_position)\u001b[0m\n\u001b[1;32m    794\u001b[0m           ._single_restoration_from_checkpoint_position(\n\u001b[1;32m    795\u001b[0m               \u001b[0mcheckpoint_position\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcurrent_position\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 796\u001b[0;31m               visit_queue=visit_queue)))\n\u001b[0m\u001b[1;32m    797\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    798\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36m_single_restoration_from_checkpoint_position\u001b[0;34m(self, checkpoint_position, visit_queue)\u001b[0m\n\u001b[1;32m    821\u001b[0m             child_position)\n\u001b[1;32m    822\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 823\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mchild_position\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind_object\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpointable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlocal_object\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    824\u001b[0m           \u001b[0;31m# This object's correspondence is new, so dependencies need to be\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    825\u001b[0m           \u001b[0;31m# visited. Delay doing it so that we get a breadth-first dependency\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36mbind_object\u001b[0;34m(self, checkpointable)\u001b[0m\n\u001b[1;32m    243\u001b[0m                   proto_id=slot_restoration.slot_variable_id),\n\u001b[1;32m    244\u001b[0m               \u001b[0mvariable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheckpointable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m               slot_name=slot_restoration.slot_name)\n\u001b[0m\u001b[1;32m    246\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m  \u001b[0;31m# New assignment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/optimizer.py\u001b[0m in \u001b[0;36m_create_or_restore_slot_variable\u001b[0;34m(self, slot_variable_position, slot_name, variable)\u001b[0m\n\u001b[1;32m   1212\u001b[0m       \u001b[0;31m# If we've either made this slot variable, or if we've pulled out an\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1213\u001b[0m       \u001b[0;31m# existing slot variable, we should restore it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1214\u001b[0;31m       \u001b[0mslot_variable_position\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mslot_variable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1215\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1216\u001b[0m       \u001b[0;31m# We didn't make the slot variable. Defer restoring until it gets created\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, checkpointable)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# This object's correspondence with a checkpointed object is new, so\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;31m# process deferred restorations for it and its dependencies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m         \u001b[0mrestore_ops\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheckpointable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restore_from_checkpoint_position\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_checkpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew_restore_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrestore_ops\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36m_restore_from_checkpoint_position\u001b[0;34m(self, checkpoint_position)\u001b[0m\n\u001b[1;32m    794\u001b[0m           ._single_restoration_from_checkpoint_position(\n\u001b[1;32m    795\u001b[0m               \u001b[0mcheckpoint_position\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcurrent_position\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 796\u001b[0;31m               visit_queue=visit_queue)))\n\u001b[0m\u001b[1;32m    797\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    798\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36m_single_restoration_from_checkpoint_position\u001b[0;34m(self, checkpoint_position, visit_queue)\u001b[0m\n\u001b[1;32m    806\u001b[0m     \u001b[0;31m# restoration on to our dependencies.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    807\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore_uid\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_uid\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 808\u001b[0;31m       \u001b[0mrestore_ops\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheckpoint_position\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore_ops\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    809\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_uid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheckpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore_uid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py\u001b[0m in \u001b[0;36mrestore_ops\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    405\u001b[0m             saveable_index:saveable_index + num_specs]\n\u001b[1;32m    406\u001b[0m         \u001b[0msaveable_index\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnum_specs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 407\u001b[0;31m         \u001b[0mrestore_op\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msaveable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msaveable_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrestored_shapes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    408\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    409\u001b[0m           \u001b[0;32massert\u001b[0m \u001b[0msaveable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_checkpoint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore_ops_by_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/training/saver.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, restored_tensors, restored_shapes)\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0mrestored_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0midentity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrestored_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    160\u001b[0m         return resource_variable_ops.shape_safe_assign_variable_handle(\n\u001b[0;32m--> 161\u001b[0;31m             self.handle_op, self._var_shape, restored_tensor)\n\u001b[0m\u001b[1;32m    162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrite_version\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msaver_pb2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSaverDef\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mV2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36mshape_safe_assign_variable_handle\u001b[0;34m(handle, shape, value, name)\u001b[0m\n\u001b[1;32m    158\u001b[0m   \u001b[0;32mwith\u001b[0m \u001b[0m_handle_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m     \u001b[0mvalue_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m   \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massert_is_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue_tensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m   return gen_resource_variable_ops.assign_variable_op(handle,\n\u001b[1;32m    162\u001b[0m                                                       \u001b[0mvalue_tensor\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/framework/tensor_shape.py\u001b[0m in \u001b[0;36massert_is_compatible_with\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    846\u001b[0m     \"\"\"\n\u001b[1;32m    847\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 848\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Shapes %s and %s are incompatible\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    850\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mmost_specific_compatible_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Shapes (128, 14) and (32, 14) are incompatible"
     ]
    }
   ],
   "source": [
    "#restore checkpoint\n",
    "\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: I feel hungry\n",
      "Predicted translation: 나는 배가 고프다 <eos> \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAJjCAYAAAD56u3EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFz9JREFUeJzt3X+w5Xdd3/HXe38kAUMMCb+1IkHKD/mlzaAgiC0RqtWpdNRaYtU4GrUz1tSxFAbFsSiKtqU/QCUdYgGp9Qc0DO10hKbGgik4qdUy0lBMQSnS8CPEQFDY7L77xzlJTm72vXt395577u4+HjN3cu8533PO+87Z3Of9nO/3e251dwDgaPZtegAA9i6RAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAWKqq91TVS6rqMZueZa8QCYB7XJXk4Umuq6rfq6p/VFWP2vRQm1Teuwngvqrq0iTftPz4TJJ/l+TXu/tDGx1sl4kEwDFU1Rcn+cdJLk/y0STvTfKD3f3eDY61a7zcBLBFVZ1fVd9dVdcnuS7JzUku6e6HJ3lNkn+/yfl2k5UEwFJV/fUk35nkuUl+M8k1Sa7rlR+UVXVuklu6+8LNTLm7RAJgqap+L8lrk/zb7v7ksM3BJA/q7o/s6nAbIhIAS1X13Ule192HNz3LXiESAEtVdXt3X7DpOfYSO64B7vE7VXXlpofYS6wkAJaq6nuTvCLJjVnsuP7Y6vXd/fpNzLVJIgGwVFW/dIyru7u/e9eG2SNEAmCpqh5yjKvv6O47dm2YPUIkAJaq6o+SHOu9mj6a5BeT/OTZcgSUHdcA93hzkmu6e39370/ygCTfkOS3k3x5kq9PclmSl2xuxN1lJQGwVFUfSPLlW0+kq6ovT/Kq7n5GVV2S5D9192M3MuQuEwmApar6VBZnU392y+UHkvxZd3/eXdt19wM2MeNu83ITwD1uSPJ9R7n8b2fx7q+pqqck+eAuzrRRVhIAS1X16CT/Ocn/WP73UJJnJXlekr+xvPztSd7Y3a/d1Jy7SSQAVlTV/ZJ8b5JnJDknizC8urtvraoLkzyhu2/Y5Iy7SSQAVizf5fVLk9znPZy6+7/u/kSbdWDTA8CZYnm27rZ+6zobz9w9HVTVk5L8hyS3Jrlty9Wd5K/t+lAbJhKwc67f9ACcslcleXl3v2bTg+wVXm6CNVkeNvn8JI/p7pdX1QOT/K2zZYfn6aiqPpnk4u4+sulZ9gqHwMIaVNUTkvxRkn+Q5MVJsjxB6yur6js3ORvHdHOSx296iL1EJGA9XpPkZ7r7GUnuXLn8p5NctZmR2IYfTvLaqnrcpgfZK7zcBGuwPHP387v7SFXd2t0XLS+vJLefLWfrnm6q6s1Jnpjki5O8L/f9exJ2XAM74pYkj8niB02tXP6UJB/eyERsx7/Y9AB7jUjAevyzJG+uqiuyPCy2qh6f5JokP7vJwTimZx/n+t/elSn2EJGANejun1+eufv2JOdX1S1Jzk3y4919zWan4xhqy9cXJ/m6JAeT/NTuj7N59knAGlXVuUmetPzyPVvfXZS9r6rOSfK6JH/Q3T+z6Xl2m0jAmlTV85L8/SSP6u4nLM+TeF2S7+ruWzc7HSeiqh6c5MbufuSmZ9ltDoGFNaiq78/iz1y+JckjkrvPk3hrkh/f4GicnM8mOX/TQ2yClQSsQVV9MMnXd/d7txwCe/8k7+/uL9jogBxVVT3kKBc/JIv9EXd09wt2eaSNs+Ma1uPiJP/rKJd3jvLuouwZ/y/3fZPG27JYAf7Q7o+zeV5ugvX43STfvvx89YiZb03yzt0fh216eJJrk3wqi+etkjwwyXdk8c6wZx0rCViPq5L8VlU9Pcn+qvq6JH81yeVJLtvoZBzLL2QRhmdmy9nWZyv7JGCHVNXzk1zb3V1VfyfJu7J4c79nLDe5IclPd/cHNjUjx1ZVtyb5S919x6Zn2StEAnZIVd3W3RcuP7+9u+17OM1U1fuTPLO7b9n0LHuFl5tg5/xJVb08yU1JDlbVd0wbdvfrd28sTsBLk/zrqnpBd39608PsBSKxxw2H5B1Vd390nbNwXN+W5Puz2Pewb/nfo+kkIrFHHOXPzj4hyQer6p3ZsrP6bPyzs15u2uOq6kgW/4C3vqfMVt3d+3dhJLahqm7qbn+T4DRwIn8Eqrtft85Z9iKRAGDkPAkARiJxGquqKzc9A9vn+Tr9eM5E4nR31v8DPs14vk4/Z/1zJhIAjM74HdcXXrS/H/6FZ+aRvrfdejgXXnTmHdB0bp2Z/yY/8YkjufjiM/P3sgNn6O+bH/vE4Tz44jPv/7EPfuhQPn7r4eMdMZnkLDhP4uFfeCCvf+vDNj0GJ+CRBw5tegRO0IP2f96mR+AEPO15H9r2tmdm/gHYESIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIw2Homq+rGqesqm5wDgvjYeiSTPTvLA1Quq6pKq+j8bmgeApbVGoqp+saquON5mR7nscJLPrWEkAE7AulcSB5LsP842R4tEJ6mqOlhV95mxqvZV1dFuB8AOOrDpAQadRcD+MElX1ZEkR3JPdPYneVmSazY2IcBZYC9E4j4rgu7+UJLHbGAWAFbshR3XleRFVfXLVfVVVXXe0V5iOqE7rLqyqm6sqhtvu/XwDo0JcPbZ1ZVEVV2Q5GHLj4cmecfyqrcn+YPlPO9LcmdV/UWSz2axA/vIyrwHkhxc+fyK7n7n6uN099VJrk6Sxz/53F7jtwRwRtuNSLyiqn4kyV8k+ViSDy8/Pp7k/lmsJP57d1+/3P6RuzATANuw7khcleTvdfd4OOt0lFJVXZvFKuGT6xoOgGNbayS6+9OncPPLcvzDZwFYo72y43q6/NBuDgLAve2FQ2CTxXkRWx1M8p6qOpTFGdhHsgjHXTutX9ndr9y9EQHOPnshEl/d3feJRHefM91guR9jL6yCAM5oG4/E0QKxzds4AQJgzfw2DsBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMqrs3PcNaXVAX9VfUczY9BsCe8e6+Lrf3rbWdba0kABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwGhPRKKqXlZVz970HADc28YiUVX/sqp+ePnlJUm+4ARu+86q+pq1DAbA3Q6s886r6mlJHtHd1y6//sokX9jdv5HkcO6J1OEkR5bbvDfJOUnuXF63L8n5SS7t7j9dXvbn65wbgIW1RiLJk5N8TZJrl19/SZLnJLkrEnc5svL5E7v77q+ran+S25N8dGWb1dsCsCbrjsThJJdU1bclqSTPyj0rhNUw9N2frARi6cuS/HF33xkAdtW690kcziIO5yU5mMXLSIdXrrvLsQLw7blnJQLALlr3SuJIkpu7+98kSVXtS/L05XWHk7ywqr4nyUOTXLf1xlX1kCSXJ3nq1qtWtqnu7i23uzLJlUlyXu6/I98IwNlo3SuJ2vL16s7qSvJPuvtxSX5tuP2rk7ymuz98lOveWFUfSPLirVd099XdfWl3X3ow557k6ACseyVxIPcO0cGVx9y38vnWmKSqXpHkEUlecJT7rSSXd/f1OzYpAPex7kjcmeRrq+ojy687yZuWn1cW0bjr831JUlUPzWIF8cgkz+3uQ2ueEYDBWiPR3W9I8obh6n25JxL7khyoqkcneVeSX03yd7t7Oh/iPisPAHbeulcSx7J1JXGgu2+uqid29y0bnAuApY1Forv/4cqXdwdjm4GwkgDYBZtcSdytu684we0vW9csANxjT7wLLAB7k0gAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMTjkSVbW/qs6pqvOq6pydGGobj/n+qnrsbjwWwNnswHY3rKqbkjwkyZ1JPpfkUJIjSSrJ/iSfn+RXkvzAym3ekuTpy+0/u/zIcvuDSc5N8lvd/e0rt9mfpLv7yJbH39/dh5dfHknyF9v+LgE4KduORHc/7ljXV9WLsgjFqm9OcmTlh/vRbrd1NfOLSb6hqj6eRVSOJLlrhfLUle3G+wRgZ+zkPokLkvzZ6gXdfai7D1fVDVX1NXddXlXPr6r/stzmyL3vJkeSvLS7n9Tdl3b307r7qd29Goja4dkBOIqT+kFbVQ+vqh/ZcvHDknxouMnv596rgEcned+w7ZEsXtI65ghZvGQFwBqd7G/jFyb5oS2XPTbJe1YvWO5fSJJ3JfmylauemORtw30fzvEjkSTvqKo/OdoO7Kq6sqpurKobD929GwSAE7XtfRJbdFZ+kFdVZRGJm7Zs9ytVdeld21fVjVmsAi5I8lVV9cok7+rub1u5zXbCVUme1d0fOOpw3VcnuTpJLqiLenvfEgBbnWwkknvvON6X5Ju6+3OrG3T3t57E/e7bOldVnZ9FhJ6c5DeyiMR2VhsAnIJTicTdt10evfTO5YriQJI7u/tkf4M/mOQnqurFST7vrodIcnOSP0zy5iwiYYUAsGbHjcTy5aLfTXJHkk9nsWP5QJL7VdUtWRyees7ysn1Z/AB/TlX9chbnUnwui3MaDmWx+qgsQnBweZtzlp9/Q3f/zyQ/luSnk3wmyW3d/ZmjzJTl/QCwRttZSfxeknO7+9B273S5ovjL3f3n29h2X5LzsjzRrrv/dDsPEZEAWLvjRmJ5HsPWcxmOd5tOctxArNz/fVYLx+FEOoBdcCr7JDamu79k0zMAnA2ctQzASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADDakUhU1dOq6oqduK/jPM7PVtUD1v04ACycciSqal+SH03yplMf57jelOSlu/A4AGRnVhLfk+RN3X37DtzXMXX3u5NcUFVfuu7HAuAUI1FVFyX5xiSv35lxtuXHkvzULj4ewFnrVFcSP5nkJ7q777qgqi6uqtdV1a1V9cmqenVVnbNy/WVVdWNVfaaq/qiqvm/luvstt/9IVX2qqn63qp6/+oDd/dEk11fVC05xdgCO46QjUVV/Jcn+7r5x5bKDSd6W5I4kX5TkMUm+JMnLltc/K8mvJ3l5ks9P8oIkL6yqH1zexU8meViSxye5MItVw9cf5eFfleSKaSd2VV25DNGNh/LZk/0WAc56tbIIOLEbVl2b5Ie6+49XLrs8yUuSPLG7jywve2ySG5M8MMlvJnlbd79i5TZfneTaJA9N8oYkf9LdL9zG41+e5GHd/U+Ptd0FdVF/RT3nRL89gDPWu/u63N631na2PZWXm16bZOthr89IckmSW6vqtqq6Lcm7kxxM8gVJvjLJf9xym3csr39sFiuOb6mq36+ql1bVpcd4/G9N8qunMD8Ax3HSkejutyb50qp65Jar/lV3X7jl47yVFUdvuZ9e+fwPkzw6yQ8kqSRvrKq3VdX+1dtU1Tcn+Z3u/r8nOz8Ax3eqO65flHsfaXRDkmdv3aiqHrDcX3FDkudtue6ZST6X5Kaqqu4+0t3/rbt/IslTslidPHVl+/tlcdjtK09xdgCO45Qi0d03J7m56u4X/X8tSVXVP18e5XSgqp6b5Pok52Zx0t2Lquobl9c9LckvJfnR7r4zyTVV9S1VdU5VHchix/ankty08rAvTvJz3X3oVGYH4Ph24mS6n0lyVVUdWP7g/tok5yf530k+lsVLR9/V3Z9engz3/CxWILdnsU/hZ7v7F5b39YYsVgkfTvLRLPY7XNbddyRJVT0qyRd193U7MDcAx3HSRzfd606q/maSi7v7mlMf6ZiP8/NJfqq7P7zd2zi6CeDeTuTopgM78YDd/ZbVE+bW6Kru/twuPA4A2cG3Ct+NH94CAbC7/D0JAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYHdj0AOtQVVcmuTJJzsv9NzwNwOnrjFxJdPfV3X1pd196MOduehyA09YZGQkAdoZIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHASCQAGIkEACORAGAkEgCMRAKAkUgAMBIJAEYiAcBIJAAYiQQAI5EAYCQSAIxEAoCRSAAwEgkARiIBwEgkABiJBAAjkQBgJBIAjEQCgJFIADASCQBGIgHAqLp70zOsVVV9LMkfb3qONXlQko9vegi2zfN1+jlTn7NHdveDt7PhGR+JM1lV3djdl256DrbH83X68Zx5uQmAYxAJAEYicXq7etMDcEI8X6efs/45s08CgJGVBAAjkQBgJBIAjEQCgJFIADD6/3jjmhwcRW8UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sentence = 'I feel hungry'\n",
    "# sentence = 'tensorflow is a framework for deep learning'\n",
    "\n",
    "translate(sentence, encoder, decoder, source2idx, target2idx, s_max_len, t_max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}