english-hindi-NMT-small-dataset-bidirectional-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/arushi/Neural-Machine-Translation/anoopkunchukuttan-indic_nlp_library-eccde81/src/indicnlp/script/indic_scripts.py:116: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  ALL_PHONETIC_VECTORS= ALL_PHONETIC_DATA.ix[:,PHONETIC_VECTOR_START_OFFSET:].as_matrix()\n",
      "/home/arushi/Neural-Machine-Translation/anoopkunchukuttan-indic_nlp_library-eccde81/src/indicnlp/script/indic_scripts.py:117: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  TAMIL_PHONETIC_VECTORS=TAMIL_PHONETIC_DATA.ix[:,PHONETIC_VECTOR_START_OFFSET:].as_matrix()\n",
      "/home/arushi/Neural-Machine-Translation/anoopkunchukuttan-indic_nlp_library-eccde81/src/indicnlp/script/english_script.py:113: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  ENGLISH_PHONETIC_VECTORS=ENGLISH_PHONETIC_DATA.ix[:,PHONETIC_VECTOR_START_OFFSET:].as_matrix()\n"
     ]
    }
   ],
   "source": [
    "#from __future__ import unicode_literals, print_function, division\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from string import digits\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.layers import Input, LSTM, Embedding, Dense, CuDNNLSTM, TimeDistributed, Flatten, Dropout, Bidirectional\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model, Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from numpy import array, argmax\n",
    "from numpy.random import rand, shuffle\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "import scipy\n",
    "import statsmodels\n",
    "import sklearn\n",
    "import tensorflow\n",
    "import keras\n",
    "from io import open\n",
    "import unicodedata\n",
    "import random\n",
    "import math\n",
    "import os\n",
    "import time\n",
    "import re\n",
    "import sys\n",
    "from unicodedata import normalize\n",
    "from numpy import array\n",
    "from pickle import dump, load\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.layers import RepeatVector, TimeDistributed\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "INDIC_NLP_LIB_HOME=r\"/home/arushi/Neural-Machine-Translation/anoopkunchukuttan-indic_nlp_library-eccde81/src\"\n",
    "INDIC_NLP_RESOURCES=r\"/home/arushi/Neural-Machine-Translation/indic_nlp_resources-master\"\n",
    "sys.path.append(r'{}'.format(INDIC_NLP_LIB_HOME))\n",
    "from indicnlp import common\n",
    "common.set_resources_path(INDIC_NLP_RESOURCES)\n",
    "from indicnlp import loader\n",
    "loader.load()\n",
    "from indicnlp.normalize.indic_normalize import IndicNormalizerFactory\n",
    "from indicnlp.tokenize import indic_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc(filename):\n",
    "    file = open(filename, mode='rt', encoding='utf-8')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "def to_pairs(english_text, hindi_text):\n",
    "    english_lines = english_text.strip().split('\\n')\n",
    "    hindi_lines = hindi_text.strip().split('\\n')\n",
    "    pairs = []\n",
    "    for i in range(len(hindi_lines)):\n",
    "        pairs.append([])\n",
    "        pairs[i].append(pre_process_english_sentence(english_lines[i]))\n",
    "        pairs[i].append(pre_process_hindi_sentence(hindi_lines[i]))\n",
    "    return pairs\n",
    "\n",
    "def clean_text(text):\n",
    "    text = text.replace(u',','')\n",
    "    text = text.replace(u'\"','')\n",
    "    text = text.replace(u'\"','')\n",
    "    text = text.replace(u\"‘‘\",'')\n",
    "    text = text.replace(u\"’’\",'')\n",
    "    text = text.replace(u\"''\",'')\n",
    "    text = text.replace(u\"।\",'')\n",
    "    text=text.replace(u',','')\n",
    "    text=text.replace(u'\"','')\n",
    "    text=text.replace(u'(','')\n",
    "    text=text.replace(u')','')\n",
    "    text=text.replace(u'\"','')\n",
    "    text=text.replace(u':','')\n",
    "    text=text.replace(u\"'\",'')\n",
    "    text=text.replace(u\"‘‘\",'')\n",
    "    text=text.replace(u\"’’\",'')\n",
    "    text=text.replace(u\"''\",'')\n",
    "    text=text.replace(u\".\",'')\n",
    "    text=text.replace(u\"-\",'')\n",
    "    text=text.replace(u\"।\",'')\n",
    "    text=text.replace(u\"?\",'')\n",
    "    text=text.replace(u\"\\\\\",'')\n",
    "    text=text.replace(u\"_\",'')\n",
    "    text= re.sub(\"'\", '', text)\n",
    "    text=re.sub('[0-9+\\-*/.%]', '', text)\n",
    "    text=text.strip()\n",
    "    text=re.sub(' +', ' ',text)\n",
    "    exclude = set(string.punctuation)\n",
    "    text= ''.join(ch for ch in text if ch not in exclude)\n",
    "    return text\n",
    "\n",
    "def pre_process_english_sentence(line):\n",
    "    line = line.lower()\n",
    "    line = clean_text(line)\n",
    "    re_print = re.compile('[^%s]' % re.escape(string.printable))\n",
    "    line = normalize('NFD', line).encode('ascii', 'ignore')\n",
    "    line = line.decode('UTF-8')\n",
    "    line = line.split()\n",
    "    line = [re_print.sub('', w) for w in line]\n",
    "    line = [word for word in line if word.isalpha()]\n",
    "    line = ' '.join(line)\n",
    "    return line\n",
    "\n",
    "def pre_process_hindi_sentence(line):\n",
    "    line=re.sub('[a-zA-Z]', '', line)\n",
    "    line = clean_text(line)\n",
    "    remove_nuktas = False\n",
    "    factory = IndicNormalizerFactory()\n",
    "    normalizer = factory.get_normalizer(\"hi\",remove_nuktas)\n",
    "    line = normalizer.normalize(line)\n",
    "    tokens = list()\n",
    "    for t in indic_tokenize.trivial_tokenize(line):\n",
    "        tokens.append(t)\n",
    "    line = tokens\n",
    "    line = [word for word in line if not re.search(r'\\d', word)]\n",
    "    line = ' '.join(line)\n",
    "    line = 'START_ '+ line + ' _END'\n",
    "    return (line)\n",
    "\n",
    "english_text = load_doc('English_')\n",
    "hindi_text = load_doc('Hindi_')\n",
    "pairs = to_pairs(english_text, hindi_text)\n",
    "df = pd.DataFrame(pairs)\n",
    "df.columns = [\"eng\", \"mar\"]\n",
    "#df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "915\n",
      "1035\n"
     ]
    }
   ],
   "source": [
    "all_eng_words=set()\n",
    "for eng in lines.eng:\n",
    "    for word in eng.split():\n",
    "        if word not in all_eng_words:\n",
    "            all_eng_words.add(word)\n",
    "\n",
    "# Vocabulary of French \n",
    "all_marathi_words=set()\n",
    "for mar in lines.mar:\n",
    "    for word in mar.split():\n",
    "        if word not in all_marathi_words:\n",
    "            all_marathi_words.add(word)\n",
    "\n",
    "#print (all_eng_words)\n",
    "print (len(all_eng_words))\n",
    "print (len(all_marathi_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n",
      "47\n"
     ]
    }
   ],
   "source": [
    "# Max Length of source sequence\n",
    "lenght_list=[]\n",
    "for l in lines.eng:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_src = np.max(lenght_list)\n",
    "print (max_length_src)\n",
    "\n",
    "# Max Length of target sequence\n",
    "lenght_list=[]\n",
    "for l in lines.mar:\n",
    "    lenght_list.append(len(l.split(' ')))\n",
    "max_length_tar = np.max(lenght_list)\n",
    "print (max_length_tar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1036"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_words = sorted(list(all_eng_words))\n",
    "target_words = sorted(list(all_marathi_words))\n",
    "num_encoder_tokens = len(all_eng_words)\n",
    "num_decoder_tokens = len(all_marathi_words)\n",
    "num_encoder_tokens, num_decoder_tokens\n",
    "num_encoder_tokens += 1\n",
    "num_decoder_tokens += 1 # For zero padding\n",
    "num_decoder_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict([(word, i+1) for i, word in enumerate(input_words)])\n",
    "target_token_index = dict([(word, i+1) for i, word in enumerate(target_words)])\n",
    "reverse_input_char_index = dict((i, word) for word, i in input_token_index.items())\n",
    "reverse_target_char_index = dict((i, word) for word, i in target_token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((209,), (24,))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train - Test Split\n",
    "X, y = lines.eng, lines.mar\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_pickle('X_train.pkl')\n",
    "X_test.to_pickle('X_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(X = X_train, y = y_train, batch_size = 12):\n",
    "    while True:\n",
    "        # range(start, stop, step)\n",
    "        for j in range(0, len(X), batch_size):\n",
    "            encoder_input_data = np.zeros((batch_size, max_length_src),dtype='float32')\n",
    "            decoder_input_data = np.zeros((batch_size, max_length_tar),dtype='float32')\n",
    "            decoder_target_data = np.zeros((batch_size, max_length_tar, num_decoder_tokens),dtype='float32')\n",
    "            for i, (input_text, target_text) in enumerate(zip(X[j:j+batch_size], y[j:j+batch_size])):\n",
    "                for t, word in enumerate(input_text.split()):\n",
    "                    encoder_input_data[i, t] = input_token_index[word] # encoder input seq\n",
    "                for t, word in enumerate(target_text.split()):\n",
    "                    if t<len(target_text.split())-1:\n",
    "                        decoder_input_data[i, t] = target_token_index[word] # decoder input seq\n",
    "                    if t>0:\n",
    "                        # decoder target sequence (one hot encoded)\n",
    "                        # does not include the START_ token\n",
    "                        # Offset by one timestep\n",
    "                        decoder_target_data[i, t - 1, target_token_index[word]] = 1.\n",
    "            yield([encoder_input_data, decoder_input_data], decoder_target_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/arushi/.local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /home/arushi/.local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "latent_dim = 1024\n",
    "vec_len = 300 # Length of the vector that we willl get from the embedding layer\n",
    "dropout_rate = 0.2 # Rate of the dropout layers\n",
    "encoder_inputs = Input(shape=(None,))\n",
    "#enc_emb = Embedding(num_encoder_tokens, vec_len, mask_zero = True)(encoder_inputs)\n",
    "#enc_emb = Embedding(num_encoder_tokens, vec_len)(encoder_inputs)\n",
    "enc_emb = Embedding(input_dim = num_encoder_tokens, output_dim = vec_len, mask_zero = True)(encoder_inputs)\n",
    "\n",
    "encoder_dropout = (TimeDistributed(Dropout(rate = dropout_rate)))(enc_emb)\n",
    "encoder_lstm1 = Bidirectional(LSTM(latent_dim, return_sequences=True), merge_mode='sum')(encoder_dropout)\n",
    "encoder_lstm2 = Bidirectional(LSTM(latent_dim, return_sequences=True), merge_mode='sum')(encoder_lstm1)\n",
    "encoder_lstm3 = Bidirectional(LSTM(latent_dim, return_sequences=True), merge_mode='sum')(encoder_lstm2)\n",
    "encoder_LSTM4_layer = LSTM(latent_dim, return_state=True)\n",
    "encoder_outputs, state_h, state_c = encoder_LSTM4_layer(encoder_lstm3)\n",
    "encoder_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "dec_emb_layer = Embedding(num_decoder_tokens, vec_len, mask_zero = True)\n",
    "dec_emb = dec_emb_layer(decoder_inputs)\n",
    "decoder_dropout = (TimeDistributed(Dropout(rate = dropout_rate)))(dec_emb)\n",
    "# We set up our decoder to return full output sequences,\n",
    "# and to return internal states as well. We don't use the\n",
    "# return states in the training model, but we will use them in inference.\n",
    "\n",
    "\n",
    "decoder_LSTM_layer = LSTM(latent_dim, return_sequences=True)\n",
    "decoder_LSTM = decoder_LSTM_layer(decoder_dropout, initial_state = encoder_states)\n",
    "\n",
    "decoder_LSTM_2_layer = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "decoder_outputs,_,_ = decoder_LSTM_2_layer(decoder_LSTM)\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "\n",
    "# Define the model that will turn\n",
    "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 300)    274800      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 300)    0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, None, 1024)   10854400    time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, None, 1024)   16785408    bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 300)    310800      input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, None, 1024)   16785408    bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistrib (None, None, 300)    0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                   [(None, 1024), (None 8392704     bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_5 (LSTM)                   (None, None, 1024)   5427200     time_distributed_2[0][0]         \n",
      "                                                                 lstm_4[0][1]                     \n",
      "                                                                 lstm_4[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                   [(None, None, 1024), 8392704     lstm_5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 1036)   1061900     lstm_6[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 68,285,324\n",
      "Trainable params: 68,285,324\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n",
    "\n",
    "# Define a checkpoint callback :\n",
    "checkpoint_name = 'Weights-{epoch:03d}--{val_loss:.5f}.hdf5' \n",
    "checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/arushi/.local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /home/arushi/.local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Epoch 1/1\n",
      "17/17 [==============================] - 311s 18s/step - loss: 6.4806 - acc: 0.0431 - val_loss: 6.1815 - val_acc: 0.0453\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fb34f80fe80>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_samples = len(X_train)\n",
    "val_samples = len(X_test)\n",
    "batch_size = 12\n",
    "epochs = 1\n",
    "\n",
    "# Compile the model\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc'])\n",
    "model.fit_generator(generator = generate_batch(X_train, y_train, batch_size = batch_size),\n",
    "                    steps_per_epoch = train_samples//batch_size,\n",
    "                    epochs=epochs,\n",
    "                    validation_data = generate_batch(X_test, y_test, batch_size = batch_size),\n",
    "                    validation_steps = val_samples//batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save the model\n",
    "model.save_weights('nmt_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the model\n",
    "model.load_weights('nmt_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode the input sequence to get the \"thought vectors\"\n",
    "encoder_model = Model(encoder_inputs, encoder_states)\n",
    "\n",
    "# Decoder setup\n",
    "# Below tensors will hold the states of the previous time step\n",
    "decoder_state_input_h = Input(shape=(latent_dim,))\n",
    "decoder_state_input_c = Input(shape=(latent_dim,))\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "dec_emb2= dec_emb_layer(decoder_inputs) # Get the embeddings of the decoder sequence\n",
    "decoder_dropout2 = (TimeDistributed(Dropout(rate = dropout_rate)))(dec_emb2)\n",
    "decoder_LSTM2 = decoder_LSTM_layer(decoder_dropout2, initial_state = decoder_states_inputs)\n",
    "decoder_outputs2, state_h2, state_c2 = decoder_LSTM_2_layer(decoder_LSTM2)\n",
    "decoder_states2 = [state_h2, state_c2]\n",
    "decoder_outputs2 = decoder_dense(decoder_outputs2) #A dense softmax layer to generate prob dist. over the target vocabulary\n",
    "\n",
    "\n",
    "# Final decoder model\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + decoder_states_inputs,\n",
    "    [decoder_outputs2] + decoder_states2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1,1))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0] = target_token_index['START_']\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += ' '+sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '_END' or\n",
    "           len(decoded_sentence) > 90):\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1,1))\n",
    "        target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_gen = generate_batch(X_train, y_train, batch_size = 1)\n",
    "k=-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_gen = generate_batch(X_test, y_test, batch_size = 1)\n",
    "k=-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-1: 0.000000\n",
      "BLEU-2: 0.000000\n",
      "BLEU-3: 0.000000\n",
      "BLEU-4: 0.000000\n",
      "Individual 1-gram: 0.000000\n",
      "Individual 2-gram: 0.000000\n",
      "Individual 3-gram: 0.000000\n",
      "Individual 4-gram: 0.000000\n"
     ]
    }
   ],
   "source": [
    "train_gen = generate_batch(X_train, y_train, batch_size = 1)\n",
    "actual, predicted = list(), list()\n",
    "for k in range(len(y_train)):\n",
    "    (input_seq, actual_output), _ = next(train_gen)\n",
    "    decoded_sentence = decode_sequence(input_seq)\n",
    "    actual.append(y_train[k:k+1].values[0][6:-4].split())\n",
    "    predicted.append(decoded_sentence[:-4].split())\n",
    "\n",
    "# calculate BLEU score\n",
    "print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
    "print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n",
    "print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.33, 0.33, 0.33, 0)))\n",
    "print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n",
    "\n",
    "# n-gram individual BLEU\n",
    "print('Individual 1-gram: %f' % corpus_bleu(actual, predicted, weights=(1, 0, 0, 0)))\n",
    "print('Individual 2-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 1, 0, 0)))\n",
    "print('Individual 3-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 0, 1, 0)))\n",
    "print('Individual 4-gram: %f' % corpus_bleu(actual, predicted, weights=(0, 0, 0, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU-1: 0.000000\n",
      "BLEU-2: 0.000000\n",
      "BLEU-3: 0.000000\n",
      "BLEU-4: 0.000000\n",
      "Individual 1-gram: 0.000000\n",
      "Individual 2-gram: 0.000000\n",
      "Individual 3-gram: 0.000000\n",
      "Individual 4-gram: 0.000000\n"
     ]
    }
   ],
   "source": [
    "val_gen = generate_batch(X_test, y_test, batch_size = 1)\n",
    "actual_val, predicted_val = list(), list()\n",
    "for k in range(len(y_test)):\n",
    "    (input_seq, actual_output), _ = next(val_gen)\n",
    "    decoded_sentence = decode_sequence(input_seq)\n",
    "    actual_val.append(y_test[k:k+1].values[0][6:-4].split())\n",
    "    predicted_val.append(decoded_sentence[:-4].split())\n",
    "    \n",
    "\n",
    "# calculate BLEU score\n",
    "print('BLEU-1: %f' % corpus_bleu(actual_val, predicted_val, weights=(1.0, 0, 0, 0)))\n",
    "print('BLEU-2: %f' % corpus_bleu(actual_val, predicted_val, weights=(0.5, 0.5, 0, 0)))\n",
    "print('BLEU-3: %f' % corpus_bleu(actual_val, predicted_val, weights=(0.33, 0.33, 0.33, 0)))\n",
    "print('BLEU-4: %f' % corpus_bleu(actual_val, predicted_val, weights=(0.25, 0.25, 0.25, 0.25)))\n",
    "\n",
    "# n-gram individual BLEU\n",
    "print('Individual 1-gram: %f' % corpus_bleu(actual_val, predicted_val, weights=(1, 0, 0, 0)))\n",
    "print('Individual 2-gram: %f' % corpus_bleu(actual_val, predicted_val, weights=(0, 1, 0, 0)))\n",
    "print('Individual 3-gram: %f' % corpus_bleu(actual_val, predicted_val, weights=(0, 0, 1, 0)))\n",
    "print('Individual 4-gram: %f' % corpus_bleu(actual_val, predicted_val, weights=(0, 0, 0, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}