Attentive_Seq2Seq2_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Import the Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "import string\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "# from tensorflow.keras.utils import np_utils\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.layers import Input,LSTM,Embedding,Dense,Concatenate,TimeDistributed,Bidirectional,Attention, Lambda\n",
    "from tensorflow.keras.models import Model, Sequential,load_model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import tensorflow.keras.backend as k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Developing the Attentive Layer\n",
    "We will use this as a last layer to enchance the accuracy of output prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-16 04:52:15.016162: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.keras import backend as K\n",
    "\n",
    "logger = tf.get_logger()\n",
    "\n",
    "class AttentionLayer(tf.keras.layers.Layer):\n",
    "    \"\"\"\n",
    "    This class implements Bahdanau attention (https://arxiv.org/pdf/1409.0473.pdf).\n",
    "    There are three sets of weights introduced W_a, U_a, and V_a\n",
    "     \"\"\"\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert isinstance(input_shape, list)\n",
    "        # Create a trainable weight variable for this layer.\n",
    "\n",
    "        self.W_a = self.add_weight(name='W_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.U_a = self.add_weight(name='U_a',\n",
    "                                   shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.V_a = self.add_weight(name='V_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], 1)),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "\n",
    "        super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end\n",
    "\n",
    "    def call(self, inputs):\n",
    "        \"\"\"\n",
    "        inputs: [encoder_output_sequence, decoder_output_sequence]\n",
    "        \"\"\"\n",
    "        assert type(inputs) == list\n",
    "        encoder_out_seq, decoder_out_seq = inputs\n",
    "\n",
    "        logger.debug(f\"encoder_out_seq.shape = {encoder_out_seq.shape}\")\n",
    "        logger.debug(f\"decoder_out_seq.shape = {decoder_out_seq.shape}\")\n",
    "\n",
    "        def energy_step(inputs, states):\n",
    "            \"\"\" Step function for computing energy for a single decoder state\n",
    "            inputs: (batchsize * 1 * de_in_dim)\n",
    "            states: (batchsize * 1 * de_latent_dim)\n",
    "            \"\"\"\n",
    "\n",
    "            logger.debug(\"Running energy computation step\")\n",
    "\n",
    "            if not isinstance(states, (list, tuple)):\n",
    "                raise TypeError(f\"States must be an iterable. Got {states} of type {type(states)}\")\n",
    "\n",
    "            encoder_full_seq = states[-1]\n",
    "\n",
    "            \"\"\" Computing S.Wa where S=[s0, s1, ..., si]\"\"\"\n",
    "            # <= batch size * en_seq_len * latent_dim\n",
    "            W_a_dot_s = K.dot(encoder_full_seq, self.W_a)\n",
    "\n",
    "            \"\"\" Computing hj.Ua \"\"\"\n",
    "            U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim\n",
    "\n",
    "            logger.debug(f\"U_a_dot_h.shape = {U_a_dot_h.shape}\")\n",
    "\n",
    "            \"\"\" tanh(S.Wa + hj.Ua) \"\"\"\n",
    "            # <= batch_size*en_seq_len, latent_dim\n",
    "            Ws_plus_Uh = K.tanh(W_a_dot_s + U_a_dot_h)\n",
    "\n",
    "            logger.debug(f\"Ws_plus_Uh.shape = {Ws_plus_Uh.shape}\")\n",
    "\n",
    "            \"\"\" softmax(va.tanh(S.Wa + hj.Ua)) \"\"\"\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.squeeze(K.dot(Ws_plus_Uh, self.V_a), axis=-1)\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.softmax(e_i)\n",
    "\n",
    "            logger.debug(f\"ei.shape = {e_i.shape}\")\n",
    "\n",
    "            return e_i, [e_i]\n",
    "\n",
    "        def context_step(inputs, states):\n",
    "            \"\"\" Step function for computing ci using ei \"\"\"\n",
    "\n",
    "            logger.debug(\"Running attention vector computation step\")\n",
    "\n",
    "            if not isinstance(states, (list, tuple)):\n",
    "                raise TypeError(f\"States must be an iterable. Got {states} of type {type(states)}\")\n",
    "\n",
    "            encoder_full_seq = states[-1]\n",
    "\n",
    "            # <= batch_size, hidden_size\n",
    "            c_i = K.sum(encoder_full_seq * K.expand_dims(inputs, -1), axis=1)\n",
    "\n",
    "            logger.debug(f\"ci.shape = {c_i.shape}\")\n",
    "\n",
    "            return c_i, [c_i]\n",
    "\n",
    "        # we don't maintain states between steps when computing attention\n",
    "        # attention is stateless, so we're passing a fake state for RNN step function\n",
    "        fake_state_c = K.sum(encoder_out_seq, axis=1)\n",
    "        fake_state_e = K.sum(encoder_out_seq, axis=2)  # <= (batch_size, enc_seq_len, latent_dim\n",
    "\n",
    "        \"\"\" Computing energy outputs \"\"\"\n",
    "        # e_outputs => (batch_size, de_seq_len, en_seq_len)\n",
    "        last_out, e_outputs, _ = K.rnn(\n",
    "            energy_step, decoder_out_seq, [fake_state_e], constants=[encoder_out_seq]\n",
    "        )\n",
    "\n",
    "        \"\"\" Computing context vectors \"\"\"\n",
    "        last_out, c_outputs, _ = K.rnn(\n",
    "            context_step, e_outputs, [fake_state_c], constants=[encoder_out_seq]\n",
    "        )\n",
    "\n",
    "        return c_outputs, e_outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        \"\"\" Outputs produced by the layer \"\"\"\n",
    "        return [\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))\n",
    "        ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Downloading the Gigaword Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset(\"gigaword\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Renaming the column names and saving it in csv format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = dataset['train'].to_pandas()\n",
    "df.rename(columns={'document':'text','summary':'headlines'},inplace=True)\n",
    "df.to_csv(r'.\\gigaword_train.csv',index=False)\n",
    "\n",
    "df = dataset['test'].to_pandas()\n",
    "df.rename(columns={'document':'text','summary':'headlines'},inplace=True)\n",
    "df.to_csv(r'.\\gigaword_test.csv',index=False)\n",
    "\n",
    "df = dataset['validation'].to_pandas()\n",
    "df.rename(columns={'document':'text','summary':'headlines'},inplace=True)\n",
    "df.to_csv(r'.\\gigaword_valid.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the saved CSV dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=pd.read_csv('./gigaword_train.csv')\n",
    "df_test=pd.read_csv('./gigaword_test.csv')\n",
    "df_validate=pd.read_csv('./gigaword_valid.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Printing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>australia current account deficit shrunk recor...</td>\n",
       "      <td>sostok australian current account deficit narr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>least two people killed suspected bomb attack ...</td>\n",
       "      <td>sostok at least two dead in southern philippin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>australian shares closed percent monday follow...</td>\n",
       "      <td>sostok australian stocks close down percent eo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>south korea nuclear envoy kim sook urged north...</td>\n",
       "      <td>sostok envoy urges north korea to restart nucl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>south korea monday announced sweeping tax refo...</td>\n",
       "      <td>sostok skorea announces tax cuts to stimulate ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  australia current account deficit shrunk recor...   \n",
       "1  least two people killed suspected bomb attack ...   \n",
       "2  australian shares closed percent monday follow...   \n",
       "3  south korea nuclear envoy kim sook urged north...   \n",
       "4  south korea monday announced sweeping tax refo...   \n",
       "\n",
       "                                             summary  \n",
       "0  sostok australian current account deficit narr...  \n",
       "1  sostok at least two dead in southern philippin...  \n",
       "2  sostok australian stocks close down percent eo...  \n",
       "3  sostok envoy urges north korea to restart nucl...  \n",
       "4  sostok skorea announces tax cuts to stimulate ...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Pre-processing Phase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this stage, we will remove all unneeded symbols, characters, and other elements from the text that do not affect the problem's goal.\n",
    "\n",
    "Here is the dictionary that we will use for expanding the contractions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
    "                           \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n",
    "                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n",
    "                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n",
    "                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
    "                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n",
    "                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
    "                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n",
    "                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
    "                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
    "                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n",
    "                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n",
    "                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
    "                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n",
    "                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n",
    "                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n",
    "                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n",
    "                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
    "                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
    "                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n",
    "                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n",
    "                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
    "                           \"you're\": \"you are\", \"you've\": \"you have\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will perform the below pre-processing tasks for our data:\n",
    "\n",
    "1.Convert everything to lowercase\n",
    "\n",
    "2.Contraction mapping\n",
    "\n",
    "3.Remove (‘s)\n",
    "\n",
    "4.Remove any text inside the parenthesis ( )\n",
    "\n",
    "5.Eliminate punctuations and special characters\n",
    "\n",
    "6.Remove stopwords\n",
    "\n",
    "7.Remove single characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = set(stopwords.words('english')) \n",
    "\n",
    "def text_cleaner(text,num):\n",
    "    # lower\n",
    "    newString = text.lower()\n",
    "    # Remove any text inside the parenthesis\n",
    "    newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
    "    # remove double quotes\n",
    "    newString = re.sub('\"','', newString)\n",
    "    # contraction mapping\n",
    "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])  \n",
    "    # remove 's\n",
    "    newString = re.sub(r\"'s\\b\",\"\",newString)\n",
    "    # Eliminate punctuations and special characters\n",
    "    newString = re.sub(\"[^a-zA-Z]\", \" \", newString)\n",
    "    # Remove stopwords\n",
    "    if(num==0):\n",
    "        tokens = [w for w in newString.split() if not w in stop_words]\n",
    "    else:\n",
    "        tokens=newString.split()\n",
    "    long_words=[]\n",
    "    # Remove short words\n",
    "    for i in tokens:\n",
    "        if len(i)>1:                                                 \n",
    "            long_words.append(i)   \n",
    "    return (\" \".join(long_words)).strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning the Text Column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_text = []\n",
    "cleaned_text_test=[]\n",
    "cleaned_text_valid=[]\n",
    "\n",
    "for t in df_train['text']:\n",
    "    cleaned_text.append(text_cleaner(t,0))\n",
    "\n",
    "for t in df_test['text']:\n",
    "    cleaned_text_test.append(text_cleaner(t,0))\n",
    "    \n",
    "for t in df_valid['text']:\n",
    "    cleaned_text_valid.append(text_cleaner(t,0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning the Summary Column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_summary = []\n",
    "cleaned_summary_text = []\n",
    "cleaned_summary_valid = []\n",
    "\n",
    "for t in df_train['summary']:\n",
    "    cleaned_summary.append(text_cleaner(t,1))\n",
    "    \n",
    "for t in df_test['summary']:\n",
    "    cleaned_summary_test.append(text_cleaner(t,1))\n",
    "\n",
    "for t in df_valid['summary']:\n",
    "    cleaned_summary_valid.append(text_cleaner(t,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Re-writing the cleaned text and summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['text']=cleaned_text\n",
    "df_train['summary']=cleaned_summary\n",
    "\n",
    "df_test['text']=cleaned_text_test\n",
    "df_test['summary']=cleaned_summary_test\n",
    "\n",
    "df_valid['text']=cleaned_text_valid\n",
    "df_valid['summary']=cleaned_summary_valid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Droping empty rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[1;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mdf_train\u001b[49m\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m, np\u001b[38;5;241m.\u001b[39mnan, inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m      2\u001b[0m df_train\u001b[38;5;241m.\u001b[39mdropna(axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m,inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m      4\u001b[0m df_test\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m, np\u001b[38;5;241m.\u001b[39mnan, inplace\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df_train' is not defined"
     ]
    }
   ],
   "source": [
    "df_train.replace('', np.nan, inplace=True)\n",
    "df_train.dropna(axis=0,inplace=True)\n",
    "\n",
    "df_test.replace('', np.nan, inplace=True)\n",
    "df_test.dropna(axis=0,inplace=True)\n",
    "\n",
    "df_valid.replace('', np.nan, inplace=True)\n",
    "df_valid.dropna(axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Understanding the distribution of the sequences\n",
    "\n",
    "Here, we will analyze the length of the reviews and the summary to get an overall idea about the distribution of length of the text. This will help us fix the maximum length of the sequence:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEICAYAAACqMQjAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlGklEQVR4nO3df7BV5X3v8fenoIaaGFGTEwK0mEqTSyRROVfprXe6qw2i6b3YGWO0NoDhht4J3poJt1fM5A6JPxq8M8aqtTQ0UjElQcbEwk0whBr2pPUWFI2RgHE8USwwCBEQglbtMd/7x3qObDZrnbPPYe+z9jl8XjN79trftZ71PPuctc/3edaz9jqKCMzMzPL8WtkNMDOz9uUkYWZmhZwkzMyskJOEmZkVcpIwM7NCThJmZlbIScLMzAo5SQwTkrZJ+oN22Y+ZDQ9OEmZm/SRpZNltGCxOEsOApG8AvwH8X0mHJP0vSVMl/T9Jr0j6iaRK2vY/SXpZ0vj0+qOS9kv6UN5+ynpPNvxJukHSTkm/lPSspIsl3SfplpptKpJ21LzeJunPJT0t6VVJ90rqkPRw2s8/Shqdtp0gKSRdK2l7Os7/u6T/mMq/Iumvavb9W5J+KGlv+owsl3RqXd03SHoaeDW149t17+kuSXe28uc26CLCj2HwALYBf5CWxwJ7gcvIOgIfS6/fk9bfCvwQGAVsBq7L248ffrTqAXwQ2A68P72eAPwWcB9wS812FWBHzettwAagIx3ne4AngXOBd6TjemHNPgP4m7RuGvA68A/Ae2vK/17a/qz0WTkJeA/wI+Av6+p+ChifPjtjgFeBU9P6kWl/U8r++Tbz4ZHE8PQnwJqIWBMRv4qIdcAmsqQB8CXg3cBjwE7gnlJaacezt8j+GE+SdEJEbIuInzdY9u6I2B0RO4F/AjZGxI8j4nXgIbKEUevmiHg9In5A9kf9WxGxp6b8uQAR0RUR6yLijYj4BfBV4Pfq9nVXRGyPiH+LiF1kieQTad104OWIeKJfP4k25yQxPP0m8Ik0nH5F0ivAhWQ9HyLi38l6bGcDt0fqBpkNlojoAj5H1mHZI2mFpPc3WHx3zfK/5bx+50C2T6etVqRTYAeBvwfOqNvX9rrXy8g6ZaTnbzT4HoYMJ4nho/YP/XbgGxFxas3j5IhYBCBpLLAQ+DvgdkknFezHrGUi4psRcSFZpyaA28h6+r9es9n7BrFJf5HaMTkiTiH7o6+6beo/H/8AfETS2cAfAstb3cjB5iQxfOwGPpCW/x74L5IukTRC0jvSBOA4SSIbRdwLzAF2ATcX7MesJSR9UNJFqYPyOlmP/ldk5/wvk3SapPeRjTYGy7uAQ8CB1JH6874KpFNcDwLfBB6LiH9tbRMHn5PE8PEV4Ivp1NIngRnAF4BfkI0s/pzs9/1nZJN2/zudZroWuFbSf67fj6T/ObhvwY4jJwGLgJeBl8iOyRvJTtf8hGyS+AfAA4PYpi8D5wEHgO8B32mw3DJgMsPwVBOAfDrazGzgJP0G8DPgfRFxsOz2NJtHEmZmAyTp14DPAyuGY4KA7LpeMzPrJ0knk83hvUh2+euw5NNNZmZWyKebzOqkK2+eqnkclPS5dMXNOknPpeee2z8o3Y6hK93u4byafc1K2z8naVZNfIqkzanMXemqM4rqMCvLsBtJnHHGGTFhwgReffVVTj755LKb029Dsd1Dsc3Qe7ufeOKJlyPiPZJGkH0r/QJgHrAvIhZJWgCMjogbJF0G/A+yb7RfANwZERdIOo3sm+6dZNfXP0F2y4b9kh4ju9JsI7CG7Ju8D0v6P3l19PY+eo75MgzV3/1ADef323PMH7Wi7PuCNPsxZcqUiIhYv359DEVDsd1Dsc0Rvbcb2JQ9MQ14NC0/C4xJy2OAZ9Py14Cr4/A9fp5N668GvlYT/1qKjQF+VhN/e7uiOnp79BzzZRiqv/uBGs7vt+eYr3944tqsd1cB30rLHZHdrweya/s70vJYjrxdw44U6y2+IyfeWx1HkDQXmAvQ0dFBtVrt15tqlkOHDpVWdxmOt/cLvrrJrJCkE4H/SvYlryNEREhq6bna3uqIiCXAEoDOzs6oVCqtbEqharVKWXWX4Xh7v+CJa7PeXAo8GRE9N4TbLWkMQHrek+I7yW4f3WNcivUWH5cT760Os1I4SZgVu5rDp5oAVgM9VyjNAlbVxGemq5ymAgfSKaO1wDRJo9NVStOAtWndQWX/GErAzLp95dVhVgqfbjLL1/PPmv60JrYIWClpDtkXqK5M8TVkVzZ1Aa+R3Q+LiNgn6Wbg8bTdTRGxLy1/luxGi6OAh9OjtzrMSuEkYZbvVxFxem0gIvYCF9dvmK4MmZe3k4hYCizNiW8i+38e9fHcOszK4tNNZmZWyEnCzMwKOUmYmVkhz0lYQyYs+N4Rr7ct+nhJLTE7dvXHM/iYLuIkYWbWoM07DzC7JsEcD4nFp5vMzKyQk4SZmRVykjAzs0Kek7BceRN7Znb88UjCzMwKOUmYmVkhJwkzMyvkJGFmZoU8cW0DUjuxPX9yN7MXfO+4+GKR2fHGIwkzMyvkJGFmZoWcJMzMrJCThJmZFeozSUh6h6THJP1E0hZJX07xMyVtlNQl6QFJJ6b4Sel1V1o/oWZfN6b4s5IuqYlPT7EuSQtq4rl1mJnZ4GhkJPEGcFFEfBQ4B5guaSpwG3BHRJwF7AfmpO3nAPtT/I60HZImAVcBHwamA38taYSkEcA9wKXAJODqtC291GFmZoOgzyQRmUPp5QnpEcBFwIMpvgy4PC3PSK9J6y+WpBRfERFvRMQLQBdwfnp0RcTzEfEmsAKYkcoU1WFmZoOgoe9JpN7+E8BZZL3+nwOvRER32mQHMDYtjwW2A0REt6QDwOkpvqFmt7VlttfFL0hliuqob99cYC5AR0cH1WqVQ4cOUa1WG3l7baVd2j1/cnffGyUdo7Lt26Hd/dEuP2uzdtZQkoiIt4BzJJ0KPAR8qJWN6q+IWAIsAejs7IxKpUK1WqVSqZTbsAEoo935d3xt/HuW8yd3c/vmkWy7ptK0Ng2GPn7WIyQ9CJxNNnL+NPAs8AAwAdgGXBkR+9Oo907gMuA1YHZEPAkgaRbwxbTPWyJiWYpPAe4DRgFrgOsjIiSdlldHs96zWX/16+qmiHgFWA/8DnCqpJ6/JOOAnWl5JzAeIK1/N7C3Nl5Xpii+t5c6zFptPPD9iPgQ8FHgGWAB8EhETAQeSa8hm0+bmB5zgcUA6Q/+QrKR8fnAQkmjU5nFwGdqyk1P8aI6zErRyNVN70kjCCSNAj5G9oFZD1yRNpsFrErLq9Nr0vofRkSk+FXp6qczyT4YjwGPAxPTlUwnkk1ur05liuowa5kDBw4AvAu4FyAi3kwdpNr5tvp5uPvT/N0Gss7NGOASYF1E7EujgXVkF36MAU6JiA3pOL+f/Dk9z8NZ6Ro5pzAGWJbmJX4NWBkR35W0FVgh6Rbgx6QPVHr+hqQuYB/ZH30iYouklcBWoBuYl05jIek6YC0wAlgaEVvSvm4oqMOsZV544QXIjtG/k/RRsvm464GOiNiVNnsJ6EjLb8/DJT3zZ73Fd+TE6aWOI+TNw5VhqM7r5M25NfI+eubf+lNmqOszSUTE08C5OfHnyYbQ9fHXgU8U7OtW4Nac+Bqy87IN1WHWSt3d3QC/DiyOiI2S7qTutE+aP4hWtqO3OvLm4cowVOf+ZufMwzUyp3b38lXcvvnwn82hNg83EP7GtVmdcePGAbwZERtT6EHgPGB3OlVEet6T1vd3vm1nWq6P00sdZqVwkjCr8773vQ/gTUkfTKGLyU6T1s631c/DzVRmKnAgnTJaC0yTNDpNWE8D1qZ1ByVNTVdGzSR/Ts/zcFY6/z8Js3z/CixPF1M8D1xLmpOTNAd4EbgybbuG7PLXLrJLYK8FiIh9km4muzgD4KaI2JeWP8vhS2AfTg+ARQV1mJXCScIs379FRGdO/OL6QLpCaV7eTiJiKbA0J76J7DsY9fG9eXWYlcWnm8zMrJCThJmZFXKSMDOzQp6TsKapvwfUtkUfL6klZtYsHkmYmVkhJwkzMyvkJGFmZoWcJMzMrJCThJmZFXKSMDOzQk4SZmZWyEnCzMwK+ct0Zjak1X+JE/xFzmbySMLMzAp5JHEcyut5mZnl8UjCzMwKeSRhZtYkw3F+xCMJMzMr1GeSkDRe0npJWyVtkXR9in9J0k5JT6XHZTVlbpTUJelZSZfUxKenWJekBTXxMyVtTPEH0v8VRtJJ6XVXWj+hqe/ezMx61chIohuYHxGTgKnAPEmT0ro7IuKc9FgDkNZdBXwYmA78taQRkkYA9wCXApOAq2v2c1va11nAfmBOis8B9qf4HWk7MzMbJH0miYjYFRFPpuVfAs8AY3spMgNYERFvRMQLQBdwfnp0RcTzEfEmsAKYIUnARcCDqfwy4PKafS1Lyw8CF6ftzcxsEPRr4jqd7jkX2Aj8LnCdpJnAJrLRxn6yBLKhptgODieV7XXxC4DTgVciojtn+7E9ZSKiW9KBtP3Lde2aC8wF6OjooFqtcujQIarVan/eXlsYjHbPn9zd90b90DEqf5/t/vMfqseI2WBqOElIeifwbeBzEXFQ0mLgZiDS8+3Ap1vSyj5ExBJgCUBnZ2dUKhWq1SqVSqWM5hyTwWj37CZ/T2L+5G5u33z0obTtmkpT62m2Pn7WkyVtBt4CuiOiU9JpwAPABGAbcGVE7E+j2zuBy4DXgNk9o29Js4Avpn3eEhHLUnwKcB8wClgDXB8RUVRH8961Wf80dHWTpBPIEsTyiPgOQETsjoi3IuJXwN+SnU4C2AmMryk+LsWK4nuBUyWNrIsfsa+0/t1pe7PB8Ptpvq0zvV4APBIRE4FH0mvI5tkmpsdcYDFA+oO/kGzEfD6wUNLoVGYx8JmactP7qMOsFI1c3STgXuCZiPhqTXxMzWZ/BPw0La8GrkpXJp1J9gF4DHgcmJiuZDqRbHJ7dUQEsB64IpWfBayq2destHwF8MO0vVkZaufI6ufO7o/MBrJOzxjgEmBdROxLo4F1wPS07pSI2JCO5/vJn4errcOsFI2cbvpd4FPAZklPpdgXyK5OOofsdNM24E8BImKLpJXAVrIro+ZFxFsAkq4D1gIjgKURsSXt7wZghaRbgB+TJSXS8zckdQH7yBKL2WD5gaQAvpZOaXZExK607iWgIy2/PXeW9Myr9RbfkROnlzqOkDcPV4Z2mNcZyHzYQOfQ6uff6ssMxbm5vvSZJCLin4G8K4rW9FLmVuDWnPiavHIR8TyHT1fVxl8HPtFXG81a4GcRcZ6k9wLrJP2sdmWaP2jpqLa3OvLm4crQDnN/eXNsfc2HDaQMwN3LVx0x/1ZfZqD7bWf+xrVZvn8HiIg9wENknZjdPadZ0/OetG1/5+F2puX6OL3UYVYKJwmzOq+++iqkz4akk4FpZHNutXNk9XNnM5WZChxIp4zWAtMkjU4T1tOAtWndQUlT05zfTPLn4WrrMCuFb/BnVmf37t0AH5L0E7LPyDcj4vuSHgdWSpoDvAhcmYqsIbv8tYvsEthrASJin6SbyS7aALgpIval5c9y+BLYh9MDYFFBHWalcJIwq/OBD3wAYGvNpa8ARMRe4OL67dMVSvPy9hURS4GlOfFNwNk58dw6zMri001mZlbIScLMzAr5dJO1zHD8ByxmxxuPJMzMrJCThJmZFXKSMDOzQk4SZmZWyBPXw1ze5LGZWaM8kjAzs0JOEmZmVshJwszMCjlJmJlZIScJMzMr5CRhZmaFnCTMzKyQk4SZmRVykjAzs0JOEmZmVshJwszMCvWZJCSNl7Re0lZJWyRdn+KnSVon6bn0PDrFJekuSV2SnpZ0Xs2+ZqXtn5M0qyY+RdLmVOYuSeqtDjMzGxyNjCS6gfkRMQmYCsyTNAlYADwSEROBR9JrgEuBiekxF1gM2R98YCFwAXA+sLDmj/5i4DM15aaneFEdZmY2CPpMEhGxKyKeTMu/BJ4BxgIzgGVps2XA5Wl5BnB/ZDYAp0oaA1wCrIuIfRGxH1gHTE/rTomIDRERwP11+8qrw8zMBkG/bhUuaQJwLrAR6IiIXWnVS0BHWh4LbK8ptiPFeovvyInTSx317ZpLNmqho6ODarXKoUOHqFar/Xl7baHZ7Z4/ubtp+yrSMarxetrpd9LXz1rSCGATsDMi/lDSmcAK4HTgCeBTEfGmpJPIOjdTgL3AJyNiW9rHjcAc4C3gzyJibYpPB+4ERgBfj4hFKZ5bR5PfulnDGk4Skt4JfBv4XEQcTNMGAERESIoWtK+hOiJiCbAEoLOzMyqVCtVqlUql0somtUSz2z17EP6fxPzJ3dy+ubFDads1ldY2ph8a+FlfTzZyPiW9vg24IyJWSPobsj/+i9Pz/og4S9JVabtPptOyVwEfBt4P/KOk3077ugf4GFmn6HFJqyNiay91mJWioaubJJ1AliCWR8R3Unh3OlVEet6T4juB8TXFx6VYb/FxOfHe6jBrtROAjwNfh+yCDOAi4MG0vv4Ua89p0QeBi9P2M4AVEfFGRLwAdJHNx50PdEXE82mUsAKY0UcdZqXos/uXDtx7gWci4qs1q1YDs4BF6XlVTfw6SSvIJqkPRMQuSWuBv6iZrJ4G3BgR+yQdlDSV7DTWTODuPuowa7XxwKeBd6XXpwOvRETPebXa06Jvn0qNiG5JB9L2Y4ENNfusLVN/6vWCPuo4Qt4p1jK0w2ndvFOdfbVpIGXg6FOr9WUGut921sg5gt8FPgVslvRUin2B7A/3SklzgBeBK9O6NcBlZL2m14BrAVIyuBl4PG13U0TsS8ufBe4DRgEPpwe91GHWMt/97ncBuiPiCUmVcluTL+8Uaxna4bRu3inVvk5rDqQMwN3LVx1xarW+zED32876TBIR8c+AClZfnLN9APMK9rUUWJoT3wScnRPfm1eHWSs9+uijkF2Vtw14B9mcxJ0pNjL19GtPi/acSt0haSTwbrIJ7KJTrBTE9/ZSh1kp/I1rszpf+cpXAJ6OiAlkE88/jIhrgPXAFWmz+lOsPV8OvSJtHyl+laST0lVLE4HHyEbTEyWdKenEVMfqVKaoDrNSOEmYNe4G4POSusjmD+5N8XuB01P886QvfUbEFmAlsBX4PjAvIt5Ko4TrgLVkV0+tTNv2VodZKfr1PQmz401EVIFqWn6e7Mqk+m1eBz5RUP5W4Nac+Bqy+bv6eG4dZmVxkhhmJgzC9yLM7PjhJGGDqj6JbVv08ZJaYmaN8JyEmZkVcpIwM7NCThJmZlbIScLMzAo5SZiZWSEnCTMzK+QkYWZmhfw9CTNrG8fr92ja+X17JGFmZoWcJMzMrJCThJmZFXKSMDOzQk4SZmZWyEnCzMwKOUmYmVkhJwkzMyvkJGFmZoX6TBKSlkraI+mnNbEvSdop6an0uKxm3Y2SuiQ9K+mSmvj0FOuStKAmfqakjSn+gKQTU/yk9LorrZ/QtHdtZmYNaWQkcR8wPSd+R0Sckx5rACRNAq4CPpzK/LWkEZJGAPcAlwKTgKvTtgC3pX2dBewH5qT4HGB/it+RtjMzs0HUZ5KIiB8B+xrc3wxgRUS8EREvAF3A+enRFRHPR8SbwApghiQBFwEPpvLLgMtr9rUsLT8IXJy2N2up119/HeA/SPqJpC2SvgwDG/U2a2RtVpZjucHfdZJmApuA+RGxHxgLbKjZZkeKAWyvi18AnA68EhHdOduP7SkTEd2SDqTtX65viKS5wFyAjo4OqtUqhw4dolqtHsPbK8extnv+5O6+N2qyjlEDr7fM31HRzzoiAJ6NiPMknQD8s6SHgc+TjXpXSPobstHuYmpGvZKuIhv1frJuZP1+4B8l/Xaq5h7gY2TH/OOSVkfEVg6PrOvrMCvFQJPEYuBmINLz7cCnm9Wo/oqIJcASgM7OzqhUKlSrVSqVSllNGrBjbffsurtJDob5k7u5ffPADqVt11Sa25h+6ONn/av0fEJ6BNmo949TfBnwJbLPwoy0DNmo96/SqPftkTXwgqSekTWkkTWApJ6R9TO91GFWigF9siNid8+ypL8Fvpte7gTG12w6LsUoiO8FTpU0Mo0marfv2dcOSSOBd6ftLam/vbA1l6SngLPIev0/p/+j3maOrOvbdtTouQzNHrHXj0gb2XfeKLavcgMpA0ePmuvLDHS/A3nfg2VASULSmIjYlV7+EdBz5dNq4JuSvko2vJ4IPAYImCjpTLI//lcBfxwRIWk9cAXZPMUsYFXNvmYB/5LW/zDSeQCzwRAR50g6FXgI+FDJzTlC3ui5DM0esdePhBsZaeaNnvsqN5AyAHcvX3XEqLm+zED3O5D3PVj6TBKSvgVUgDMk7QAWAhVJ55ANwbcBfwoQEVskrQS2At3AvIh4K+3nOmAtMAJYGhFbUhU3ACsk3QL8GLg3xe8FvpGG6PvIEovZoIqIV1JH5nfo/6i3mSNrs1L0mSQi4uqc8L05sZ7tbwVuzYmvAdbkxJ/n8Hna2vjrwCf6ap9Zs/3iF7+ArDODpFFkE8y3Af0a9Upq5sjarBT+96VmdXbt2gXwQUlPk10mvjIivitpK/0Y9TZ5ZG1WCicJK1U7/m/fj3zkIwBbI6KzNj6QUW+zRtZmZfG9m8zMrJCThJmZFXKSMDOzQk4SZmZWyEnCzMwKOUmYmVkhJwkzMyvkJGFmZoWcJMzMrJCThJmZFXKSMDOzQk4SZmZWyEnCzMwKOUmYmVkhJwkzMyvk/ycxhNT/7wUzs1bzSMLMzAo5SZiZWSEnCTMzK+QkYWZmhZwkzMysUJ9XN0laCvwhsCcizk6x04AHgAnANuDKiNgvScCdwGXAa8DsiHgylZkFfDHt9paIWJbiU4D7gFHAGuD6iIiiOo75HVtby7uCa9uij5fQEjODxkYS9wHT62ILgEciYiLwSHoNcCkwMT3mAovh7aSyELgAOB9YKGl0KrMY+ExNuel91GHWUtu3bwf4bUlbJW2RdD1kx7GkdZKeS8+jU1yS7pLUJelpSef17EvSrLT9c6mj1BOfImlzKnNX6mAV1mFWlj6TRET8CNhXF54BLEvLy4DLa+L3R2YDcKqkMcAlwLqI2JdGA+uA6WndKRGxISICuL9uX3l1mLXUyJEjAXZExCRgKjBP0iTcObLj0EC/TNcREbvS8ktAR1oeC2yv2W5HivUW35ET762Oo0iaS/bhpKOjg2q1yqFDh6hWq/18W+Xrrd3zJ3cPbmMa1DGqtW1r1e+xj2PkNYCI+KWkZ8iOyxlAJa1fBlSBG6jpHAEbJPV0jiqkzhGApJ7OUZXUOUrxns7Rw73UYXaE+lOzrTote8zfuE7zB9GMxgy0johYAiwB6OzsjEqlQrVapVKptLJZLdFbu2e36Teu50/u5vbNrfvy/rZrKi3ZbyPHiKQJwLnARtqoc5TXMSpDsztj9Z2NRvad10Hpq9xAysDRHaL6MgPdbzPed6uOgYF+sndLGhMRu1KPaU+K7wTG12w3LsV2crh31BOvpvi4nO17q8NsUEh6J/Bt4HMRcTBNGwDld47yOkZlaHZnrL4j1EgHIa/z1Fe5gZQBuHv5qiM6RPVlBrrfZrzvVnWmBnoJ7GqgZxJuFrCqJj4zTeRNBQ6kXtFaYJqk0emc7DRgbVp3UNLUNHE3s25feXWYDQaRJYjlEfGdFNudOiz0o3NUFO+1c5RTh1kp+kwSkr4F/AvwQUk7JM0BFgEfk/Qc8AfpNWSXsD4PdAF/C3wWIJ2TvRl4PD1u6jlPm7b5eirzc7LzsvRSh1lLZVML/CbwTER8tWaVO0d23OnzdFNEXF2w6uKcbQOYV7CfpcDSnPgm4Oyc+N68Osxa7dFHHwU4HbhI0lMp/AWyjsrK1FF6EbgyrVtD9t2gLrIJ72sh6xxJ6ukcwdGdo/vIvh/0MEd2jvLqMCuFbxVuVufCCy8EeCIiOnNWu3NkxxXflsPMzAo5SZiZWSEnCTMzK+QkYWZmhZwkzMyskJOEmZkV8iWwbSzvfyuYmQ0mjyTMzKyQRxLW9gbrlshmdjSPJMzMrJBHEmY2KPz/y4cmjyTMzKyQk4SZmRVykjAzs0JOEmZmVshJwszMCjlJmJlZIScJMzMr5O9JtImea8jnT+5mtu/ZZGZtwiMJMzMr5JGEmdkw1Yz7nh1TkpC0Dfgl8BbQHRGdkk4DHgAmANuAKyNivyQBdwKXAa8BsyPiybSfWcAX025viYhlKT4FuA8YBawBro+IOJY229Dn2zuYDZ5mnG76/Yg4JyI60+sFwCMRMRF4JL0GuBSYmB5zgcUAKaksBC4AzgcWShqdyiwGPlNTbnoT2mvWiAmS9kj6aU9A0mmS1kl6Lj2PTnFJuktSl6SnJZ1XU2ZW2v651BnqiU+RtDmVuSt1ogrrMCtLK+YkZgDL0vIy4PKa+P2R2QCcKmkMcAmwLiL2RcR+YB0wPa07JSI2pNHD/TX7Mmu1lzm6UzIYHaCiOsxKcaxzEgH8QFIAX4uIJUBHROxK618COtLyWGB7TdkdKdZbfEdO/CiS5pJ9OOno6KBarXLo0CGq1eoxvLXBNX9yNwAdow4vDxXt0OaB/K77OEYOAfvqYjOASlpeBlSBG6jpAAEbJPV0gCqkDhCApJ4OUJXUAUrxng7Qw73UYVaKY00SF0bETknvBdZJ+lntyoiIlEBaKiWnJQCdnZ1RqVSoVqtUKpVWV900s2sugb1989C6nqAd2rztmkq/ywzgGBmMDlBRHUfI6xiVoT+dsbyORH3Z+m0a2Xcj+21GGTi6Q9RX+xvdbzPed16Zgey33jF9siNiZ3reI+khsiH1bkljImJX6k3tSZvvBMbXFB+XYjs53HPqiVdTfFzO9malG4wOUG915HWMytCfRJv3/Z/65F6/TSPJv5H9NqMMwN3LVx3RIeqr/Y3utxnvO6/MQPZbb8BzEpJOlvSunmVgGvBTYDXQM0E3C1iVllcDM9Mk31TgQOoxrQWmSRqdztdOA9amdQclTU2TejNr9mVWht2p40M/OkBF8aIOUFEdZqU4lpFEB/BQuihjJPDNiPi+pMeBlZLmAC8CV6bt15Bd/tpFdgnstQARsU/SzcDjabubes7hAp/l8CWwD6fHsJB3Gae1vZ4O0CKO7gBdJ2kF2ST1gTSSXgv8Rc1k9TTgxnTMH0ydpY1kHaC7+6jDrBQDThIR8Tzw0Zz4XuDinHgA8wr2tRRYmhPfBJw90DaaHYMzgX8BzpC0g+wqpUW0vgNUVIdZKYbWDKnZ4Hmh5rs/tVraASrqZJmVxUnChoVm3H7AzI7mG/yZmVkhjyQGgSepzWyo8kjCzMwKOUmYmVkhJwkzMyvkJGFmZoU8cW3Dki+JNWsOjyTMzKyQk4SZmRVykjAzs0Kek2gBf3nOzIYLJwk7LuQl7vumn1xCS8yGFp9uMjOzQk4SZmZWyEnCzMwKOUmYmVkhT1w3ga9mMrPhyiMJM+u3CQu+x4QF32PzzgPuJA1zThJmZlbIScLMzAp5TqKfPLQ2s+NJ2ycJSdOBO4ERwNcjYlHJTTJrucE87n1bdetNW59ukjQCuAe4FJgEXC1pUrmtMmstH/fWTtp9JHE+0BURzwNIWgHMALYOVgN8eslKUPpxb9ZDEVF2GwpJugKYHhH/Lb3+FHBBRFxXt91cYG56+UHgWeAM4OVBbG6zDMV2D8U2Q+/t/s2IeM9gNqZHI8d9wTFfhqH6ux+o4fx+c4/5dh9JNCQilgBLamOSNkVEZ0lNGrCh2O6h2GYYuu2G/GO+DEP5ZzgQx9v7hTafkwB2AuNrXo9LMbPhzMe9tY12TxKPAxMlnSnpROAqYHXJbTJrNR/31jba+nRTRHRLug5YS3Yp4NKI2NJg8dKH4gM0FNs9FNsMbdruYzzuB1tb/gxb6Hh7v+09cW1mZuVq99NNZmZWIicJMzMrNCyThKTpkp6V1CVpQdntySNpqaQ9kn5aEztN0jpJz6Xn0WW2MY+k8ZLWS9oqaYuk61O8bdsu6R2SHpP0k9TmL6f4mZI2puPkgTRJbA2QtE3SZklPSdpUdntaYah+Rptt2CWJIXRLg/uA6XWxBcAjETEReCS9bjfdwPyImARMBealn287t/0N4KKI+ChwDjBd0lTgNuCOiDgL2A/MKa+JQ9LvR8Q5w/h7A/cxND+jTTXskgQ1tzSIiDeBnlsatJWI+BGwry48A1iWlpcBlw9mmxoREbsi4sm0/EvgGWAsbdz2yBxKL09IjwAuAh5M8bZqs5VvqH5Gm204JomxwPaa1ztSbCjoiIhdafkloKPMxvRF0gTgXGAjbd52SSMkPQXsAdYBPwdeiYjutMlQOk7aQQA/kPREukXI8aKtj/NWaOvvSRzPIiIkte31yZLeCXwb+FxEHJT09rp2bHtEvAWcI+lU4CHgQ+W2aMi7MCJ2SnovsE7Sz1LP+7jRjsd5KwzHkcRQvqXBbkljANLznpLbk0vSCWQJYnlEfCeFh0TbI+IVYD3wO8Cpkno6SkPpOCldROxMz3vIku755bZo0AyJ47yZhmOSGMq3NFgNzErLs4BVJbYll7Ihw73AMxHx1ZpVbdt2Se9JIwgkjQI+RjaXsh64Im3WVm1uZ5JOlvSunmVgGvDT3ksNG217nLfKsPzGtaTLgL/k8C0Nbi23RUeT9C2gQnbr4d3AQuAfgJXAbwAvAldGRP3EWakkXQj8E7AZ+FUKf4FsXqIt2y7pI2STjCPIOkYrI+ImSR8gu7DhNODHwJ9ExBvltXRoSD+3h9LLkcA32/EzdqyG6me02YZlkjAzs+YYjqebzMysSZwkzMyskJOEmZkVcpIwM7NCThJmZlbIScLMzAo5SZiZWaH/D0WEInWItkTkAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "text_word_count = []\n",
    "summary_word_count = []\n",
    "\n",
    "# populate the lists with sentence lengths\n",
    "for i in df_train['text']:\n",
    "      text_word_count.append(len(i.split()))\n",
    "\n",
    "for i in df_train['summary']:\n",
    "      summary_word_count.append(len(i.split()))\n",
    "\n",
    "length_df = pd.DataFrame({'text':text_word_count, 'summary':summary_word_count})\n",
    "\n",
    "length_df.hist(bins = 30)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### NOTE: We can fix the maximum length of the text to 27 and of summary 13 since that seems to be the majority summary length.\n",
    "\n",
    "Let us understand the proportion of the length of summaries and text below 13 and 27 respectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_text_len=27\n",
    "max_summary_len=13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9674311023921702\n"
     ]
    }
   ],
   "source": [
    "cnt=0\n",
    "for i in df_train['summary']:\n",
    "    if(len(i.split()) <= max_summary_len):\n",
    "        cnt=cnt+1\n",
    "print(cnt/len(df_train['summary']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9758745933869699\n"
     ]
    }
   ],
   "source": [
    "cnt=0\n",
    "for i in df_train['text']:\n",
    "    if(len(i.split()) <= max_text_len):\n",
    "        cnt=cnt+1\n",
    "print(cnt/len(df_train['text']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding special tokens in the start and end of the sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['summary'] = df_test['summary'].apply(lambda x : 'sostok '+ x + ' eostok')\n",
    "df_test['summary'] = df_test['summary'].apply(lambda x : 'sostok '+ x + ' eostok')\n",
    "df_valid['summary'] = df_test['summary'].apply(lambda x : 'sostok '+ x + ' eostok')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train-Test split and Preparing the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=df_train['text'].to_numpy()\n",
    "y_train=df_train['summary'].to_numpy()\n",
    "x_test=df_test['text'].to_numpy()\n",
    "y_test=df_test['summary'].to_numpy()\n",
    "\n",
    "\n",
    "x_tokenizer = Tokenizer() \n",
    "x_tokenizer.fit_on_texts(list(x_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rarewords and its Coverage on Reviews column\n",
    "\n",
    "The threshold is taken as 4 which means word whose count is below 4 is considered as a rare word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% of rare words in vocabulary: 0.5807224512172836\n",
      "Total Coverage of rare words: 0.0023057233312542137\n"
     ]
    }
   ],
   "source": [
    "thresh=4\n",
    "\n",
    "cnt=0\n",
    "tot_cnt=0\n",
    "freq=0\n",
    "tot_freq=0\n",
    "\n",
    "for key,value in x_tokenizer.word_counts.items():\n",
    "    tot_cnt=tot_cnt+1\n",
    "    tot_freq=tot_freq+value\n",
    "    if(value<thresh):\n",
    "        cnt=cnt+1\n",
    "        freq=freq+value\n",
    "    \n",
    "print(\"% of rare words in vocabulary:\", (cnt/tot_cnt)*100)\n",
    "print(\"Total Coverage of rare words:\", (freq/tot_freq)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating X and Y tokenizers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_tokenizer = Tokenizer(num_words=tot_cnt-cnt) # num_words: the maximum number of words to keep, based on word frequency.\n",
    "x_tokenizer.fit_on_texts(list(x_train))\n",
    "\n",
    "x_train_seq = x_tokenizer.texts_to_sequences(x_train)\n",
    "x_test_seq = x_tokenizer.texts_to_sequences(x_test)\n",
    "\n",
    "x_train = pad_sequences(x_train_seq,maxlen=max_text_len,padding='post')\n",
    "x_test = pad_sequences(x_test_seq,maxlen=max_text_len,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97927"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_vocab = x_tokenizer.num_words +1\n",
    "x_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_tokenizer = Tokenizer()\n",
    "y_tokenizer.fit_on_texts(list(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% of rare words in vocabulary: 0.8996681832175965\n",
      "Total Coverage of rare words: 0.003717958052718753\n"
     ]
    }
   ],
   "source": [
    "thresh=4\n",
    "\n",
    "count=0\n",
    "total_count=0\n",
    "frequency=0\n",
    "total_frequency=0\n",
    "\n",
    "for key,value in y_tokenizer.word_counts.items():\n",
    "    total_count=total_count+1\n",
    "    total_frequency=total_frequency+value\n",
    "    if(value<thresh):\n",
    "        count=count+1\n",
    "        frequency=frequency+value\n",
    "    \n",
    "print(\"% of rare words in vocabulary:\",(count/total_count)*100)\n",
    "print(\"Total Coverage of rare words:\",(frequency/total_frequency)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of vocabulary in Y = 57941\n"
     ]
    }
   ],
   "source": [
    "#prepare a tokenizer for reviews on training data\n",
    "y_tokenizer = Tokenizer(num_words=total_count-count) \n",
    "y_tokenizer.fit_on_texts(list(y_train))\n",
    "\n",
    "#convert text sequences into integer sequences (i.e one hot encode the text in Y)\n",
    "y_train_seq    =   y_tokenizer.texts_to_sequences(y_train) \n",
    "y_test_seq   =   y_tokenizer.texts_to_sequences(y_test) \n",
    "\n",
    "#padding zero upto maximum length\n",
    "y_train    =   pad_sequences(y_train_seq, maxlen=max_text_len, padding='post')\n",
    "y_test   =   pad_sequences(y_test_seq, maxlen=max_text_len, padding='post')\n",
    "\n",
    "#size of vocabulary\n",
    "y_vocab  =   y_tokenizer.num_words +1\n",
    "print(\"Size of vocabulary in Y = {}\".format(y_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind=[]\n",
    "for i in range(len(y_train)):\n",
    "    count=0\n",
    "    for j in y_train[i]:\n",
    "        if j!=0:\n",
    "            count=count+1\n",
    "    if(count==2):\n",
    "        ind.append(i)\n",
    "\n",
    "y_train=np.delete(y_train,ind, axis=0)\n",
    "x_train=np.delete(x_train,ind, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind=[]\n",
    "for i in range(len(y_test)):\n",
    "    count=0\n",
    "    for j in y_test[i]:\n",
    "        if j!=0:\n",
    "            count=count+1\n",
    "    if(count==2):\n",
    "        ind.append(i)\n",
    "\n",
    "y_test=np.delete(y_test,ind, axis=0)\n",
    "x_test=np.delete(x_test,ind, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the created numpy arrays and tokenziers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import save\n",
    "\n",
    "save(r'x_train.npy',x_train)\n",
    "save(r'x_test.npy',x_test)\n",
    "save(r'y_train.npy',y_train)\n",
    "save(r'y_test.npy',y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(r'x_tokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(x_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open(r'y_tokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(y_tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the saved numpy arrays and tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import load\n",
    "\n",
    "x_train = load(r'x_train.npy')\n",
    "x_test = load(r'x_test.npy')\n",
    "x_valid = load('x_validate.npy')\n",
    "y_train = load(r'y_train.npy')\n",
    "y_test = load(r'y_test.npy')\n",
    "y_valid=load('y_validate.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X-Vocab: 97927 \n",
      "Y-Vocab: 57941\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "with open(r'x_tokenizer.pickle', 'rb') as handle:\n",
    "    x_tokenizer = pickle.load(handle)\n",
    "\n",
    "with open(r'y_tokenizer.pickle', 'rb') as handle:\n",
    "    y_tokenizer = pickle.load(handle)\n",
    "\n",
    "x_vocab = x_tokenizer.num_words +1\n",
    "y_vocab = y_tokenizer.num_words +1\n",
    "\n",
    "print(f'X-Vocab: {x_vocab} \\nY-Vocab: {y_vocab}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Creating the Word2Vec Embedding Layer\n",
    "\n",
    "Setting the window size as 5 and embedding dimesionality 300 as stated in research paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2id = x_tokenizer.word_index\n",
    "id2word = x_tokenizer.index_word\n",
    "vocab_size = x_vocab\n",
    "emded_size=300\n",
    "window_size=5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the function which will produce training batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context (X): ['australia', 'current', 'account', 'deficit', 'shrunk', 'billion', 'dollars', 'lrb', 'billion', 'us'] -> Target (Y): record\n",
      "Context (X): ['current', 'account', 'deficit', 'shrunk', 'record', 'dollars', 'lrb', 'billion', 'us', 'rrb'] -> Target (Y): billion\n",
      "Context (X): ['account', 'deficit', 'shrunk', 'record', 'billion', 'lrb', 'billion', 'us', 'rrb', 'june'] -> Target (Y): dollars\n",
      "Context (X): ['deficit', 'shrunk', 'record', 'billion', 'dollars', 'billion', 'us', 'rrb', 'june', 'quarter'] -> Target (Y): lrb\n",
      "Context (X): ['shrunk', 'record', 'billion', 'dollars', 'lrb', 'us', 'rrb', 'june', 'quarter', 'due'] -> Target (Y): billion\n",
      "Context (X): ['record', 'billion', 'dollars', 'lrb', 'billion', 'rrb', 'june', 'quarter', 'due', 'soaring'] -> Target (Y): us\n",
      "Context (X): ['billion', 'dollars', 'lrb', 'billion', 'us', 'june', 'quarter', 'due', 'soaring', 'commodity'] -> Target (Y): rrb\n",
      "Context (X): ['dollars', 'lrb', 'billion', 'us', 'rrb', 'quarter', 'due', 'soaring', 'commodity', 'prices'] -> Target (Y): june\n",
      "Context (X): ['lrb', 'billion', 'us', 'rrb', 'june', 'due', 'soaring', 'commodity', 'prices', 'figures'] -> Target (Y): quarter\n",
      "Context (X): ['billion', 'us', 'rrb', 'june', 'quarter', 'soaring', 'commodity', 'prices', 'figures', 'released'] -> Target (Y): due\n",
      "Context (X): ['us', 'rrb', 'june', 'quarter', 'due', 'commodity', 'prices', 'figures', 'released', 'monday'] -> Target (Y): soaring\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import numpy as np\n",
    "\n",
    "def generate_context_word_pairs(corpus, window_size, vocab_size):\n",
    "    context_length = window_size*2\n",
    "    for words in corpus:\n",
    "        sentence_length = len(words)\n",
    "        for index, word in enumerate(words):\n",
    "            context_words = []\n",
    "            label_word   = []            \n",
    "            start = index - window_size\n",
    "            end = index + window_size + 1\n",
    "            \n",
    "            context_words.append([words[i] \n",
    "                                 for i in range(start, end) \n",
    "                                 if 0 <= i < sentence_length \n",
    "                                 and i != index])\n",
    "            label_word.append(word)\n",
    "\n",
    "            x = pad_sequences(context_words, maxlen=context_length)\n",
    "            y = tf.keras.utils.to_categorical(label_word, vocab_size)\n",
    "            yield (x, y)\n",
    "            \n",
    "            \n",
    "# Test this out for some samples\n",
    "i = 0\n",
    "for x, y in generate_context_word_pairs(corpus=x_train, window_size=window_size, vocab_size=vocab_size):\n",
    "    if 0 not in x[0]:\n",
    "        print('Context (X):', [id2word[w] for w in x[0]], '-> Target (Y):', id2word[np.argwhere(y[0])[0][0]])\n",
    "    \n",
    "        if i == 10:\n",
    "            break\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the Word2Vec model\n",
    "\n",
    "This model will act as a first layer in our Attentive Sequence to Sequence Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 300)           29378100  \n",
      "_________________________________________________________________\n",
      "lambda (Lambda)              (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 97927)             29476027  \n",
      "=================================================================\n",
      "Total params: 58,854,127\n",
      "Trainable params: 58,854,127\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-16 04:52:34.172795: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
      "2023-02-16 04:52:34.173803: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2023-02-16 04:52:34.449111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n",
      "pciBusID: 0000:07:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:34.450149: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 1 with properties: \n",
      "pciBusID: 0000:08:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:34.452535: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 2 with properties: \n",
      "pciBusID: 0000:0e:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:34.455943: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 3 with properties: \n",
      "pciBusID: 0000:0f:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:34.455974: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-02-16 04:52:34.459644: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2023-02-16 04:52:34.459708: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n",
      "2023-02-16 04:52:34.460859: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2023-02-16 04:52:34.461180: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2023-02-16 04:52:34.465226: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2023-02-16 04:52:34.466333: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2023-02-16 04:52:34.466544: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2023-02-16 04:52:34.483536: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0, 1, 2, 3\n",
      "2023-02-16 04:52:34.487180: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
      "2023-02-16 04:52:35.071895: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 0 with properties: \n",
      "pciBusID: 0000:07:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:35.072756: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 1 with properties: \n",
      "pciBusID: 0000:08:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:35.074793: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 2 with properties: \n",
      "pciBusID: 0000:0e:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:35.076822: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Found device 3 with properties: \n",
      "pciBusID: 0000:0f:00.0 name: Tesla V100-DGXS-32GB computeCapability: 7.0\n",
      "coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s\n",
      "2023-02-16 04:52:35.076855: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-02-16 04:52:35.076890: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2023-02-16 04:52:35.076914: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublasLt.so.11\n",
      "2023-02-16 04:52:35.076935: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2023-02-16 04:52:35.076955: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2023-02-16 04:52:35.076976: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2023-02-16 04:52:35.076996: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2023-02-16 04:52:35.077017: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2023-02-16 04:52:35.096833: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1888] Adding visible gpu devices: 0, 1, 2, 3\n",
      "2023-02-16 04:52:35.096873: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-02-16 04:52:36.808120: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1287] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2023-02-16 04:52:36.808163: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1293]      0 1 2 3 \n",
      "2023-02-16 04:52:36.808174: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 0:   N Y Y Y \n",
      "2023-02-16 04:52:36.808181: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 1:   Y N Y Y \n",
      "2023-02-16 04:52:36.808187: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 2:   Y Y N Y \n",
      "2023-02-16 04:52:36.808194: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1306] 3:   Y Y Y N \n",
      "2023-02-16 04:52:36.823834: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30849 MB memory) -> physical GPU (device: 0, name: Tesla V100-DGXS-32GB, pci bus id: 0000:07:00.0, compute capability: 7.0)\n",
      "2023-02-16 04:52:36.824962: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 384 MB memory) -> physical GPU (device: 1, name: Tesla V100-DGXS-32GB, pci bus id: 0000:08:00.0, compute capability: 7.0)\n",
      "2023-02-16 04:52:36.827157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:2 with 30978 MB memory) -> physical GPU (device: 2, name: Tesla V100-DGXS-32GB, pci bus id: 0000:0e:00.0, compute capability: 7.0)\n",
      "2023-02-16 04:52:36.829352: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:3 with 30978 MB memory) -> physical GPU (device: 3, name: Tesla V100-DGXS-32GB, pci bus id: 0000:0f:00.0, compute capability: 7.0)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow.keras.backend as k\n",
    "from tensorflow.keras.layers import Input,LSTM,Embedding,Dense,Concatenate,TimeDistributed,Bidirectional,Attention, Lambda\n",
    "from tensorflow.keras.models import Model, Sequential,load_model\n",
    "\n",
    "k.clear_session()\n",
    "with tf.device('/gpu:1'):\n",
    "    cbow = Sequential();\n",
    "    cbow.add(Embedding(input_dim=vocab_size, output_dim=emded_size, input_length=window_size*2));\n",
    "    cbow.add(Lambda(lambda x: k.mean(x, axis=1), output_shape=(emded_size,)));\n",
    "    cbow.add(Dense(vocab_size, activation='softmax'));\n",
    "    cbow.compile(loss='categorical_crossentropy', optimizer='rmsprop');\n",
    "\n",
    "print(cbow.summary());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.03713595, -0.03713151, -0.03945891, ..., -0.04383446,\n",
       "         0.01875928,  0.00950094],\n",
       "       [-0.03075793,  0.03271589, -0.03610778, ...,  0.02128861,\n",
       "         0.00047736,  0.03518878],\n",
       "       [-0.00527374,  0.02335833, -0.0175792 , ..., -0.03730728,\n",
       "         0.036261  , -0.03657601],\n",
       "       ...,\n",
       "       [ 0.03451934, -0.0452996 , -0.02604081, ..., -0.00889795,\n",
       "         0.00864307,  0.04188018],\n",
       "       [-0.04608679,  0.02723633,  0.02396525, ..., -0.03396793,\n",
       "        -0.00232814,  0.04900494],\n",
       "       [ 0.0392028 ,  0.00392539,  0.00334841, ...,  0.03203825,\n",
       "         0.03364015, -0.04880544]], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = cbow.get_weights()[0]\n",
    "weights = weights[1:]\n",
    "weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_layer = Embedding(vocab_size-1,\n",
    "                     300,\n",
    "                     mask_zero=False,\n",
    "                     weights=[weights],\n",
    "                     input_length=max_text_len,\n",
    "                     trainable=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Creating the Attentive Seq2Seq Model\n",
    "\n",
    "Creating the architecture of Attentive Seq2Seq model with the latent_dim of 200 and embedding dimensionality of 300 using bi-directional LSTM layers as encoders and decoders as stated in research paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer1 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer2 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer2 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Encoder_BiLSTM_Layer2 will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "WARNING:tensorflow:Layer Decoder_LSTM_Layer will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n",
      "Model: \"Attn_Seq2Seq\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Encoder_Inputs (InputLayer)     [(None, 27)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 27, 300)      29377800    Encoder_Inputs[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   [(None, 27, 400), (N 801600      embedding_1[2][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Decoder_Inputs (InputLayer)     [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) [(None, 27, 400), (N 961600      bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Decoder_Embedding_Inputs (Embed (None, None, 300)    17382300    Decoder_Inputs[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 400)          0           bidirectional_1[0][1]            \n",
      "                                                                 bidirectional_1[0][3]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 400)          0           bidirectional_1[0][2]            \n",
      "                                                                 bidirectional_1[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "Decoder_LSTM_Layer (LSTM)       [(None, None, 400),  1121600     Decoder_Embedding_Inputs[0][0]   \n",
      "                                                                 concatenate_2[0][0]              \n",
      "                                                                 concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Attention_Layer (AttentionLayer ((None, None, 400),  320400      bidirectional_1[0][0]            \n",
      "                                                                 Decoder_LSTM_Layer[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Concat_layer (Concatenate)      (None, None, 800)    0           Decoder_LSTM_Layer[0][0]         \n",
      "                                                                 Attention_Layer[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed (TimeDistribut (None, None, 57941)  46410741    Concat_layer[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 96,376,041\n",
      "Trainable params: 96,376,041\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "k.clear_session()\n",
    "\n",
    "latent_dim = 200\n",
    "embedding_dim = 300\n",
    "\n",
    "# Encoder\n",
    "encoder_inputs = Input(shape=(max_text_len, ),name='Encoder_Inputs')\n",
    "\n",
    "# Embedding layer\n",
    "enc_emb = wv_layer(encoder_inputs)\n",
    "\n",
    "# Encoder LSTM 1 \n",
    "encoder_lstm1 = Bidirectional(LSTM(latent_dim, return_sequences=True,\n",
    "                     return_state=True, dropout=0.2,\n",
    "                     recurrent_dropout=0.2,name='Encoder_BiLSTM_Layer1'))\n",
    "(encoder_output1, forward_state_h1, forward_state_c1,backward_state_h1,backward_state_c1) = encoder_lstm1(enc_emb)\n",
    "state_h1=Concatenate()([forward_state_h1,backward_state_h1])\n",
    "state_c1=Concatenate()([forward_state_c1,backward_state_c1])\n",
    "\n",
    "# Encoder LSTM 2\n",
    "encoder_lstm2 = Bidirectional(LSTM(latent_dim, return_sequences=True,\n",
    "                     return_state=True, dropout=0.2,\n",
    "                     recurrent_dropout=0.2,name='Encoder_BiLSTM_Layer2'))\n",
    "(encoder_outputs, forward_state_h2, forward_state_c2,backward_state_h2,backward_state_c2) = encoder_lstm2(encoder_output1)\n",
    "state_h=Concatenate()([forward_state_h2,backward_state_h2])\n",
    "state_c=Concatenate()([forward_state_c2,backward_state_c2])\n",
    "\n",
    "# Set up the decoder, using encoder_states as the initial state\n",
    "decoder_inputs = Input(shape=(None, ),name='Decoder_Inputs')\n",
    "\n",
    "# Embedding layer\n",
    "dec_emb_layer = Embedding(y_vocab, embedding_dim, trainable=True, name='Decoder_Embedding_Inputs')\n",
    "dec_emb = dec_emb_layer(decoder_inputs)\n",
    "\n",
    "# Decoder LSTM1\n",
    "decoder_lstm = LSTM(latent_dim*2, return_sequences=True,\n",
    "                    return_state=True, dropout=0.2,\n",
    "                    recurrent_dropout=0.2,name='Decoder_LSTM_Layer')\n",
    "(decoder_outputs, decoder_fwd_state, decoder_back_state) = decoder_lstm(dec_emb, initial_state=[state_h, state_c])\n",
    "\n",
    "# Attention Layer\n",
    "attn_layer = AttentionLayer(name='Attention_Layer')\n",
    "attn_out,attn_states=attn_layer([encoder_outputs,decoder_outputs])\n",
    "\n",
    "decoder_concat_input=Concatenate(axis=-1,name='Concat_layer')([decoder_outputs,attn_out])\n",
    "# Dense layer\n",
    "decoder_dense = TimeDistributed(Dense(y_vocab, activation='softmax',name='TimeDistribution_Layer'))\n",
    "decoder_outputs = decoder_dense(decoder_concat_input)\n",
    "\n",
    "# Define the model\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs,name='Attn_Seq2Seq')\n",
    "\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating the learning rate scheduler\n",
    "\n",
    "This will reduce the learning rate by 25% after each epoch. Initially, lr is set to 0.002 as stated in research paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "import math\n",
    "# learning rate schedule\n",
    "def step_decay(epoch):\n",
    "    initial_lrate = 0.002\n",
    "    drop = 0.75\n",
    "    epochs_drop = 1.0\n",
    "    lrate = initial_lrate * math.pow(drop, math.floor((epoch)/epochs_drop))\n",
    "    return lrate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Adam Optimizer\n",
    "\n",
    "Creating Adam optimizer with loss evaluation as 'sparse_categorical_crossentropy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.0), loss='sparse_categorical_crossentropy')\n",
    "lrate = LearningRateScheduler(step_decay,verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Early Stopping \n",
    "\n",
    "It will stop training the model if the val_loss does not decreases after 2 consecutive epochs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=2)\n",
    "\n",
    "checkpoint_filepath = 'Attentive Seq2Seq Model'\n",
    "checkpoint= tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    monitor='val_loss',\n",
    "    mode='min',\n",
    "    verbose=1,\n",
    "    save_best_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.002.\n",
      "7057/7057 [==============================] - 9066s 1s/step - loss: 1.2910 - val_loss: 1.1132\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.11319, saving model to Attentive Seq2Seq Model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-16 07:46:33.258384: W tensorflow/python/util/util.cc:348] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: Attentive Seq2Seq Model/assets\n",
      "Epoch 2/20\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.0015.\n",
      "7057/7057 [==============================] - 8772s 1s/step - loss: 0.9648 - val_loss: 1.0492\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.11319 to 1.04918, saving model to Attentive Seq2Seq Model\n",
      "INFO:tensorflow:Assets written to: Attentive Seq2Seq Model/assets\n",
      "Epoch 3/20\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0011250000000000001.\n",
      "7057/7057 [==============================] - 8766s 1s/step - loss: 0.8864 - val_loss: 1.0269\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.04918 to 1.02692, saving model to Attentive Seq2Seq Model\n",
      "INFO:tensorflow:Assets written to: Attentive Seq2Seq Model/assets\n",
      "Epoch 4/20\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.00084375.\n",
      "7057/7057 [==============================] - 8750s 1s/step - loss: 0.8403 - val_loss: 1.0189\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.02692 to 1.01891, saving model to Attentive Seq2Seq Model\n",
      "INFO:tensorflow:Assets written to: Attentive Seq2Seq Model/assets\n",
      "Epoch 5/20\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.0006328125.\n",
      "7057/7057 [==============================] - 8747s 1s/step - loss: 0.8088 - val_loss: 1.0164\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.01891 to 1.01641, saving model to Attentive Seq2Seq Model\n",
      "INFO:tensorflow:Assets written to: Attentive Seq2Seq Model/assets\n",
      "Epoch 6/20\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.000474609375.\n",
      "7057/7057 [==============================] - 8745s 1s/step - loss: 0.7861 - val_loss: 1.0165\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.01641\n",
      "Epoch 7/20\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.00035595703125.\n",
      "7057/7057 [==============================] - 8743s 1s/step - loss: 0.7694 - val_loss: 1.0187\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 1.01641\n",
      "Epoch 00007: early stopping\n"
     ]
    }
   ],
   "source": [
    "with tf.device('/gpu:0'):\n",
    "    history = model.fit(\n",
    "        [x_train, y_train[:, :-1]],\n",
    "        y_train.reshape(y_train.shape[0], y_train.shape[1], 1)[:, 1:],\n",
    "        epochs=20,\n",
    "        callbacks=[es,checkpoint,lrate],\n",
    "        batch_size=64,\n",
    "        shuffle=True,\n",
    "        use_multiprocessing=True,\n",
    "        workers=-1,\n",
    "        validation_data=([x_valid, y_valid[:, :-1]],\n",
    "                         y_valid.reshape(y_valid.shape[0], y_valid.shape[1], 1)[:\n",
    "                         , 1:]),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn/ElEQVR4nO3de3xU9Z3/8ddnJpchN5KQBAghCSD3OwQI3u+CipdarZe0u7YW3dqtta2/arft1nZ3291a27qrdq2y2kVxrXcqKlZFrMolIPe7CiQEknBPCIFcvr8/ZoCACWAyyclM3s/HYx7MnHPmnM8Aec833/M932POOUREJPL5vC5ARETCQ4EuIhIlFOgiIlFCgS4iEiUU6CIiUSLGqwNnZGS4/Px8rw4vIhKRlixZstM5l9ncOs8CPT8/n+LiYq8OLyISkcxsS0vr1OUiIhIlFOgiIlHilIFuZjPMrMLMVrWw/mozW2Fmy8ys2MzODn+ZIiJyKqfTh/4k8F/An1pY/zbwqnPOmdko4DlgSHjKExE5Xl1dHaWlpdTW1npdSrsKBALk5OQQGxt72u85ZaA75+abWf5J1lc3eZkIaHIYEWk3paWlJCcnk5+fj5l5XU67cM6xa9cuSktL6dev32m/Lyx96GZ2rZmtA14Dvn6S7aaHumWKKysrw3FoEeliamtr6dGjR9SGOYCZ0aNHjy/8W0hYAt0595JzbghwDfCLk2z3mHOuwDlXkJnZ7DBKEZFTiuYwP6I1nzGso1ycc/OB/maWEc79NvVJZTU/n72Gw/WN7XUIEZGI1OZAN7MzLPRVYmbjgHhgV1v325Ktu2qY8cFnzF2zo70OISLSor179/LII4984fddfvnl7N27N/wFNXE6wxZnAR8Bg82s1My+YWZ3mNkdoU2uA1aZ2TLgYeArrh3vmnHuoExy0roxc0GLF0uJiLSblgK9vr7+pO+bM2cOqamp7VRV0OmMcrnpFOv/Hfj3sFV0Cn6fcfOkXP7jjfVsqqjijKzkjjq0iAj33nsvn3zyCWPGjCE2NpZAIEBaWhrr1q1jw4YNXHPNNZSUlFBbW8tdd93F9OnTgWPTnVRXVzN16lTOPvtsPvzwQ/r06cMrr7xCt27d2lybZ3O5tMUNBX357VsbmLlgKz+7arjX5YiIR+6fvZo1ZfvDus9h2Sn887SWc+VXv/oVq1atYtmyZcybN48rrriCVatWHR1eOGPGDNLT0zl48CATJkzguuuuo0ePHsftY+PGjcyaNYs//vGP3HDDDbzwwgsUFRW1ufaIvPQ/IymeqSN688LSUmoOn/zXHBGR9jRx4sTjxoo/9NBDjB49msLCQkpKSti4cePn3tOvXz/GjBkDwPjx49m8eXNYaonIFjpAUWEery4vY/byMr4yIdfrckTEAydrSXeUxMTEo8/nzZvHX//6Vz766CMSEhI4//zzmx1LHh8ff/S53+/n4MGDYaklIlvoABPy0xjUM4mZC7Z6XYqIdCHJyclUVVU1u27fvn2kpaWRkJDAunXrWLBgQYfWFrGBbmYUFeaxcts+lpfs9bocEekievTowVlnncWIESO45557jls3ZcoU6uvrGTp0KPfeey+FhYUdWpu14wjDkyooKHBtvcFFVW0dk/7tba4Y2ZtfXz86TJWJSGe2du1ahg4d6nUZHaK5z2pmS5xzBc1tH7EtdIDkQCxXj+nD7BVl7Kup87ocERFPRXSgAxQV5lJb18jzS0u9LkVExFMRH+jDs7szNjeVpxduwavuIxGRziDiAx2gaFIen1Ye4KNP2m0KGRGRTi8qAv2KUb1JTYhl5kLN7yIiXVdUBHog1s/143OYu7qciv3RfVsqEZGWREWgA9w8KY/6Rsezi0u8LkVEolhrp88F+N3vfkdNTU2YKzomagK9X0Yi5wzMYNairdQ36OYXItI+OnOgR+xcLs25ZVIed8xcwjvrKrh0eC+vyxGRKNR0+txLLrmErKwsnnvuOQ4dOsS1117L/fffz4EDB7jhhhsoLS2loaGBn/zkJ5SXl1NWVsYFF1xARkYG7777bthri6pAv3hoFr1SAsxcuFWBLtIVvH4v7FgZ3n32GglTf9Xi6qbT586dO5fnn3+eRYsW4ZzjqquuYv78+VRWVpKdnc1rr70GBOd46d69Ow8++CDvvvsuGRntc5fOqOlyAYjx+7hpYi7zN1SyZdcBr8sRkSg3d+5c5s6dy9ixYxk3bhzr1q1j48aNjBw5krfeeosf/vCHvP/++3Tv3r1D6omqFjrAjRP78tA7G3lm4Vbuu7xrzPcg0mWdpCXdEZxz3Hfffdx+++2fW7d06VLmzJnDj3/8Yy666CJ++tOftns9UdVCB+iZEuDSYT15rriE2roGr8sRkSjTdPrcyy67jBkzZlBdXQ3Atm3bqKiooKysjISEBIqKirjnnntYunTp597bHqKuhQ7Bm1+8vmoHr6/azrVjc7wuR0SiSNPpc6dOncrNN9/M5MmTAUhKSmLmzJls2rSJe+65B5/PR2xsLI8++igA06dPZ8qUKWRnZ7fLSdGInj63Jc45LvrNe6QlxvHCP5zZLscQEW9o+twonT63JWbGzZNyWbJlT9hvICsi0llFZaADfHl8DvExPs3vIiJdRtQGempCHNNGZ/Pyx9uoqtXNL0SiSVeYKrs1nzFqAx2CJ0drDjfw8sfbvC5FRMIkEAiwa9euqA515xy7du0iEAh8ofdF5SiXI0bndGdEnxRmLthKUWEeZuZ1SSLSRjk5OZSWllJZWel1Ke0qEAiQk/PFRulFdaCbGUWT8rj3xZUUb9nDhPx0r0sSkTaKjY2lX79+XpfRKUV1lwvAVWOySQ7EMHOBTo6KSHSL+kBPiIvhunE5vL5yB7uqD3ldjohIuzlloJvZDDOrMLNVLay/xcxWmNlKM/vQzEaHv8y2uWVSLocbGnmuuNTrUkRE2s3ptNCfBKacZP1nwHnOuZHAL4DHwlBXWA3smcykfuk8s2gLjY3Re2ZcRLq2Uwa6c24+sPsk6z90zu0JvVwAdMrJU4oK8yjZfZD3Nkb3mXER6brC3Yf+DeD1llaa2XQzKzaz4o4ecnTZ8F5kJMXztE6OikiUClugm9kFBAP9hy1t45x7zDlX4JwryMzMDNehT0tcjI+vTMjhnXUVbNt7sEOPLSLSEcIS6GY2CngcuNo5tysc+2wPN03MxQGzFm71uhQRkbBrc6CbWS7wIvBV59yGtpfUfnLSErhwcBbPLi7hcH2j1+WIiITV6QxbnAV8BAw2s1Iz+4aZ3WFmd4Q2+SnQA3jEzJaZWftMch4mRYV57Kw+xNw1O7wuRUQkrE556b9z7qZTrL8NuC1sFbWzcwdlkpPWjZkLtnDlqGyvyxERCZuov1L0RH5f8OYXCz7dzaaK9ru3n4hIR+tygQ5wQ0FfYv3GzAU6OSoi0aNLBnpGUjxTR/TmhaWl1Byu97ocEZGw6JKBDvDVyXlU1dYze3mZ16WIiIRFlw30grw0BvdMVreLiESNLhvoZkZRYS4rt+1jecler8sREWmzLhvoANeM7UNCnF83vxCRqNClAz05EMs1Y/swe0UZ+2rqvC5HRKRNunSgAxRNyqO2rpHnl+rmFyIS2bp8oA/LTmFcbipPL9iCc7r5hYhEri4f6BCc3+XTnQf48JNOO1GkiMgpKdCBy0f2Ji0hVidHRSSiKdCBQKyf6wv6MndNOeX7a70uR0SkVRToITdPzKWh0fHsohKvSxERaRUFekh+RiLnDMxg1qKt1Dfo5hciEnkU6E0UFeaxY38tb6+r8LoUEZEvTIHexEVDsujdPaCToyISkRToTcT4fdw4IZf3N+5k884DXpcjIvKFKNBPcOPEvvh9xjOLNAujiEQWBfoJeqYEuHRYT/5cXEJtXYPX5YiInDYFejOKCvPYU1PHnJXbvS5FROS0KdCbceaAHvTPSNTJURGJKAr0ZpgZN0/KZenWvawp2+91OSIip0WB3oIvj88hPsbHzIVqpYtIZFCgtyA1IY5po7N5+eNtVNXq5hci0vlFXqA31EPFug45VFFhHjWHG3j5420dcjwRkbaIvEBf/RI8MgmevQW2LWnXQ43O6c6IPinMXLBVN78QkU4v8gL9jIvgvB/C5vfhjxfCn66Bz96HdghcM6NoUh7ry6so3rIn7PsXEQmnUwa6mc0wswozW9XC+iFm9pGZHTKzH4S/xBMkpMMFP4LvroKL74fy1fDUlTDjMtjwZtiD/aox2SQHYjSEUUQ6vdNpoT8JTDnJ+t3Ad4AHwlHQaQukwNnfhe+ugMsfgP1l8MwN8IdzYNWL0BieqzwT4mK4blwOr6/cwa7qQ2HZp4hIezhloDvn5hMM7ZbWVzjnFgPeDAWJ7QYTvwnf+RiufgTqD8Lzt8LDE+HjmVB/uM2HKCrM5XBDI88Vl4ahYBGR9hF5fegt8cfC2FvgzkVw/ZPBoH/lTnhoLCx8DOoOtnrXZ2QlU9g/nWcWbaGxUSdHRaRz6tBAN7PpZlZsZsWVlZXtcxCfH4ZfC7e/D7c8D91z4PV74Hcj4f0HobZ1V34WFeZRsvsg721sp7pFRNqoQwPdOfeYc67AOVeQmZnZvgczg4GXwNffgL+fA71Gwdv3w29HwDv/Agd2faHdXTqsFxlJ8Tytk6Mi0klFT5dLS8wg/yz46ovwzXeh/7kw/9fwuxHwxo+CJ1NPQ1yMjxsn9OWddRVs29v67hsRkfZyOsMWZwEfAYPNrNTMvmFmd5jZHaH1vcysFPge8OPQNintW3Yr9RkHX5kJ31oIQ6+ChX+A34+G2XfB7k9P+fabJuUCMGuhbn4hIp2PeXUFZEFBgSsuLvbk2Eft/gw+fCg4GqaxHkZ8Gc6+G3oOa/Ettz21mGUl+/jw3guJi4n+X3BEpHMxsyXOuYLm1nXtRErvB1f+Fu5aAYXfgnWvwaOTTzqtwC2FeeysPsSbq3d0cLEiIifXtQP9iJTecNm/wt2rTjmtwHkDM+mb3k1XjopIp6NAb+rItAJ3r4ZLfn5sWoEnLj06rYDPZ9w8MY+Fn+1mY3mV1xWLiBylQG9OfDKcddexaQWqth83rcAN43oT5/fxtE6OikgnokA/mabTClzzKNTXwvO30uPJs/lZ36W8smQzNYfrva5SRARQoJ8efyyMuRnuXAjXPwVxidy84z/4C//I2pd/3aZpBUREwkWB/kX4/DD8Grh9Pu7mP7MnJovxa36Fa+O0AiIi4aBAbw0zbNClfHzxs9xw6CdUpQ5r07QCIiLhoEBvg2vG9mFV7Ah+nvoLmD6v1dMKiIiEgwK9DZIDsVwztg+zl5exN3V4m6YVEBFpKwV6GxVNyuNQfSPPLwnd/CJrCHzpv+E7S2HsV2HZLPjP8fDCbVC+xttiRSSqKdDbaFh2CuNyU3lm4VaOmxcnLR+ufDA4ln3ynbBuTnBagVk3tzitgIhIWyjQw6CoMI9Pdx7gw0+aORma3Asu/ZfQtAL3wpYPQtMKXA2fzQ/7Ta1FpOvq2rMthkltXQOTf/k2hf178GjR+JNvfKgKimfAh/8FByogYzD0GhnsqskcAplDg5OG+fwdU7yIRJSTzbYY09HFRKNArJ/rC/ryxN8+o3x/LT1TAi1vfGRagYnTYdnTsP4NKFkIq54/to0/HjIGhgJ+yLGwT+sHfv2TiUjz1EIPk807D3D+A/O4++JB3HXxwC++g0NVULkBKtdB5VqoWAeV62Ffk/li/HGQMQgyBwdb8pmDIWuogl6kC1ELvQPkZyRy7qBMZi3ayp0XDCDG/wVPT8QnQ8744KOpQ1Wwc0Mo4NcGQ75kMax64dg2/jjoMbBJt03okd5fQS/SheinPYyKJuUy/X+X8Pa6Ci4b3is8O41Phj7jg4+mDlXDzvXBgK9YG2zZl7YQ9Eda8kda9gp6kaikn+owunBIFr27B5i5YEv4Ar0l8UknCfpQ101FqEW/rRhWv3hsG19s83306f2DE5GJSERSoIdRjN/HTRNzefCtDWzeeYD8jMSOLyI+KXgz7D7jjl9++EAw3CvXH+uj37akhaA/oY9eQS8SERToYXbjhL489PZGnlm0lR9dPtTrco6JS2w56E/so9+2FFa/dGwbXyz0OOPzffQ9BijoRToRBXqYZaUEuHR4T/5cXML3LhlEILaTjyePS4TsscFHU0eCvmkffdnHsPplIDQy6kjQH2nJJ/eGQHcIpEB89ybPUyD2JEM5RSQsFOjtoGhSHnNW7mDOyu18aVyO1+W0TotBX/P5Pvrty2DNKxwN+ub444+Fe9OgD4SC/3PLU45fHp+iE7kip6CfkHYweUAP+mcmMnPBlsgN9JbEJUD2mOCjqbqDcGAnHNoPtfuCN/s4+nxf88v3bz/2vK7m1MeOTTwh6Ft63twXRHeISwKfZruQ6KVAbwdmxi2T8vjFX9awpmw/w7JTvC6p/cV2g9S+rX9/Q11wzP1xXwD7m3m+79jzml2w+7Nj2zQcPsVBrIXfEJp57o8Lnh/wxRz70xcb/C3BFxta5m/y/Mi6puubvjcGzFr/9yPecg5cY/D/aWN96NEAjSe+rm+yTUOTdScsS+8f7KYMMwV6O/nyuBx+/eY6Zi7cwr9dO9Lrcjo/fywkpAcfrVVXe8JvAvua+W3hhOf7S6GiyXLXGL7PdCJfS18GMU3Cv8mXhi/mhC+Qk20Xe/w6n//YshM1e3V4M8s61XYE/22OBmQoHD8XsM2EZ4sB23D8/hrqT9j/CY9wOuu7cMn94d0nCvR20z0hlmmjsnn5423cN3UIyQGNBml3sYHgIymrde93Lngy+EhrvyH0g980NBrqQsuaBMOR9Ue3qzvhvQ1Nntc1CaIm2zV9f9Pt6g9D44FmajjJ8U52LiMaHPli88Uc+2I87nXT34z8TdbFQkw8+BKbbOM/YX8nPvzHf5l+0eM1fd30eEk92+WvRoHejooK8/jzklJe+ngbX5uc73U5cipmwXH88UleV9I2jY3Hf9E029XTzLJI2E5dVyelQG9Ho/umMrJPd2Yu2MJXC/Mw/UeUjuDzgS8+2BqVLuWUp/zNbIaZVZjZqhbWm5k9ZGabzGyFmY1rbruuqqgwlw3l1SzevMfrUkQkyp3OGK4ngSknWT8VGBh6TAcebXtZ0WPa6GySAzHMXLDF61JEJMqdMtCdc/OB3SfZ5GrgTy5oAZBqZr3DVWCkS4iL4bpxOby+ajs7qw95XY6IRLFwXGXRByhp8ro0tExCigpzqWtwPFdccuqNRURaqUMvmzOz6WZWbGbFlZWVHXloT52RlUxh/3SeWbiVhsYoH1ImIp4JR6BvA5peIpgTWvY5zrnHnHMFzrmCzMzMMBw6chQV5lG65yDzN3SdLzIR6VjhCPRXga+FRrsUAvucc9vDsN+ocumwXmQkxevkqIi0m1OOQzezWcD5QIaZlQL/DMQCOOf+AMwBLgc2ATXAre1VbCSLi/Fx44S+PDxvE6V7ashJS/C6JBGJMqcMdOfcTadY74A7w1ZRFLtpUi6PzNvErEVbueeyIV6XIyJRRnOJdqA+qd24cEgW/7e4hMP17TgJlIh0SQr0DnZLYR47qw/z5uodXpciIlFGgd7BzhuYSd/0bjo5KiJhp0DvYD5f8OYXCz/bzcbyKq/LEZEookD3wPXjc4jz+3h64VavSxGRKKJA90CPpHguH9mLF5aUUnM4zHdCEZEuS4HukaLCPKoO1fPqsjKvSxGRKKFA98j4vDSG9ErmkXmfsKxkr9fliEgUUKB7xMz46bRhHKxr4JqHP+CePy+nskrT64pI6ynQPXTmgAze+f553H5uf15eto0LH5jH4+9/qouORKRVFOgeSw7Ect/lQ3nju+cyLi+Nf3ltLVN/P1+zMorIF6ZA7yQGZCbx5K0TeOLvCqhvdHxtxiK++aditu6q8bo0EYkQCvROxMy4aGhP5t59Lv9vymA+2LSTi3/7Hg+8uV7DG0XklBTonVB8jJ9vnX8G7/7gfK4Y2Zv/encTFz7wHq8uLyM4uaWIyOcp0DuxnikBfvuVMTx/x2QykuP4zqyP+cp/L2B12T6vSxORTkiBHgEK8tN55c6z+eWXRrKpsppp//k3/umllew+cNjr0kSkE1GgRwi/z7hpYi7vfv98vjY5n2cXl3DBA/P400ebqW/QMEcRUaBHnO4JsfzsquHM+c45DM9O4aevrObK//wbH32yy+vSRMRjCvQINbhXMk/fNok/FI2jqraem/64gDufWcq2vQe9Lk1EPKJAj2BmxpQRvXn7++dx98WD+Ouaci76zTweensjtXUNXpcnIh1MgR4FArF+7rp4IG9//zwuGtKTB9/awMUPvscbq7ZrmKNIF6JAjyI5aQk8fMs4nvnmJBLjYrhj5lKKnljIBt0ZSaRLUKBHoTMHZPDad87m/quGs2rbfqb+/n3un72afQfrvC5NRNqRAj1Kxfh9/N2Z+bz7g/O5cUJfnvxwMxc8MI9nF22loVHdMCLRSIEe5dIT4/jXa0cy+9tnMyAzkXtfXMk1D3/Aki27vS5NRMJMgd5FjOjTnedun8zvbxxDZdUhrnv0I+7+v2WU76/1ujQRCRMFehdiZlw9pg9vf/887rxgAK+t2M4FD8zj0XmfcKhewxxFIp0CvQtKjI/hnsuG8Nb3zuXMARn8+xvruOy383lnXbnXpYlIGyjQu7C8Hok8/ncFPPX1ifh8xtefLObW/1nEp5XVXpcmIq1wWoFuZlPMbL2ZbTKze5tZn2dmb5vZCjObZ2Y54S9V2st5gzJ5465z+fEVQ1m8eQ+X/W4+v3x9LdWHdFMNkUhyykA3Mz/wMDAVGAbcZGbDTtjsAeBPzrlRwM+BX4a7UGlfcTE+bjunP+/84DyuGdOH/37vUy54YB4vLCmlUcMcRSLC6bTQJwKbnHOfOucOA88CV5+wzTDgndDzd5tZLxEiKznAr68fzUvfOpPs1G58/8/Lue4PH7KidK/XpYnIKZxOoPcBSpq8Lg0ta2o58KXQ82uBZDPrceKOzGy6mRWbWXFlpe5q35mNzU3jpX84k19/eRQlu2u4+uEP+OHzK9hZfcjr0kSkBeE6KfoD4Dwz+xg4D9gGfG4cnHPuMedcgXOuIDMzM0yHlvbi8xnXF/TlnR+cz21n9+OFpaVc8MA8nvjbZ9Tpphoinc7pBPo2oG+T1zmhZUc558qcc19yzo0F/im0bG+4ihRvpQRi+acrhvHGd89lbG4av/jLGqb+/n3+tnGn16WJSBOnE+iLgYFm1s/M4oAbgVebbmBmGWZ2ZF/3ATPCW6Z0BmdkJfHUrRN4/GsFHK5vpOiJhdz+v8WU7K7xujQR4TQC3TlXD3wbeBNYCzznnFttZj83s6tCm50PrDezDUBP4F/bqV7xmJlx8bCezL37XO65bDDzN+zkogff4zdz13NAwxxFPGVe3QChoKDAFRcXe3JsCZ/t+w7yq9fX8cqyMrrF+rloaBbTRmdz/uBM4mP8XpcnEnXMbIlzrqDZdQp0CYdlJXt5fkkJc1buYPeBwyQHYrhseC+mjc7mzAE9iPXromSRcFCgS4epa2jkw092MXt5GW+u2kHVoXrSE+O4fGQvpo3KZkJ+Oj6feV2mSMRSoIsnausaeG9DJbOXl/HXteXU1jXSKyXAFaN6c9XobEbldMdM4S7yRSjQxXMHDtXz9roKXl1WxnsbKqhrcOSmJzBtdG+mjc5mSK8Ur0sUiQgKdOlU9tXU8eaaHcxeXsaHn+yiodExqGcS00Zlc+XobPplJHpdokinpUCXTmtn9SFeX7md2cu3s2hz8LZ4I/t0Z9ro3lw5Kpvs1G4eVyjSuSjQJSKU7T3InJXbeXV5GStK9wEwIT+NaaOzmTqiN5nJ8R5XKOI9BbpEnM07D/CXFWXMXr6d9eVV+AzOHJDBVaOzuWx4L7onxHpdoognFOgS0dbvqGL28jJmryhjy64aYv3GeYMymTY6m4uH9iQxPsbrEkU6jAJdooJzjpXb9jF7eRl/WbGd7ftqCcT6uGhoT6aNCl6dGojV1akS3RToEnUaGx3FW/Ywe3kZc1ZuZ9eBwyTHx3Dp8F5MG92bs87I0NWpEpUU6BLV6hsa+ejTXby6rIw3Vu+gqraetIRYpo7szbRR2Uzsl45fV6dKlFCgS5dxqL6B+Rt2Mnt5GW+tKedgXQM9U+K5YmQ200b3ZkzfVF2dKhFNgS5dUs3het5eW8Hs5WXMW1/J4YZGctK6MW10NleNzmZIr2SFu0QcBbp0eftr65i7upxXl5fxwaadNDQ6zsgKXp06bXRv+mcmeV2iyGlRoIs0sav6EK+vCk49sGjzbpyDEX1Sjk490EdXp0onpkAXacGOfbXBC5hWbGd5yV4AxuWmcs7ATCbkpzM2N1Xj3KVTUaCLnIatu2qYvaKM11dtZ3XZfpwDv88Ynp1CQV46E/ulMT4vXVMQiKcU6CJf0P7aOj7eupfFn+1m8ebdLCvZy6H6RgD6ZSRSkJfGhH7pTMhPJ79Hgk6uSodRoIu00eH6RlZu20fx5t0s3ryH4i272VtTB0BGUhwFeemhgE9jWO8UYnRRk7QTBbpImDU2Oj6prA6G++bdLNq8m9I9BwFIiPMzLjeNgvw0JuSnM6av+uElfBToIh1g+76DFB8N+D2s23GsH35EdgoF+cEWvPrhpS0U6CIe2F9bx9IteyjevIdFoX74w6F++P4ZiRTkp1GQn87E/HTy1A8vp0mBLtIJHKpvYNW2/SzevPtoX/y+g0f64eOZ0CTgh/ZOVj+8NOtkga6OPZEOEh/jZ3xeGuPz0uC8ATQ2OjZVVocCfg+LN+/m9VU7gGP98BNC3TRjclNJiNOPq5ycWugincj2fQePnmhd3Ew//IT8dAry0ynITyMjSf3wXZG6XEQi1L6DdSzdeizgT+yHnxAK9wnqh+8yFOgiUSLYD7+PxZv3sPiz3RRvOdYPn5kc6ofPC17wpH746KRAF4lSTfvhg1e17mHb3uB4+MQ4P6P7pjKkVwqDeiYxsGcyg3omkRzQDbYjWZsD3cymAL8H/MDjzrlfnbA+F3gKSA1tc69zbs7J9qlAF2kfZXsPUrwl2IJfVrKXjRVV1NY1Hl2f3T3AoF7JDOqZzMCsJAb3SuaMrCSddI0QbQp0M/MDG4BLgFJgMXCTc25Nk20eAz52zj1qZsOAOc65/JPtV4Eu0jEaGx0le2rYUF7NhvKq0KOaTyqrj/bHm0FOWjcG90xmYM/k0J9JDMhM0o23O5m2DlucCGxyzn0a2tmzwNXAmibbOCAl9Lw7UNb6ckUknHw+I69HInk9ErlkWM+jy+sbGtmyu4aN5VWs31HNhooqNpZXMW99JfWNwYaezyC/RyIDeyYdDftBPZPpl5FIXIz65zub0wn0PkBJk9elwKQTtvkZMNfM/hFIBC5ubkdmNh2YDpCbm/tFaxWRMIrx+xiQGWyFTxlxbPnh+kY27zoQbMnvqDrasn9rTTmhnCfGZ/TLSAx23WQF++YH9UomLz1BJ2I9FK5Os5uAJ51zvzGzycD/mtkI51xj042cc48Bj0GwyyVMxxaRMIqL8TEo1BJn1LHltXUNfFp54Lhum5Wl+5izcjtHem7j/D76ZyYyONRHPyh0IrZvWgI+n4ZUtrfTCfRtQN8mr3NCy5r6BjAFwDn3kZkFgAygIhxFioj3ArF+hmWnMCw75bjlNYfr2VRRzYby6mD3TXkVxZv38Mqysibv9TEwK9gvP6hJH32f1G4aOx9GpxPoi4GBZtaPYJDfCNx8wjZbgYuAJ81sKBAAKsNZqIh0TglxMYzKSWVUTupxy6tq69hYUX20j35jRRUfbNrJi0uPtQeT4mM4Iysp2GUTatEP7pVMVnK8gr4VThnozrl6M/s28CbBIYkznHOrzeznQLFz7lXg+8AfzexugidI/955NcBdRDqF5EAs43LTGJebdtzyfTV1bKioYv2O4EnYDeXVvL22gueKS49ukxKICQZ8r2QGZQXDfkBWEplJ8eq6OQldWCQincKu6kPHDa3cWF7N+vKqo1fCQrCPvk9aN3KOPhLok3rseVZy9Ae+ZlsUkU6vR1I8k5PimTygx9Flzjkqqw6xvryKzbtqKN1TQ+meg5TuOchba8rZWX34uH3E+X1kpwbISUs4Gvp9QmGfk9aNrOQA/igOfAW6iHRaZkZWSoCslADnDPz8+oOHG9i291jIBx/B139dW8HO6kPHbR/rN7KPtOhTEz4X+D1TIjvwFegiErG6xfk5IyuZM7KSm10fDPyDbNt78LjWfemeGt5ZX0Fl1fGBH+NrEvgndumkJ9Crkwe+Al1EolYw8JM4Iyup2fW1daHAP6F1X7qnhnnrK6loJvB7pwaOtu5z0hKO69PvlRLw9MIqBbqIdFmBWP/Rq2WbU1vXwPZ9tccF/ZFW/vyNlZTvPz7w/T6jd/fA0bDPSesWauEHn/fu3r6Br0AXEWlBINZPv4xE+mUkNrv+UH0D2/fWfq51X7rnIH/buJPyqlqaDiT0+4xeKQFuPSuf287pH/Z6FegiIq0UH+MnPyOR/BYC/3B9I9v3ndidc5DM5Pa5faACXUSkncTF+I7OdNkRNC2aiEiUUKCLiEQJBbqISJRQoIuIRAkFuohIlFCgi4hECQW6iEiUUKCLiEQJz25wYWaVwJZWvj0D2BnGcrykz9I5RctniZbPAfosR+Q55zKbW+FZoLeFmRW3dMeOSKPP0jlFy2eJls8B+iynQ10uIiJRQoEuIhIlIjXQH/O6gDDSZ+mcouWzRMvnAH2WU4rIPnQREfm8SG2hi4jICRToIiJRIuIC3cymmNl6M9tkZvd6XU9rmdkMM6sws1Ve19IWZtbXzN41szVmttrM7vK6ptYys4CZLTKz5aHPcr/XNbWVmfnN7GMz+4vXtbSFmW02s5VmtszMir2up7XMLNXMnjezdWa21swmh3X/kdSHbmZ+YANwCVAKLAZucs6t8bSwVjCzc4Fq4E/OuRFe19NaZtYb6O2cW2pmycAS4JoI/TcxINE5V21mscDfgLuccws8Lq3VzOx7QAGQ4py70ut6WsvMNgMFzrmIvrDIzJ4C3nfOPW5mcUCCc25vuPYfaS30icAm59ynzrnDwLPA1R7X1CrOufnAbq/raCvn3Hbn3NLQ8ypgLdDH26paxwVVh17Ghh6R0+I5gZnlAFcAj3tdi4CZdQfOBZ4AcM4dDmeYQ+QFeh+gpMnrUiI0PKKRmeUDY4GFHpfSaqEuimVABfCWcy5iPwvwO+D/AY0e1xEODphrZkvMbLrXxbRSP6AS+J9QN9jjZhbWm41GWqBLJ2VmScALwHedc/u9rqe1nHMNzrkxQA4w0cwisjvMzK4EKpxzS7yuJUzOds6NA6YCd4a6LCNNDDAOeNQ5NxY4AIT1PGCkBfo2oG+T1zmhZeKhUH/zC8DTzrkXva4nHEK/Cr8LTPG4lNY6C7gq1Pf8LHChmc30tqTWc85tC/1ZAbxEsPs10pQCpU1+63ueYMCHTaQF+mJgoJn1C51QuBF41eOaurTQicQngLXOuQe9rqctzCzTzFJDz7sRPPm+ztOiWsk5d59zLsc5l0/w5+Qd51yRx2W1ipklhk64E+qiuBSIuNFhzrkdQImZDQ4tuggI6+CBmHDurL055+rN7NvAm4AfmOGcW+1xWa1iZrOA84EMMysF/tk594S3VbXKWcBXgZWhvmeAHznn5nhXUqv1Bp4KjabyAc855yJ6uF+U6Am8FGw7EAM845x7w9uSWu0fgadDDdJPgVvDufOIGrYoIiIti7QuFxERaYECXUQkSijQRUSihAJdRCRKKNBFRKKEAl1EJEoo0EVEosT/BxbiiG78pe5YAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "pyplot.plot(history.history['loss'], label='train')\n",
    "pyplot.plot(history.history['val_loss'], label='test')\n",
    "pyplot.legend()\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, let’s build the dictionary to convert the index to word for target and source vocabulary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "reverse_target_word_index = y_tokenizer.index_word\n",
    "reverse_source_word_index = x_tokenizer.index_word\n",
    "target_word_index = y_tokenizer.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference\n",
    "\n",
    "Set up the inference for the encoder and decoder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encode the input sequence to get the feature vector\n",
    "encoder_model = Model(inputs=encoder_inputs, outputs=[encoder_outputs, state_h, state_c])\n",
    "\n",
    "# Decoder setup\n",
    "# Below tensors will hold the states of the previous time step\n",
    "decoder_state_input_h = Input(shape=(latent_dim*2,))\n",
    "decoder_state_input_c = Input(shape=(latent_dim*2,))\n",
    "decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim*2))\n",
    "\n",
    "# Get the embeddings of the decoder sequence\n",
    "dec_emb2= dec_emb_layer(decoder_inputs) \n",
    "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
    "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
    "\n",
    "#attention inference\n",
    "attn_out_inf, attn_states_inf = attn_layer([decoder_hidden_state_input, decoder_outputs2])\n",
    "decoder_inf_concat = Concatenate(axis=-1, name='concat')([decoder_outputs2, attn_out_inf])\n",
    "\n",
    "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
    "decoder_outputs2 = decoder_dense(decoder_inf_concat) \n",
    "\n",
    "# Final decoder model\n",
    "decoder_model = Model(\n",
    "    [decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n",
    "    [decoder_outputs2] + [state_h2, state_c2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us define the functions to convert an integer sequence to a word sequence for summary as well as the reviews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
    "    \n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1,1))\n",
    "    \n",
    "    # Populate the first word of target sequence with the start word.\n",
    "    target_seq[0, 0] = target_word_index['sostok']\n",
    "\n",
    "    stop_condition = False\n",
    "    decoded_sentence = ''\n",
    "    while not stop_condition:\n",
    "      \n",
    "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c])\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
    "        \n",
    "        if(sampled_token!='eostok'):\n",
    "            decoded_sentence += ' '+sampled_token\n",
    "\n",
    "        # Exit condition: either hit max length or find stop word.\n",
    "        if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_summary_len-1)):\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1,1))\n",
    "        target_seq[0, 0] = sampled_token_index\n",
    "\n",
    "        # Update internal states\n",
    "        e_h, e_c = h, c\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq2summary(input_seq):\n",
    "    newString=''\n",
    "    for i in input_seq:\n",
    "        if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
    "            newString=newString+reverse_target_word_index[i]+' '\n",
    "    return newString\n",
    "\n",
    "def seq2text(input_seq):\n",
    "    newString=''\n",
    "    for i in input_seq:\n",
    "        if(i!=0):\n",
    "            newString=newString+reverse_source_word_index[i]+' '\n",
    "    return newString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0, 2):\n",
    "    print(\"Review:\",seq2text(x_test[i]))\n",
    "    print(\"Original summary:\",seq2summary(y_test[i]))\n",
    "    print(\"Predicted summary:\",decode_sequence(x_test[i].reshape(1,max_text_len)))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Rouge Scores\n",
    "\n",
    "Calculating the Rouge scores and saving it in CSV file for Graph Plotting and Analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge import Rouge\n",
    "ROUGE = Rouge()\n",
    "Original_summary = list()\n",
    "Predicted_summary = list()\n",
    "for i in range(len(x_test)):\n",
    "    Review = seq2text(x_test[i])\n",
    "    Original_summary.append(seq2summary(y_test[i]))\n",
    "    Predicted_summary.append(decode_sequence(x_test[i].reshape(1,max_text_len)))\n",
    "    \n",
    "scores = ROUGE.get_scores(Predicted_summary,Original_summary,avg=True)\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(scores)\n",
    "df.to_csv('./Attentive_Results.csv')\n",
    "print(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "6b07439ecfe0abbcae4d197daa9d8514f9745d93ca336d1633c7b3ec373a5ae2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}