quora-question-pairs-training-attention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Quora question pairs: training a model with attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime, time, json\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Bidirectional, LSTM, dot, Flatten, Dense, Reshape, add, Dropout, BatchNormalization\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.regularizers import l2\n",
    "from keras.callbacks import Callback, ModelCheckpoint\n",
    "from keras import backend as K\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Initialize global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "Q1_TRAINING_DATA_FILE = 'q1_train.npy'\n",
    "Q2_TRAINING_DATA_FILE = 'q2_train.npy'\n",
    "LABEL_TRAINING_DATA_FILE = 'label_train.npy'\n",
    "WORD_EMBEDDING_MATRIX_FILE = 'word_embedding_matrix.npy'\n",
    "NB_WORDS_DATA_FILE = 'nb_words.json'\n",
    "MODEL_WEIGHTS_FILE = 'question_pairs_weights.h5'\n",
    "MAX_SEQUENCE_LENGTH = 25\n",
    "WORD_EMBEDDING_DIM = 300\n",
    "SENT_EMBEDDING_DIM = 128\n",
    "VALIDATION_SPLIT = 0.1\n",
    "TEST_SPLIT = 0.1\n",
    "RNG_SEED = 13371447\n",
    "NB_EPOCHS = 25\n",
    "DROPOUT = 0.2\n",
    "BATCH_SIZE = 516"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Load the dataset, embedding matrix and word count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "q1_data = np.load(open(Q1_TRAINING_DATA_FILE, 'rb'))\n",
    "q2_data = np.load(open(Q2_TRAINING_DATA_FILE, 'rb'))\n",
    "labels = np.load(open(LABEL_TRAINING_DATA_FILE, 'rb'))\n",
    "word_embedding_matrix = np.load(open(WORD_EMBEDDING_MATRIX_FILE, 'rb'))\n",
    "with open(NB_WORDS_DATA_FILE, 'r') as f:\n",
    "    nb_words = json.load(f)['nb_words']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Partition the dataset into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = np.stack((q1_data, q2_data), axis=1)\n",
    "y = labels\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n",
    "Q1_train = X_train[:,0]\n",
    "Q2_train = X_train[:,1]\n",
    "Q1_test = X_test[:,0]\n",
    "Q2_test = X_test[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "question1 = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "question2 = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "\n",
    "q1 = Embedding(nb_words + 1, \n",
    "                 WORD_EMBEDDING_DIM, \n",
    "                 weights=[word_embedding_matrix], \n",
    "                 input_length=MAX_SEQUENCE_LENGTH, \n",
    "                 trainable=False)(question1)\n",
    "q1 = Bidirectional(LSTM(SENT_EMBEDDING_DIM, return_sequences=True), merge_mode=\"sum\")(q1)\n",
    "\n",
    "q2 = Embedding(nb_words + 1, \n",
    "                 WORD_EMBEDDING_DIM, \n",
    "                 weights=[word_embedding_matrix], \n",
    "                 input_length=MAX_SEQUENCE_LENGTH, \n",
    "                 trainable=False)(question2)\n",
    "q2 = Bidirectional(LSTM(SENT_EMBEDDING_DIM, return_sequences=True), merge_mode=\"sum\")(q2)\n",
    "\n",
    "attention = dot([q1,q2], [1,1])\n",
    "attention = Flatten()(attention)\n",
    "attention = Dense((MAX_SEQUENCE_LENGTH*SENT_EMBEDDING_DIM))(attention)\n",
    "attention = Reshape((MAX_SEQUENCE_LENGTH, SENT_EMBEDDING_DIM))(attention)\n",
    "\n",
    "merged = add([q1,attention])\n",
    "merged = Flatten()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "\n",
    "is_duplicate = Dense(1, activation='sigmoid')(merged)\n",
    "\n",
    "model = Model(inputs=[question1,question2], outputs=is_duplicate)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_3 (InputLayer)             (None, 25)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_4 (InputLayer)             (None, 25)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)          (None, 25, 300)       28679100    input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)          (None, 25, 300)       28679100    input_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional)  (None, 25, 128)       439296      embedding_3[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional)  (None, 25, 128)       439296      embedding_4[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dot_2 (Dot)                      (None, 128, 128)      0           bidirectional_3[0][0]            \n",
      "                                                                   bidirectional_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)              (None, 16384)         0           dot_2[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "dense_7 (Dense)                  (None, 3200)          52432000    flatten_3[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)              (None, 25, 128)       0           dense_7[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "add_2 (Add)                      (None, 25, 128)       0           bidirectional_3[0][0]            \n",
      "                                                                   reshape_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)              (None, 3200)          0           add_2[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "dense_8 (Dense)                  (None, 200)           640200      flatten_4[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)              (None, 200)           0           dense_8[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNorm (None, 200)           800         dropout_5[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_9 (Dense)                  (None, 200)           40200       batch_normalization_5[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)              (None, 200)           0           dense_9[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNorm (None, 200)           800         dropout_6[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_10 (Dense)                 (None, 200)           40200       batch_normalization_6[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)              (None, 200)           0           dense_10[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNorm (None, 200)           800         dropout_7[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_11 (Dense)                 (None, 200)           40200       batch_normalization_7[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)              (None, 200)           0           dense_11[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNorm (None, 200)           800         dropout_8[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_12 (Dense)                 (None, 1)             201         batch_normalization_8[0][0]      \n",
      "====================================================================================================\n",
      "Total params: 111,432,993\n",
      "Trainable params: 54,073,193\n",
      "Non-trainable params: 57,359,800\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train the model, checkpointing weights with best validation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training at 2017-06-01 23:44:48.287525\n",
      "Train on 327474 samples, validate on 36387 samples\n",
      "Epoch 1/25\n",
      "242s - loss: 0.5341 - acc: 0.7256 - val_loss: 0.4826 - val_acc: 0.7525\n",
      "Epoch 2/25\n",
      "239s - loss: 0.4395 - acc: 0.7847 - val_loss: 0.4441 - val_acc: 0.7805\n",
      "Epoch 3/25\n",
      "239s - loss: 0.3921 - acc: 0.8124 - val_loss: 0.4037 - val_acc: 0.8045\n",
      "Epoch 4/25\n",
      "239s - loss: 0.3501 - acc: 0.8375 - val_loss: 0.4056 - val_acc: 0.8121\n",
      "Epoch 5/25\n",
      "238s - loss: 0.3093 - acc: 0.8595 - val_loss: 0.4167 - val_acc: 0.8150\n",
      "Epoch 6/25\n",
      "239s - loss: 0.2681 - acc: 0.8814 - val_loss: 0.4222 - val_acc: 0.8171\n",
      "Epoch 7/25\n",
      "239s - loss: 0.2270 - acc: 0.9023 - val_loss: 0.4548 - val_acc: 0.8181\n",
      "Epoch 8/25\n",
      "239s - loss: 0.1894 - acc: 0.9203 - val_loss: 0.4795 - val_acc: 0.8230\n",
      "Epoch 9/25\n",
      "239s - loss: 0.1555 - acc: 0.9365 - val_loss: 0.5398 - val_acc: 0.8231\n",
      "Epoch 10/25\n",
      "229s - loss: 0.1291 - acc: 0.9486 - val_loss: 0.5389 - val_acc: 0.8200\n",
      "Epoch 11/25\n",
      "229s - loss: 0.1077 - acc: 0.9575 - val_loss: 0.6279 - val_acc: 0.8185\n",
      "Epoch 12/25\n",
      "229s - loss: 0.0925 - acc: 0.9639 - val_loss: 0.6428 - val_acc: 0.8208\n",
      "Epoch 13/25\n",
      "229s - loss: 0.0791 - acc: 0.9697 - val_loss: 0.6564 - val_acc: 0.8225\n",
      "Epoch 14/25\n",
      "229s - loss: 0.0677 - acc: 0.9741 - val_loss: 0.6822 - val_acc: 0.8222\n",
      "Epoch 15/25\n",
      "229s - loss: 0.0605 - acc: 0.9770 - val_loss: 0.7541 - val_acc: 0.8209\n",
      "Epoch 16/25\n",
      "238s - loss: 0.0541 - acc: 0.9798 - val_loss: 0.7523 - val_acc: 0.8241\n",
      "Epoch 17/25\n",
      "228s - loss: 0.0485 - acc: 0.9819 - val_loss: 0.7647 - val_acc: 0.8232\n",
      "Epoch 18/25\n",
      "229s - loss: 0.0453 - acc: 0.9835 - val_loss: 0.8060 - val_acc: 0.8239\n",
      "Epoch 19/25\n",
      "229s - loss: 0.0436 - acc: 0.9843 - val_loss: 0.7953 - val_acc: 0.8223\n",
      "Epoch 20/25\n",
      "238s - loss: 0.0390 - acc: 0.9858 - val_loss: 0.8509 - val_acc: 0.8248\n",
      "Epoch 21/25\n",
      "238s - loss: 0.0371 - acc: 0.9865 - val_loss: 0.7879 - val_acc: 0.8263\n",
      "Epoch 22/25\n",
      "229s - loss: 0.0349 - acc: 0.9874 - val_loss: 0.8984 - val_acc: 0.8212\n",
      "Epoch 23/25\n",
      "229s - loss: 0.0330 - acc: 0.9883 - val_loss: 0.8845 - val_acc: 0.8186\n",
      "Epoch 24/25\n",
      "229s - loss: 0.0307 - acc: 0.9891 - val_loss: 0.8727 - val_acc: 0.8263\n",
      "Epoch 25/25\n",
      "229s - loss: 0.0291 - acc: 0.9896 - val_loss: 0.8777 - val_acc: 0.8220\n",
      "Training ended at 2017-06-02 01:22:23.716517\n",
      "Minutes elapsed: 97.590475\n"
     ]
    }
   ],
   "source": [
    "print(\"Starting training at\", datetime.datetime.now())\n",
    "t0 = time.time()\n",
    "callbacks = [ModelCheckpoint(MODEL_WEIGHTS_FILE, monitor='val_acc', save_best_only=True)]\n",
    "history = model.fit([Q1_train, Q2_train],\n",
    "                    y_train,\n",
    "                    epochs=NB_EPOCHS,\n",
    "                    validation_split=VALIDATION_SPLIT,\n",
    "                    verbose=2,\n",
    "                    batch_size=BATCH_SIZE,\n",
    "                    callbacks=callbacks)\n",
    "t1 = time.time()\n",
    "print(\"Training ended at\", datetime.datetime.now())\n",
    "print(\"Minutes elapsed: %f\" % ((t1 - t0) / 60.))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Plot training and validation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAFHCAYAAABTfW5sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFOW59/Hv3bOwzjAsCgQYVhcCMYgG0WiCuBGjRzGi\ngogQT8zrEqOv5rjkKBqvxGNMNCdG4xIEXHEnJhwjBgNiTISoExRBPAoMCGLYZBhm7+f8Ud1Dz9AD\nzTBV3V39+1xXXV3bVD9z08zd9dxVT5lzDhEREQmHSLobICIiIm1HiV1ERCRElNhFRERCRIldREQk\nRJTYRUREQkSJXUREJER8TexmNsPMNpnZsr3s82sz+8jMysxshJ/tERERCTu/z9hnAqe1tNHMvgUM\nds4dAnwfeMDn9oiIiISar4ndOfcGsG0vu5wFPBrb9y2gi5n19LNNIiIiYZbuGnsfYF3C8qexdSIi\nItIK6U7slmSdxrgVERFppfw0v/96oF/Ccl9gQ7IdzUwJX0REcopzLtkJ8F4FccZuJD8zB3gJmAJg\nZqOB7c65TS0dyDmnaR/T9OnT096GXJsUc8U8FybFPPiptXw9YzezJ4ExQHczKwemA4WAc8495Jz7\nHzM73cz+F6gEpvnZnlywZs2adDch5yjmwVPMg+dnzJ2DaBTq66GhIbXX5uuS7bc/2xPXJZv2Z3s0\n6k3x36ulaV/bW8vXxO6cm5TCPlf62QYRkUznHNTVQW2tN9XV7Ts5pbqtvn73cROnxPdLZduyZXDC\nCbsTV+JrsnX7em2eUCMRyMuD/Px9v8bnU90/1X2TTQUFLW/b28+Zeb9TS1Mq20eNat3nKd01dmlj\nU6dOTXcTco5iHry2iLlzUFMDVVXeVF29+zVxPtm6lra3lCj3lUzr670EUljoTYnJK1liSmU5Pp94\n3MLCPZc7d9779vi0fPlUjjpqdwJu6XVv2xJfmyfoSLov5Q4RO5B+/CCZmcuWtorIgXPOS3qVlXtO\nu3btfX5f2+Pz+fnQocPuqX17b4rPt/Ta0rp27fadJJOty8/3ztBEEpkZrhUXzymxh8zChQsZM2ZM\nupuRUxTz3aJR2LEDtm1LPlVUJE/ULU1m0KnTnlNNzUL69RtDp07QsSNNXvdnPj/H+iwHDBjA2rVr\n090MaaZ///5Jr2FobWLPsY+1iKSivh62bIHPP4d//Qu2bm05WSdOX3zhJcyuXZNPxcXQp8+eSbal\nqbAwefsWLgR9l9p/a9euPaCrrcUf1sbdNTpjF8kBzsHOnbBpk5esk02J27Zv9xLxwQfDQQclT9Ld\nuu25rqQk986Cs0nsDDDdzZBmWvp3UVe8SA6qrISNG+Gzz7zX+Pxnn+2ZxPPyoGdPL1k3n5qv797d\n21/CRYk9Mymxy16p3hu8to55NAqbN+9O1s2TduJrfT307g29enmv8fn4FE/UBx3kdW2HhT7nraPE\nnpnaOrGr00wkYNGol5Q//hg++cSb4vPl5d7ZdXFx00Tduzf07w+jRzddV1ysq6lF4i677DL69u3L\nj3/84zbdN9vojF3EB7t2werVTZN2fH7NGujSBQYPhkGDdk+DB0Npqdct3q5dun8DCaNMP2MfOHAg\nM2bMYOzYseluSqB0xi6SIWpqYMUKeP99L2HHE/jHH3tXiA8Y0DR5n3SStzxwYLi6xUWC0NDQQJ4u\n/EiJxvoJmYULF6a7CaHjHGzYAH/6E9x5J1x4IXzlK94V4JMnwyOPLKS+Hk48EX76U1i61DtjX7kS\n5s2De++Fa66Bs86C4cOV1NuCPufhM2XKFMrLyznjjDMoLi7mrrvuIhKJ8Mgjj9C/f39OOukkAM47\n7zx69+5N165dGTNmDB988EHjMaZNm8Ytt9wCwKJFi+jXrx933303PXv2pE+fPsyaNatV+27dupUz\nzzyTLl26cMwxx3DzzTdzwgkn+B+UVtIZu0iC6mpYvtwbF3vZMvjnP71XM/jqV+GII+CUU+C662Do\nUG+0Md1TLXLgHn30URYvXswjjzzCiSeeyNq1a7n++ut5/fXXWblyJZHYmLOnn346s2bNoqCggOuv\nv54LL7yQd999N+kxP/vsMyoqKtiwYQPz58/n3HPPZfz48XTp0mW/9r388sspKiri888/55NPPuG0\n005jwIABfobjgCixh4yuFE6Nc7B+/Z4JfPVqOOQQL4EfcQSMG+e99urV8kVqinnwFHN/tNWFmAdS\nxk+sNZsZt912Gx06dGhcl/icgFtuuYVf/epXVFRUUFRUtMexCgsLufnmm4lEInzrW9+ic+fOfPjh\nh4xK8nSVlvY9+uijeeGFF/jggw9o164dQ4cO5eKLL2bRokWt/yV9psQuOaGhAd57D15/3ZsWL/bW\nx8/Cv/1tuPFGOPxwXbgmuSsTr6vr27dv43w0GuWmm27iueeeY/PmzZgZZsbmzZuTJvbu3bs3nukD\ndOzYkZ07dyZ9n5b2/de//kVDQ0OTdvTr168tfjXfqMYeMqo9eurq4K234K674MwzoUcPuOAC70K3\ns8+GJUu8AVzmz4df/AIuushL8q1J6op58BTzcEo2tGriuieffJI//OEPvPbaa2zfvp01a9bgnPP1\nSv+DDjqI/Px81q9f37hu3bp1vr1fW1Bil1CoqoJFi+D2270aeLducOmlsG4dXHyxdyHbypXw0EPe\nBW/9+6e7xSLSXK9evfjkk08AkibsiooK2rVrR9euXamsrOTGG29s83HWm4tEIpxzzjnceuutVFVV\nsXLlSh599FFf3/NAKbGHTK7UHisq4JVX4Mc/hhNO8EZW+4//8J4sdtVVsHatVzf/9a/h3HO9e8P9\nkisxzySKeTjdcMMN3H777XTr1o3nn39+j6Q9ZcoUSktL6dOnD8OHD+e4447br+Pvz5eAxH3vvfde\ntm/fTu/evbn44ouZNGkS7TK4ZqcBaiQrVFZ6tfEFC7zXDz6Ao46Cb3zDm449Fjp3TncrRTJbpg9Q\nky1uuOEGNm3axMyZM9vkeG09QI3O2EMmLLXHaBT+8Q+44w7v/vBevbx7yEtK4Je/9MZST+x6T2dS\nD0vMs4liLkH68MMPee+99wBYsmQJM2bM4Jxzzklzq1qmq+IlY5SXw6uvehe0LVjgPbzk1FO9e8a/\n+U2dkYtIelRUVDBx4kQ2btzIwQcfzI9+9CPOPPPMdDerReqKl7TZscMb3CWezLdtg5NP9s7ATzkF\nEu4uEZE2oK74zKTHtkrWqq/3utfjibysDI45xjsrP+UU73aziIpDIr5RYs9MqrHLXmVa7XHjRu8W\ns+98x+ta//734Ysv4D//07uP/M9/9q5mP/LI7E3qmRbzXKCYi7RMNXZpc6tWwdy58OKL8OGH3rCs\n48fDffd5F8GJiIh/1BUvByx+Bfvcud70xRfek8zGj/cueissTHcLRQTUFZ+pVGOXjFBb691uNncu\n/P73UFTkJfKzz4ajj87ebnWRMFNiz0yqscte+Vl73LkTnnvOG5K1Vy+4+WYoLfXq5CtWwM9+BqNG\n5V5SV703eIq5xMWfpR43fPhwXn/99ZT23V+XXXYZP/3pT1v980FRjV326vPP4aWXvDPz11+H447z\nzsp//nP40pfS3ToRkabDv77//vsp77s3s2fP5ne/+x2L44+CBH7729+2roEBU2IPmbYYQ7u8HJ5/\nHl54wXvU6WmnwYUXwhNPQJcuB97GsNG45cFTzMVvzjnfHzDjlxzrNJWWfPKJdxZ+zDEwciQsX+49\nn3zTJnj6aZg4UUldRPxz5513MmHChCbrrr76aq6++mpmzZrFl7/8ZYqLixkyZAgPPfRQi8cZOHAg\nr732GgDV1dVMnTqVbt26MXz4cJYuXbrHew4ZMoTi4mKGDx/O3LlzAVi5ciWXXXYZf/vb3ygqKqJb\nt24ATJs2jVtuuaXx5x9++GEOOeQQevTowdlnn83GjRsbt0UiER588EEOPfRQunfvzpVXXnlgAdoP\nSuwhsz+1x1WrvLr4yJHeQ1Q++QR++lPv3vPf/Q5OP711zyfPNar3Bk8xD5+JEyfy8ssvs3PnTgCi\n0SjPPPMMkyZNomfPnsybN48dO3Ywc+ZMrrnmGsrKyvZ5zFtvvZXVq1ezevVqXnnlFWbPnt1k+5Ah\nQ/jrX//Kjh07mD59OpMnT2bTpk0cfvjhPPDAAxx77LFUVFSwdevWPY792muvcdNNN/Hcc8+xceNG\nSktLueCCC5rsM2/ePN5++23Kysp45plnmD9//gFEKHXqis8xH3zgXQD33HPeg1TOOQfuvtt79Gle\nXrpbJyLpZLe1Tdezm77/V96XlpYycuRI5s6dy+TJk1mwYAGdOnVi1KhRTfY74YQTOPXUU1m8eDEj\nRozY6zGfffZZHnjgAbp06UKXLl246qqruP322xu3f+c732mcnzBhAj/72c9YsmRJSuPAP/nkk1xy\nySV89atfBeCOO+6ga9eulJeXU1paCsCNN95IUVERRUVFnHjiiZSVlXHqqaemHJPWUmIPmea1R+e8\nOnk8mVdUeM8nv/9+70K4XLuC3Q+q9wZPMfdHaxJyW5o4cSJPPfUUkydP5qmnnmLSpEkAvPzyy/zk\nJz9h1apVRKNRqqqqOOKII/Z5vA0bNtA34aET/fv3b7L90Ucf5Z577mHNmjUAVFZWsnnz5pTaumHD\nBo466qjG5U6dOtG9e3c+/fTTxsTes2fPxu0dO3Zs7I3wmxJ7CDkH77yzO5nX13vJfOZM+NrXlMxF\nJDNNmDCB6667jk8//ZQXX3yRt956i9raWs4991wef/xxzjrrLCKRCOPHj0/pfvzevXuzbt06hg4d\nCsDatWsbt5WXl3PppZfyl7/8hWOPPRaAI488svG4+7pw7ktf+lKT41VWVrJly5YmXyTSRX/iQ8I5\nWLIEzj9/IYMGQbzUM2eOVzu/6y7vwjgl9banem/wFPNw6tGjB9/85jeZNm0agwYN4tBDD6W2tpba\n2lp69OhBJBLh5ZdfTrlWfd5553HHHXewfft21q9fz29+85vGbZWVlUQiEXr06EE0GmXmzJlNbpXr\n2bMn69evp66uLumxJ02axMyZM1m2bBk1NTXcdNNNjB49+oDuk28r+jOfxaJR+Pvf4dprYcAAmDLF\nG7517lzvwrg77oCjjoIsvWNDRHLQpEmTWLBgARdeeCEAnTt35te//jUTJkygW7duzJkzh7POOqvF\nn088054+fTqlpaUMHDiQcePGMWXKlMZtQ4cO5dprr2X06NH06tWL5cuXc/zxxzduHzt2LMOGDaNX\nr14cfPDBe7zP2LFjuf322znnnHPo06cPq1evZs6cOUnbkWzZTxpSNsvEk/mzz3rd7EVFMGGCNw0b\npiQuIi3TkLKZqa2HlFWNPQtEo/Dmm14yf/55KCnxEvkrr8CXv5zu1omISCZRV3yGamjwhnD9wQ+g\nb1+4/HLo0QNefRXefx+mT0+e1FV7DJ5iHjzFXKRlOmPPIA0NsHixd2b+wgvQs6d3Zv6Xv8Bhh6W7\ndSIikg1UY0+zaNRL5nPmwIsveg9WmTDBuz3tkEPS3ToRCRPV2DOTauwhsXo1PPoozJ4NnTp5j0L9\n619h8OB0t0xERLKZauwBqqz0kvmJJ3oDxWze7F3ZvmwZXH992yR11R6Dp5gHTzEXaZnO2H3mnNfV\nPmuW19V+/PFw5ZVwxhl6wIqIBKt///5Z+yjSMGs+1O2BUo3dJ2vXemfns2ZB+/YwbZr3TPPevdPd\nsuxSVVfF1qqtbKnawpZdW6iorSDqooD3vGSHa5wHcLiU5gHa5bWjc2HnpFPHgo459QfQOUdtQ23j\nVNNQ473W11AfrcfMMGyP14hFUlpn5q3PszwK8gooiBSQH8knP5LfZnGO/w47a3c2mSpqK/ZYt7N2\nJxU13vq6aB15lkdeJI88yyM/kp90Pi8SW25h37xIHs45GlwDURelIdrQZD7qojS4hpTmAQrzCveY\n2uW1S7q+MK+QdvnJt9VH66mqq2JX3S6q6mOv+1huaVttQy11DXWNn5O66O75+mg9BZECCvIKmrx/\nQWT3cnxbbUMtW3ZtYUvVFmobauneoTvdO3anR8ce9OjYg+4dujd97dh0ubhdceD/P6vqqli3Yx3l\nX5Sz7gvvtfyL8t3rdqyjIFJAvy79iFiE7dXb2V69ncraSoraFdG1fVdK2pdQ0r6Erh26UtKupOly\nfD5hv75d+raqxq7E3oZ27fKuZp81C8rK4PzzYepUOPpoDRxT11DHtuptjf+Zt1ZtbXk+YV2Da2j8\nT9+tQzeK2xV7iQMvoPHEEZ8HGhPJ3uaBpEkgPlXXV9OxoGOLib9zYWeKCovoXNiZTgWdkv5Bbf4H\nrvkfuWT7Oeeoqq+iqq6K6vrqxvlkr9X11XtuS/jZmoYaauprkibr5st10brGtsUTRDyJ5Ee8jr2o\nizZ+QUp8jbpoyusaog3UReuoj9ZT11BHg2toTPDxpBBP+vH5gryCPbZHLMKuul17JO+IRfb579V8\nKogUNCbW+Gt9tL7JuvpofdL55vuaWWOSj3+RSZyPWKTxi8De5oEmSTPx3622oZbaaG2TbXtsT/g3\nz4/k07GgIx3yO3ivBR2aLre0vtly+/z2Tb48NE/YBZGCViXbmvoatlRtYfOuzWzetZktu7z5+LrG\n14T1u+p2Nf5dKCosomNBxxanTgWd9r690NvunGP9jvVJE3b5F+VU1FTQt7gvpV1K6delH6XFsdcu\npfQr7ke/Lv0oble8x+9XH61nR80OtldvZ1vVtsaEv606Yb5qG9trtjfZZ1v1Nj677jMl9nRwzhs8\nZtYsb/CY0aO9ZP5v/+adqQelIdrArrpdvPraq4w8diTV9dXeH/f6msb5fU01DU33jX8LT2WK/8FL\nNtU11FFdX03XDl3p3sFL0N07dt89n2xdbD5dZ84N0QYq6ypbTPyJ0/Ily+n9ld5NzmTif3ibrEty\nltO4LrYfQIeCDnTI70D7/PaN881f2+e3T7o+cXu7/HaNybl5sm6+XJhXmJY4Jyb7uoa6Jkk/vq4+\nWr/H9rfffJuvf+PreyTpwrzCwH+HXLFw4cKMeapebUMtW6u2snnXZiprK9lVt4vKusrG3obEKb59\nV90udtW3vM3hGhN0YtKOJ+6DOh3U+KUrKBl7VbyZjQN+hXeh3gzn3J3NtvcDZgMlsX1udM697He7\nDtTGjd7T0mbN8p5jPm2aN3DMl76U+jFqG2r5dMenrNuxjm1V29hZu5PKukoqayubvO51fWy+pr6G\njgUdKVhXQPGHxbTPb7/3Kc/7wx9fLmpXxEH5BzXZJ/FsKdkU74pMZepU2Cnw/xQHIi+SR3G74qTf\nwJtb2JA5f/CyTcQiRPIiFOQVQEHqPxddHWVUn1H73lFCqTCvkF6de9Grc690NyUj+XrGbmYRYBVw\nErABWApc4JxbmbDPg8A7zrkHzWwo8D/OuYFJjpX2M3bnYNEi71nmr74K550H3/0ujBq1Z1d71EXZ\ntHNTY1fOui/WNenaWffFOjbv2kyvzr3o16Uf3Tt0p1NhJzoVxKbCTo3dvI3r97KuQ36HnKoJi4iE\nXaaesY8CPnLOrQUwsznAWcDKhH2iQPy0qAT41Oc27bcdO7wL4e6/30vgl18ODz7UwLqa5ZR/Uc5v\n/1HemLjjyXtDxQZK2pfs0bVzTJ9j6NelH/2K+9G7qHdj/VJERKQt+J1V+gDrEpbX4yX7RLcB883s\nKqAjcLLPbUrZe+95yXzOHDjlFLjvPkfJ4f/k8WWPMWzGUxS3K2Zwt8Fe8i7ux6mDT6VfsVeX6VPc\nh/b5ARbZYzKpDpYrFPPgKebBU8yzh9+JPVkXQvP+9InATOfcPWY2GngcGOZzu1pUW+tdBHf//d7o\ncJdeCvPfKmfB50/yg2WPs/OfO5l8xGQWTFnA0IOGpquZIiIiSfmd2NcDpQnLffFq7YkuAU4DcM79\n3czam1kP59zm5gebOnUqAwYMAKCkpIQRI0Y0foOMj0TV2uWnn17IH/4Af/7zGIYNg+PH/pFRvRfx\nlw7/4L/nLuO4+uP4/uDvc8V5VxCxCAsXLmQTm9rs/bWcvctjxozJqPbkwnJ8Xaa0J1eW4zKlPWFb\njs+vWbOGA+H3xXN5wId4F89tBJYAE51zKxL2mQc845ybHbt47lXnXN8kx2rzi+eiUfjzn72z88WL\nYeLkWg4/82UWbXuc+R/P5+RBJzP5K5M5/ZDTaZevYeJERCQ4rb14ztf7j5xzDcCVwHxgOTDHObfC\nzG4zszNiu10HfM/MyoAngIv9bBPA1q1w993eo1Cv+5HjsJPf5JwZl/N03z48s+6XnDLoFFb/cDXP\nn/c844eOz6qk3vybtfhPMQ+eYh48xTx7+H5JtnPuT8BhzdZNT5hfARzvdzsAamrguuvgscfgG+NX\nccItT7Bo2+P8ngIu6nYRS/59CQO77nGnnYiISNbImZHntm+HM8/dQcWgx4gc+SgbKtdywfALuOiI\nixjZe6TuARcRkYzS2q74nEjsC95bznl33UflwKc4c9gpXDLyu5w86GTdQy4iIhkrI2vs6VTXUMez\ny5/la/eN4dTHTmHkYQfz8bXLefa8Zxg3ZFxok7rqYMFTzIOnmAdPMc8eoctuGys28vA7D/PQ2w/R\nPTKI1c9cwcwrxjPlQj0cQkREwi8UXfHOOd4of4P7lt7HKx+/wvnDzmfQlsv5xY+OYM4cGDs24MaK\niIgcoJyssVfWVvLEe09w39L7qK6v5vKjL+fiERfzyP0l3HMPzJsHRxyRpgaLiIgcgJyqsa/asoqr\n/3Q1pb8qZd5H8/jFKb9gxRUr+MGoH3LbjSXMmAF//WtuJnXVwYKnmAdPMQ+eYp49sqrG/tKHL3Hf\n0vt4d+O7XHLkJbxz6Tv0L+kPQHU1TJkCmzbBG29A165pbqyIiEgaZFVX/KiHR3HF167gvGHnNXly\n2rZtcPbZ0LOn93jV9sE/VE1ERKRN5WSNHaC8HL71LTj1VPjlLyGSlcUFERGRpnKqxh63bBl8/etw\nySVwzz1K6qA6WDoo5sFTzIOnmGePrKqxJ3rtNbjgArj3Xjj//HS3RkREJDNkZVf8k0/CNdfA009D\nwuOZRUREQqO1XfFZdcbuHPziF95Z+oIFMHx4ulskIiKSWbKqKv3DH3pXvb/5ppJ6S1QHC55iHjzF\nPHiKefbIqjP2996DxYuhpCTdLREREclMWVVjr652tGuX7paIiIj4Lydud1NSFxER2busSuyyb6qD\nBU8xD55iHjzFPHsosYuIiIRIVtXYs6WtIiIiByonauwiIiKyd0rsIaM6WPAU8+Ap5sFTzLOHEruI\niEiIqMYuIiKSgVRjFxERESX2sFEdLHiKefAU8+Ap5tlDiV1ERCREVGMXERHJQKqxi4iIiBJ72KgO\nFjzFPHiKefAU8+yhxC4iIhIiqrGLiIhkINXYRURERIk9bFQHC55iHjzFPHiKefZQYhcREQkR1dhF\nREQykGrsIiIiosQeNqqDBU8xD55iHjzFPHsosYuIiISIauwiIiIZSDV2ERERUWIPG9XBgqeYB08x\nD55inj2U2EVEREJENXYREZEMpBq7iIiIKLGHjepgwVPMg6eYB08xzx6+J3YzG2dmK81slZld38I+\n55nZcjN7z8we97tNIiIiYeVrjd3MIsAq4CRgA7AUuMA5tzJhnyHA08CJzrkdZtbDObc5ybFUYxcR\nkZyRqTX2UcBHzrm1zrk6YA5wVrN9vgfc55zbAZAsqYuIiEhq/E7sfYB1CcvrY+sSHQocZmZvmNmb\nZnaaz20KNdXBgqeYB08xD55inj3yfT5+si6E5v3p+cAQ4BtAKbDYzIbFz+ATTZ06lQEDBgBQUlLC\niBEjGDNmDLD7Q5fry3GZ0h4ta9mP5bKysoxqTy4sl5WVZVR7wrgcn1+zZg0Hwu8a+2jgVufcuNjy\nDYBzzt2ZsM9vgb855x6NLf8ZuN4593azY6nGLiIiOSNTa+xLgSFm1t/MCoELgJea7TMXGAtgZj2A\nQ4BPfG6XiIhIKPma2J1zDcCVwHxgOTDHObfCzG4zszNi+7wCbDGz5cAC4Drn3DY/2xVmiV06EgzF\nPHiKefAU8+zhd40d59yfgMOarZvebPla4Fq/2yIiIhJ2GiteREQkA2VqjV1EREQCpMQeMqqDBU8x\nD55iHjzFPHuklNjN7Hkz+3ZsiFgRERHJUCnV2M3sZGAaMBp4FpiVON57EFRjFxGRXNLaGvt+XTxn\nZl2AicCP8YaKfRh4PDYOvK+U2EVEJJf4fvGcmXUHpgL/DrwL/DcwEnh1f99U/KM6WPAU8+Ap5sFT\nzLNHSvexm9kLwOHAY8CZzrmNsU1Pm9k//GqciIiI7J9Ua+xjnXOvBdCevbVBXfEiIpIz/O6KH2pm\nJQlv1tXMLt/fNxMRERF/pZrYv+ec2x5fiI3l/j1/miQHQnWw4CnmwVPMg6eYZ49UE3vEzBq7A8ws\nDyj0p0kiIiLSWqnW2O8CBgAPAA74f8C62MNbAqEau4iI5BJf72OPjTj3feAkwPAew/q72GNZA6HE\nLiIiucTXi+ecc1Hn3G+dc+c6577jnHswyKQuqVMdLHiKefAU8+Ap5tkj1fvYDwHuAL4MtI+vd84N\n8qldIiIi0gqpdsW/AUwH7gHOxBs3PuKcu8Xf5jVpg7riRUQkZ/h9H3sH59wCvC8Ca51ztwLf3t83\nExEREX+lmtirYxfQfWRmV5rZeKCzj+2SVlIdLHiKefAU8+Ap5tkj1cR+NdARuAo4CpgMXOxXo0RE\nRKR19lljjw1Gc6dz7rpgmtRiO1RjFxGRnOFbjT12W9vxrWqViIiIBCrVrvh3zewlM7vIzM6JT762\nTFpFdbDgKebBU8yDp5hnj5TuY8e7d30LMDZhnQNeaPMWiYiISKuldB97JlCNXUREcklra+ypjjw3\nE+8MvQnn3Hf39w1FRETEP6nW2P8IzItNC4BiYKdfjZLWUx0seIp58BTz4Cnm2SOlM3bn3POJy2b2\nFPCGLy0SERGRVmtVjd3MDgPmOeeGtH2TWnxP1dhFRCRn+F1jr6Bpjf0z4Pr9fTMRERHxV6rPYy9y\nzhUnTIdhsRPsAAANdElEQVQ2756XzKA6WPAU8+Ap5sFTzLNHSondzMabWZeE5RIzO9u/ZomIiEhr\npPo89jLn3Ihm6951zh3pW8v2bINq7CIikjP8fh57sv1SHbVOREREApJqYv+Hmd1tZoPNbJCZ3QO8\n7WfDpHVUBwueYh48xTx4inn2SDWx/wCoBZ4GngGqgCv8apSIiIi0jsaKFxERyUC+1tjN7FUzK0lY\n7mpmr+zvm4mIiIi/Uu2K7+Gc2x5fcM5tAw72p0lyIFQHC55iHjzFPHiKefZINbFHzaw0vmBmA0jy\ntDcRERFJr1TvYx8HPAQsiq36BnCpcy6w7njV2EVEJJe0tsae8sVzZnYwcClQBrQHPnfOvb6/b9ha\nSuwiIpJL/L547t/xnsN+bWx6DLh1f99M/Kc6WPAU8+Ap5sFTzLNHqjX2HwJfA9Y6504EjgS27/1H\nREREJGip1tiXOue+ZmZlwDHOuRozW+6cG+Z/ExvboK54ERHJGb4+jx1YH7uPfS7wqpltA9bu75uJ\niIiIv1J9Hvt459x259ytwM3ADECPbc1AqoMFTzEPnmIePMU8e6RaY2/knFvknHvJOVebyv5mNs7M\nVprZKjO7fi/7nWtmUTMbub9tEhEREY+vY8WbWQRYBZwEbACWAhc451Y2268zMA8oAK50zr2T5Fiq\nsYuISM7w+3nsrTUK+Mg5t9Y5VwfMAc5Kst/twJ1Ajc/tERERCTW/E3sfYF3C8vrYukZmNgLo65z7\nH5/bkhNUBwueYh48xTx4inn2SPWq+NZK1oXQ2J9uZgbcA1y8j58BYOrUqQwYMACAkpISRowYwZgx\nY4DdH7pcX47LlPZoWct+LJeVlWVUe3JhuaysLKPaE8bl+PyaNWs4EH7X2EcDtzrnxsWWbwCcc+7O\n2HIx8L/ATryE3gvYAvxb8zq7auwiIpJLfB8rvjXMLA/4EO/iuY3AEmCic25FC/v/Bfj/zrl3k2xT\nYhcRkZyRkRfPOecagCuB+cByYI5zboWZ3WZmZyT7EfbSFS/7ltilI8FQzIOnmAdPMc8eftfYcc79\nCTis2brpLew71u/2iIiIhJmvXfFtSV3xIiKSSzKyK15ERESCpcQeMqqDBU8xD55iHjzFPHsosYuI\niISIauwiIiIZSDV2ERERUWIPG9XBgqeYB08xD55inj2U2EVEREJENXYREZEMpBq7iIiIKLGHjepg\nwVPMg6eYB08xzx5K7CIiIiGiGruIiEgGUo1dRERElNjDRnWw4CnmwVPMg6eYZw8ldhERkRBRjV1E\nRCQDqcYuIiIiSuxhozpY8BTz4CnmwVPMs4cSu4iISIioxi4iIpKBVGMXERERJfawUR0seIp58BTz\n4Cnm2UOJXUREJERUYxcREclAqrGLiIiIEnvYqA4WPMU8eIp58BTz7KHELiIiEiKqsYuIiGQg1dhF\nREREiT1sVAcLnmIePMU8eIp59lBiFxERCRHV2EVERDKQauwiIiKixB42qoMFTzEPnmIePMU8eyix\ni4iIhIhq7CIiIhlINXYRERFRYg8b1cGCp5gHTzEPnmKePZTYRUREQkQ1dhERkQykGruIiIgosYeN\n6mDBU8yDp5gHTzHPHkrsIiIiIaIau4iISAZSjV1ERET8T+xmNs7MVprZKjO7Psn2a8xsuZmVmdmr\nZtbP7zaFmepgwVPMg6eYB08xzx6+JnYziwC/AU4DhgETzezwZru9AxzlnBsBPA/c5WebREREwszX\nGruZjQamO+e+FVu+AXDOuTtb2H8EcK9z7oQk21RjFxGRnJGpNfY+wLqE5fWxdS25BHjZ1xaJiIiE\nmN+JPdk3jaSn3WY2GTgKdcUfENXBgqeYB08xD55inj3yfT7+eqA0YbkvsKH5TmZ2MnAj8A3nXF1L\nB5s6dSoDBgwAoKSkhBEjRjBmzBhg94cu15fjMqU9WtayH8tlZWUZ1Z5cWC4rK8uo9oRxOT6/Zs0a\nDoTfNfY84EPgJGAjsASY6JxbkbDPkcCzwGnOuY/3cizV2EVEJGdkZI3dOdcAXAnMB5YDc5xzK8zs\nNjM7I7bbz4FOwLNm9q6ZzfWzTSIiImHm+33szrk/OecOc84d4pz7r9i66c65P8bmT3HO9XbOjXTO\nHemcO9vvNoVZYpeOBEMxD55iHjzFPHto5DkREZEQ0VjxIiIiGSgja+wiIiISLCX2kFEdLHiKefAU\n8+Ap5tlDiV1ERCREVGMXERHJQKqxi4iIiBJ72KgOFjzFPHiKefAU8+yhxC4iIhIiqrGLiIhkINXY\nRURERIk9bFQHC55iHjzFPHiKefZQYhcREQkR1dhFREQykGrsIiIiosQeNqqDBU8xD55iHjzFPHso\nsYuIiISIauwiIiIZSDV2ERERUWIPG9XBgqeYB08xD55inj2U2EVEREJENXYREZEMpBq7iIiIKLGH\njepgwVPMg6eYB08xzx5K7CIiIiGiGruIiEgGUo1dRERElNjDRnWw4CnmwVPMg6eYZw8ldhERkRBR\njV1ERCQDqcYuIiIiSuxhozpY8BTz4CnmwVPMs4cSu4iISIioxi4iIpKBVGMXERERJfawUR0seIp5\n8BTz4Cnm2UOJXUREJERUYxcREclAqrGLiIiIEnvYqA4WPMU8eIp58BTz7KHELiIiEiKqsYuIiGQg\n1dhFREREiT1sVAcLnmIePMU8eIp59lBiFxERCRHV2EVERDKQauwiIiLif2I3s3FmttLMVpnZ9Um2\nF5rZHDP7yMz+ZmalfrcpzFQHC55iHjzFPHiKefbwNbGbWQT4DXAaMAyYaGaHN9vtEmCrc+4Q4FfA\nz/1sU9iVlZWluwk5RzEPnmIePMU8e/h9xj4K+Mg5t9Y5VwfMAc5qts9ZwOzY/HPAST63KdS2b9+e\n7ibkHMU8eIp58BTz7OF3Yu8DrEtYXh9bl3Qf51wDsN3MuvncLhERkVDyO7Enu5qv+aXtzfexJPtI\nitasWZPuJuQcxTx4innwFPPs4evtbmY2GrjVOTcutnwD4Jxzdybs83Jsn7fMLA/Y6Jw7OMmxlOxF\nRCSntOZ2t3w/GpJgKTDEzPoDG4ELgInN9vkDcDHwFjABeC3ZgVrzy4mIiOQaXxO7c67BzK4E5uN1\n+89wzq0ws9uApc65PwIzgMfM7CNgC17yFxERkVbImpHnREREZN808lyImNkaM/unmb1rZkvS3Z4w\nMrMZZrbJzJYlrOtqZvPN7EMze8XMuqSzjWHTQsynm9l6M3snNo1LZxvDxsz6mtlrZvaBmb1nZlfF\n1uuz7pMkMf9BbP1+f9Z1xh4iZvYJcJRzblu62xJWZnY8sBN41Dl3RGzdncAW59zPY6MrdnXO3ZDO\ndoZJCzGfDlQ45+5Oa+NCysx6Ab2cc2Vm1hl4G2/MkWnos+6LvcT8fPbzs64z9nAx9G/qK+fcG0Dz\nL06JgyzNBs4OtFEh10LMIfnttNIGnHOfOefKYvM7gRVAX/RZ900LMY+P+7Jfn3UlgXBxwCtmttTM\nvpfuxuSQg51zm8D7zwkclOb25IorzKzMzH6nLmH/mNkAYATwd6CnPuv+S4j5W7FV+/VZV2IPl+Oc\nc0cDp+N9EI5Pd4NEfHI/MNg5NwL4DFCXvA9iXcLPAT+MnUWqduuzJDHf78+6EnuIxL5B45z7F/Ai\n3lj94r9NZtYTGutkn6e5PaHnnPuX232B0MPA19LZnjAys3y8BPOYc+73sdX6rPsoWcxb81lXYg8J\nM+sY+6aHmXUCTgXeT2+rQstoWvN6CZgam78Y+H3zH5AD1iTmsaQSdw76rPvhEeAD59x/J6zTZ91f\ne8S8NZ91XRUfEmY2EO8s3eENPPSEc+6/0tuq8DGzJ4ExQHdgEzAdmAs8C/QDyoEJzjk9CquNtBDz\nE/FqkFFgDfD9eO1XDpyZfR14HXgP72+KA24ClgDPoM96m9tLzCexn591JXYREZEQUVe8iIhIiCix\ni4iIhIgSu4iISIgosYuIiISIEruIiEiIKLGLiIiEiBK7iBwwM/ummf0h3e0QESV2EWk7GhRDJAMo\nsYvkEDO70MzeMrN3zOy3ZhYxswozu9vM3jezV82se2zfEWb2t9hTpZ6PP1XKzAbH9iszs3/ERj0E\nKDKzZ81shZk9lrZfUiTHKbGL5AgzOxw4H+8pgCPxhqi8EOgILHHODccb0nJ67EdmAz+KPVXq/YT1\nTwD3xtYfB2yMrR8BXAV8GRhsZsf5/1uJSHP56W6AiATmJGAksNTMDGiPN/Z6FG/8b4DHgefNrBjo\n4px7I7Z+NvBM7EFDfZxzLwE452oBvMOxxDm3MbZcBgwA3gzg9xKRBErsIrnDgNnOuR83WWl2c7P9\nXML+yY7RkpqE+Qb090UkLdQVL5I7FgDnmtlBAGbW1cxKgTzg3Ng+FwJvOOd2AFtjT5wCuAhY5Jyr\nANaZ2VmxYxSaWYdAfwsR2St9oxbJEc65FWb2n8B8M4sAtcCVQCUwKnbmvgmvDg/e87YfjCXuT4Bp\nsfUXAQ+Z2U9ix5iQ7O38+01EZG/02FaRHGdmFc65onS3Q0TahrriRUTf7kVCRGfsIiIiIaIzdhER\nkRBRYhcREQkRJXYREZEQUWIXEREJESV2ERGREFFiFxERCZH/A/z+uEd2Eo1JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f407a3d1f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = pd.DataFrame({'epoch': [ i + 1 for i in history.epoch ],\n",
    "                    'training': history.history['acc'],\n",
    "                    'validation': history.history['val_acc']})\n",
    "ax = acc.iloc[:,:].plot(x='epoch', figsize={5,8}, grid=True)\n",
    "ax.set_ylabel(\"accuracy\")\n",
    "ax.set_ylim([0.0,1.0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Print best validation accuracy and epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum accuracy at epoch 21 = 0.8263\n"
     ]
    }
   ],
   "source": [
    "max_val_acc, idx = max((val, idx) for (idx, val) in enumerate(history.history['val_acc']))\n",
    "print('Maximum accuracy at epoch', '{:d}'.format(idx+1), '=', '{:.4f}'.format(max_val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Evaluate the model with best validation accuracy on the test partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.7956, accuracy = 0.8243\n"
     ]
    }
   ],
   "source": [
    "model.load_weights(MODEL_WEIGHTS_FILE)\n",
    "loss, accuracy = model.evaluate([Q1_test, Q2_test], y_test, verbose=0)\n",
    "print('loss = {0:.4f}, accuracy = {1:.4f}'.format(loss, accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}