DeepLearningProject2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DZCY4u4pU_FO"
      },
      "source": [
        "># ***Deep Learning Programming Assignment - 2 (Group 28)***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W-XP2cUtUic0"
      },
      "source": [
        ">#### ***Submitted by:***\n",
        "*   Keshav Garg [SE20UCSE65]\n",
        "*   Soumna Nema [SE20UCAM035]\n",
        "*   Khushang Zaveri [SE20UCAM017]\n",
        "*   Teena Sachdeva [SE20UCSE205]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Phase 1 - Tensorflow and Keras"
      ],
      "metadata": {
        "id": "tgx5AEqVZm3k"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "UfecJ0phVt-v"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Activation, Dense\n",
        "from tensorflow.keras import layers, regularizers\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.metrics import categorical_crossentropy\n",
        "import seaborn as sns\n",
        "sns.set_style(\"darkgrid\")\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "HzhUSl0YW4IU"
      },
      "outputs": [],
      "source": [
        "#since we are using GPU\n",
        "devices = tf.config.experimental.list_physical_devices('GPU')\n",
        "tf.config.experimental.set_memory_growth(devices[0], True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "2oEW8bjOYglD"
      },
      "outputs": [],
      "source": [
        "#creating layers\n",
        "#these are all hidden layers,\n",
        "#with the first hidden layer knowing what shape of the input to expect\n",
        "#5 Layers, using tanh as activation function(refer to assignment 1)\n",
        "#layers in model are fully connected\n",
        "\n",
        "model = Sequential([\n",
        "    layers.Dense(units=25, input_shape=(4,), activation = 'tanh', kernel_regularizer=regularizers.l2(0.001)),\n",
        "    layers.Dense(units=10, activation = 'tanh', kernel_regularizer=regularizers.l2(0.001)),\n",
        "    layers.Dense(units=5, activation = 'tanh', kernel_regularizer=regularizers.l2(0.001)),\n",
        "    layers.Dense(units=1)\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "54cp_Q1ucv2n",
        "outputId": "a2ca0f98-d5e7-480c-eb96-5ee97448b64d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " dense_4 (Dense)             (None, 25)                125       \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 10)                260       \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 5)                 55        \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 1)                 6         \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 446 (1.74 KB)\n",
            "Trainable params: 446 (1.74 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "Fv0k4fxCc0Tx"
      },
      "outputs": [],
      "source": [
        "#data pre-processing\n",
        "\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "sheets = pd.read_excel('/content/Folds5x2_pp.xlsx', sheet_name=None)\n",
        "data = pd.concat(sheets, ignore_index=True)\n",
        "\n",
        "#final dataset\n",
        "data.to_csv('/content/Data.csv', index=False)\n",
        "\n",
        "#splitting data into 70 : 20 : 10 for training : validation : testing:\n",
        "\n",
        "\n",
        "# Determine the sizes for training, validation, and test sets\n",
        "total_length = len(data)\n",
        "train_ratio = 0.7\n",
        "val_ratio = 0.2\n",
        "test_ratio = 0.1\n",
        "\n",
        "# Calculate sizes for training and validation sets\n",
        "train_size = int(total_length * train_ratio)\n",
        "val_size = int(total_length * val_ratio)\n",
        "test_size=int(total_length*test_ratio)\n",
        "\n",
        "# Extract a continuous chunk for the test set\n",
        "test_start_index = train_size + val_size\n",
        "test = data[test_start_index:]\n",
        "\n",
        "#for shuffling\n",
        "data = data.sample(frac=1).reset_index(drop=True)\n",
        "# Split data into training and validation sets\n",
        "train = data[:train_size]\n",
        "val = data[train_size:train_size + val_size]\n",
        "\n",
        "\n",
        "# Save the split datasets to new CSV files if needed\n",
        "train.to_csv('train.csv', index=False)\n",
        "val.to_csv('validation.csv', index=False)\n",
        "test.to_csv('test.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "yszxPgDlYXhy"
      },
      "outputs": [],
      "source": [
        "#for training data\n",
        "\n",
        "# Read the original CSV file with 5 columns\n",
        "train_data = pd.read_csv('/content/train.csv')\n",
        "# Extract the x variable into a separate DataFrame\n",
        "train_x = train_data[['AT', 'V', 'AP', 'RH']]\n",
        "# Extract the y variable into a separate DataFrame\n",
        "train_y = train_data[['PE']]\n",
        "\n",
        "#for validation data\n",
        "\n",
        "# Read the original CSV file with 5 columns\n",
        "validation_data = pd.read_csv('/content/validation.csv')\n",
        "# Extract the x variable into a separate DataFrame\n",
        "validation_x = validation_data[['AT', 'V', 'AP', 'RH']]\n",
        "# Extract the y variable into a separate DataFrame\n",
        "validation_y = validation_data[['PE']]\n",
        "\n",
        "#for test data\n",
        "\n",
        "# Read the original CSV file with 5 columns\n",
        "test_data = pd.read_csv('/content/test.csv')\n",
        "# Extract the x variable into a separate DataFrame\n",
        "test_x = test_data[['AT', 'V', 'AP', 'RH']]\n",
        "# Extract the y variable into a separate DataFrame\n",
        "test_y = test_data[['PE']]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "7O8LcL_YbK0l"
      },
      "outputs": [],
      "source": [
        "#normalizing datasets\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "scaler = StandardScaler()\n",
        "train_x_normalized = scaler.fit_transform(train_x)\n",
        "train_y_normalized = scaler.fit_transform(train_y)\n",
        "validation_x_normalized = scaler.fit_transform(validation_x)\n",
        "validation_y_normalized = scaler.fit_transform(validation_y)\n",
        "test_x_normalized = scaler.fit_transform(test_x)\n",
        "test_y_normalized = scaler.fit_transform(test_y)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_x_normalized.reshape(4,33488)\n",
        "train_y_normalized.reshape(1,33488)\n",
        "validation_x_normalized.reshape(4,9568)\n",
        "validation_y_normalized.reshape(1,9568)\n",
        "test_x_normalized.reshape(4,4784)\n",
        "test_y_normalized.reshape(1,4784)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TCGIirlJQ0y8",
        "outputId": "4f770425-0dbe-4bc2-92a0-d4073f2698d5"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-1.56305562, -0.45568859,  2.08059549, ...,  0.67381413,\n",
              "        -0.20171067, -0.15860433]])"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "id": "GJctwWXqerLy"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def plot_mape_vs_epochs(mape_values, epochs):\n",
        "    plt.figure(figsize=(10, 6))\n",
        "    plt.plot(epochs, mape_values, label='MAPE')\n",
        "    plt.xlabel('Epochs')\n",
        "    plt.ylabel('MAPE')\n",
        "    plt.title('Mean Absolute Percentage Error using Tensorflow vs Epochs')\n",
        "    plt.legend()\n",
        "    plt.grid(True)\n",
        "    plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def MAPE(actual, predicted):\n",
        "    return np.mean(np.abs(predicted - actual) / actual) * 100"
      ],
      "metadata": {
        "id": "8JamEdomJ9G_"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batchSizes = [1,64,256,47840]\n",
        "learningRates = [0.01,0.001,0.0001]"
      ],
      "metadata": {
        "id": "qzAAErr4kUOy"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mapeDict = {}\n",
        "for e in batchSizes:\n",
        "  mapeDict[e] = []"
      ],
      "metadata": {
        "id": "9Oz-LiUtlD-m"
      },
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 105,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "yLCnaVxdXlhQ",
        "outputId": "3cac58bd-6e7c-4230-af66-ce5f8789925b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n",
            "33488/33488 - 101s - loss: 0.0950 - val_loss: 0.1116 - 101s/epoch - 3ms/step\n",
            "Epoch 2/3\n",
            "33488/33488 - 101s - loss: 0.0956 - val_loss: 0.1790 - 101s/epoch - 3ms/step\n",
            "Epoch 3/3\n",
            "33488/33488 - 99s - loss: 0.0959 - val_loss: 0.1982 - 99s/epoch - 3ms/step\n",
            "150/150 [==============================] - 0s 2ms/step - loss: 0.1927\n",
            "150/150 [==============================] - 0s 2ms/step\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGiCAYAAADA0E3hAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdYklEQVR4nO3df2xfdb348VdbHERFs36Kg4jGcM1+0Hauk8XQW7IwNZjJuJdhEadO5sJ+uOGPiw4TdVgQqmbDUR1xY7sDrjPLAmyCFOJPJoZukbllYRB17JpVmNh+uitMBnXt+f7Bd9W6iTulXT/v9vFIGrLDOZ/P+9NX1z13zvmsZVmWZQEAkIDy4V4AAMDJEi4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMnKHy69+9atYtGhRNDQ0xIQJE+InP/nJvzxmx44dcfnll0dNTU28//3vj/vuu29AiwUARrfc4fLiiy/GhAkT4oYbbjip/dvb22PhwoXxnve8J37wgx/EJz7xifjyl78cjz76aO7FAgCj22l5D5g+fXpMnz79pPfftGlTnHvuufHFL34xIiL+7d/+LXbu3Bl33nlnXHTRRXmfHgAYxYb8Hpfdu3fHhRde2G9bQ0ND7N69e6ifGgAYYYY8XDo7O6Oqqqrftqqqqjh8+HC89NJLQ/30AMAIkvtS0XDq6nohsmy4VzG6lZVFVFaeaRYlwCxKh1mUFvMoHcdmMZiGPFyqqqqis7Oz37bOzs544xvfGGeccUaux8qyiN7ewVwdeZWVvfLf3t7wDWGYmUXpMIvSYh6lo3wIrusM+aWiKVOmxPbt2/tte+yxx2LKlClD/dQAwAiTO1z+8pe/xFNPPRVPPfVURET84Q9/iKeeeiqeffbZiIhYuXJlLFu2rG//q666Ktrb2+Ob3/xmPP3007Fx48Z46KGH4uqrrx6cVwAAjBq5LxU98cQTMXfu3L5fNzc3R0TE5ZdfHl//+tejo6MjDh482Pf/3/a2t8WaNWuiubk57r777jj77LPja1/7mrdCAwC5lWVZOlcAi8UX3OMyzMrKIqqqzozOTje9DTezKB1mUVrMo3SUl0cUCoN7c66fVQQAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDIGFC4bN26MGTNmRG1tbTQ2NsaePXtedf8777wzLrnkkpg8eXJMnz49brnllnj55ZcHtGAAYPTKHS6tra3R3NwcS5YsiS1btsTEiRNj/vz5USwWT7j/Aw88ECtXroylS5dGa2tr3HzzzdHa2hq33nrra148ADC65A6XDRs2xJVXXhlXXHFFvPOd74ympqY444wz4t577z3h/rt27YqpU6fGrFmz4txzz42Ghoa49NJL/+VZGgCAf3Ranp27u7tj7969sXDhwr5t5eXlUV9fH7t27TrhMXV1dXH//ffHnj17YvLkydHe3h7btm2L//iP/8i92LKyVz4YPsc+/+Yw/MyidJhFaTGP0jEUM8gVLocOHYqenp4oFAr9thcKhdi/f/8Jj5k1a1YcOnQo5syZE1mWxdGjR+Oqq66KRYsW5V5sZeWZuY9haBQKZlEqzKJ0mEVpMY+RKVe4DMSOHTtizZo1ccMNN8TkyZPjwIEDcfPNN8fq1atjyZIluR6rq+uF6O0dooVyUsrKXvlmUCy+EFk23KsZ3cyidJhFaTGP0lFePvgnHXKFy9ixY6OiouK4G3GLxWJUVVWd8JjbbrstLrvssmhsbIyIiAkTJsSLL74Yy5cvj8WLF0d5+cnfZpNl4YuwRJhF6TCL0mEWpcU8ht9QfP5z3Zw7ZsyYqK6ujra2tr5tvb290dbWFnV1dSc85qWXXjouTioqKiIiIvMVBQDkkPtS0bx58+L666+PmpqamDx5ctx1111x5MiRmD17dkRELFu2LMaNGxfXXXddRERcfPHFsWHDhjj//PP7LhXddtttcfHFF/cFDADAycgdLjNnzoyurq5oaWmJjo6OmDRpUqxbt67vUtHBgwf7nWFZvHhxlJWVxapVq+K5556LysrKuPjii+Nzn/vc4L0KAGBUKMsSul5TLLo5d7iVlUVUVZ0ZnZ1uehtuZlE6zKK0mEfpKC8f/Hd3+VlFAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkY0DhsnHjxpgxY0bU1tZGY2Nj7Nmz51X3f/7556OpqSkaGhqipqYmLrnkkti2bduAFgwAjF6n5T2gtbU1mpubo6mpKd71rnfFXXfdFfPnz4+HH344CoXCcft3d3fHvHnzolAoxG233Rbjxo2LZ599Nt70pjcNygsAAEaP3OGyYcOGuPLKK+OKK66IiIimpqZ45JFH4t57740FCxYct/+9994bf/7zn2PTpk3xute9LiIizj333Ne4bABgNMoVLt3d3bF3795YuHBh37by8vKor6+PXbt2nfCYn/3sZzFlypS48cYb46c//WlUVlbGpZdeGtdcc01UVFTkWmxZ2SsfDJ9jn39zGH5mUTrMorSYR+kYihnkCpdDhw5FT0/PcZeECoVC7N+//4THtLe3x/bt22PWrFmxdu3aOHDgQDQ1NcXRo0dj6dKluRZbWXlmrv0ZOoWCWZQKsygdZlFazGNkyn2pKK8sy6JQKMRNN90UFRUVUVNTE88991ysX78+d7h0db0Qvb1DtFBOSlnZK98MisUXIsuGezWjm1mUDrMoLeZROsrLB/+kQ65wGTt2bFRUVESxWOy3vVgsRlVV1QmPOeuss+K0007rd1novPPOi46Ojuju7o4xY8ac9PNnWfgiLBFmUTrMonSYRWkxj+E3FJ//XG+HHjNmTFRXV0dbW1vftt7e3mhra4u6uroTHjN16tQ4cOBA9P7dqZLf//73cdZZZ+WKFgCA3P+Oy7x582Lz5s2xZcuWePrpp+OrX/1qHDlyJGbPnh0REcuWLYuVK1f27f+Rj3wk/u///i9uvvnm+N///d945JFHYs2aNfHRj3508F4FADAq5L7HZebMmdHV1RUtLS3R0dERkyZNinXr1vVdKjp48GCUl/+th84555xYv359NDc3x2WXXRbjxo2LuXPnxjXXXDN4rwIAGBXKsiydK4DFoptzh1tZWURV1ZnR2emmt+FmFqXDLEqLeZSO8vLBf3eXn1UEACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyBhQuGzdujBkzZkRtbW00NjbGnj17Tuq4Bx98MCZMmBCf+tSnBvK0AMAolztcWltbo7m5OZYsWRJbtmyJiRMnxvz586NYLL7qcX/4wx/iG9/4RlxwwQUDXiwAMLqdlveADRs2xJVXXhlXXHFFREQ0NTXFI488Evfee28sWLDghMf09PTE5z//+bj22mtj586d8fzzzw9osWVlr3wwfI59/s1h+JlF6TCL0mIepWMoZpArXLq7u2Pv3r2xcOHCvm3l5eVRX18fu3bt+qfHrV69OgqFQjQ2NsbOnTsHvNjKyjMHfCyDq1Awi1JhFqXDLEqLeYxMucLl0KFD0dPTE4VCod/2QqEQ+/fvP+Exjz/+eNxzzz2xdevWAS/ymK6uF6K39zU/DK9BWdkr3wyKxRciy4Z7NaObWZQOsygt5lE6yssH/6RD7ktFeRw+fDiWLVsWN910U1RWVr7mx8uy8EVYIsyidJhF6TCL0mIew28oPv+5wmXs2LFRUVFx3I24xWIxqqqqjtu/vb09nnnmmVi8eHHftt7/f8rk/PPPj4cffjje/va3D2TdAMAolCtcxowZE9XV1dHW1hbve9/7IuKVEGlra4uPfexjx+1/3nnnxQMPPNBv26pVq+Ivf/lLfOlLX4qzzz77NSwdABhtcl8qmjdvXlx//fVRU1MTkydPjrvuuiuOHDkSs2fPjoiIZcuWxbhx4+K6666L008/PcaPH9/v+De96U0REcdtBwD4V3KHy8yZM6OrqytaWlqio6MjJk2aFOvWreu7VHTw4MEoL/cP8gIAg68sy9K5dalY9K6i4VZWFlFVdWZ0drpbf7iZRekwi9JiHqWjvHzw35bu1AgAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkYULhs3LgxZsyYEbW1tdHY2Bh79uz5p/tu3rw55syZE9OmTYtp06bF1Vdf/ar7AwD8M7nDpbW1NZqbm2PJkiWxZcuWmDhxYsyfPz+KxeIJ99+xY0d88IMfjLvvvjs2bdoU55xzTnzyk5+M55577jUvHgAYXcqyLMvyHNDY2Bi1tbWxfPnyiIjo7e2N6dOnx8c//vFYsGDBvzy+p6cnpk2bFsuXL4///M//zLXYrq4Xorc31yEMsrKyiELhzCgWX4h8XzkMNrMoHWZRWsyjdJSXR1RWnjmoj3lanp27u7tj7969sXDhwr9bVHnU19fHrl27Tuoxjhw5EkePHo03v/nN+VYag//iGbhCwSxKhVmUDrMoLeYxMuUKl0OHDkVPT08UCoV+2wuFQuzfv/+kHmPFihXxlre8Jerr6/M8dUQ441IK/E2mdJhF6TCL0mIepWPYz7i8VmvXro3W1ta4++674/TTT899fJaFL8ISYRalwyxKh1mUFvMYfkPx+c8VLmPHjo2KiorjbsQtFotRVVX1qseuX78+1q5dGxs2bIiJEyfmXykAMOrlelfRmDFjorq6Otra2vq29fb2RltbW9TV1f3T4+644464/fbbY926dVFbWzvw1QIAo1ruS0Xz5s2L66+/PmpqamLy5Mlx1113xZEjR2L27NkREbFs2bIYN25cXHfddRHxyuWhlpaWWLlyZbz1rW+Njo6OiIh4/etfH294wxsG8aUAACNd7nCZOXNmdHV1RUtLS3R0dMSkSZNi3bp1fZeKDh48GOXlfzuRs2nTpvjrX/8an/70p/s9ztKlS+Paa699jcsHAEaT3P+Oy3AqFr2raLiVlUVUVZ0ZnZ3u1h9uZlE6zKK0mEfpKC8f/Lel+1lFAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJAM4QIAJEO4AADJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkY0DhsnHjxpgxY0bU1tZGY2Nj7Nmz51X3f+ihh+IDH/hA1NbWxqxZs2Lbtm0DWiwAMLrlDpfW1tZobm6OJUuWxJYtW2LixIkxf/78KBaLJ9z/17/+dVx33XXxoQ99KLZu3Rrvfe97Y8mSJfHb3/72NS8eABhdyrIsy/Ic0NjYGLW1tbF8+fKIiOjt7Y3p06fHxz/+8ViwYMFx+3/2s5+NI0eOxJo1a/q2XXnllTFx4sS48cYbcy22q+uFyLdaBltZWURl5ZlmUQLMonSYRWkxj9JxbBaD6bQ8O3d3d8fevXtj4cKFfdvKy8ujvr4+du3adcJjdu/eHVdffXW/bQ0NDfGTn/wk92IH+8UzcGZROsyidJhFaTGPkSnXpaJDhw5FT09PFAqFftsLhUJ0dnae8JjOzs6oqqo66f0BAP4Z7yoCAJKRK1zGjh0bFRUVx92IWywWjzurckxVVdVxZ1debX8AgH8mV7iMGTMmqquro62trW9bb29vtLW1RV1d3QmPmTJlSmzfvr3ftsceeyymTJmSf7UAwKiW+1LRvHnzYvPmzbFly5Z4+umn46tf/WocOXIkZs+eHRERy5Yti5UrV/btP3fu3Hj00Ufjv//7v+Ppp5+Ob3/72/HEE0/Exz72scF7FQDAqJDrXUURETNnzoyurq5oaWmJjo6OmDRpUqxbt67v0s/BgwejvPxvPTR16tRYsWJFrFq1Km699dZ4xzveEatXr47x48cP3qsAAEaF3P+OCwDAcPGuIgAgGcIFAEiGcAEAkiFcAIBklEy4bNy4MWbMmBG1tbXR2NgYe/bsedX9H3roofjABz4QtbW1MWvWrNi2bdspWunIl2cWmzdvjjlz5sS0adNi2rRpcfXVV//L2XHy8v6+OObBBx+MCRMmxKc+9akhXuHokXcWzz//fDQ1NUVDQ0PU1NTEJZdc4vvUIMk7izvvvDMuueSSmDx5ckyfPj1uueWWePnll0/RakeuX/3qV7Fo0aJoaGiICRMmnNTPINyxY0dcfvnlUVNTE+9///vjvvvuy//EWQl48MEHs+rq6uyee+7Jfve732Vf/vKXswsuuCDr7Ow84f47d+7MJk2alN1xxx3Zvn37sm9961tZdXV19pvf/OYUr3zkyTuL//qv/8q+973vZU8++WS2b9++7Itf/GL27ne/O/vjH/94ilc+8uSdxTHt7e3ZRRddlM2ZMydbvHjxKVrtyJZ3Fi+//HI2e/bs7Jprrskef/zxrL29PduxY0f21FNPneKVjzx5Z3H//fdnNTU12f3335+1t7dnjz76aPbv//7v2S233HKKVz7yPPLII9mtt96a/ehHP8rGjx+f/fjHP37V/Q8cOJC9613vypqbm7N9+/Zl//M//5NNmjQp+8UvfpHreUsiXD70oQ9lTU1Nfb/u6enJGhoasjVr1pxw/8985jPZggUL+m1rbGzMvvKVrwzpOkeDvLP4R0ePHs3q6uqyLVu2DNEKR4+BzOLo0aPZhz/84Wzz5s3Z9ddfL1wGSd5ZfP/738/e+973Zt3d3adqiaNG3lk0NTVlc+fO7betubk5u+qqq4Z0naPNyYTLN7/5zeyDH/xgv22f/exns09+8pO5nmvYLxV1d3fH3r17o76+vm9beXl51NfXx65du054zO7du+PCCy/st62hoSF27949lEsd8QYyi3905MiROHr0aLz5zW8eqmWOCgOdxerVq6NQKERjY+OpWOaoMJBZ/OxnP4spU6bEjTfeGPX19XHppZfGd7/73ejp6TlVyx6RBjKLurq62Lt3b9/lpPb29ti2bVtMnz79lKyZvxmsP7tz/8u5g+3QoUPR09MThUKh3/ZCoRD79+8/4TGdnZ3H/ZDGQqFw3A9zJJ+BzOIfrVixIt7ylrf0+8ZCfgOZxeOPPx733HNPbN269RSscPQYyCza29tj+/btMWvWrFi7dm0cOHAgmpqa4ujRo7F06dJTsewRaSCzmDVrVhw6dCjmzJkTWZbF0aNH46qrropFixadiiXzd070Z3dVVVUcPnw4XnrppTjjjDNO6nGG/YwLI8fatWujtbU1vvOd78Tpp58+3MsZVQ4fPhzLli2Lm266KSorK4d7OaNelmVRKBTipptuipqampg5c2YsWrQoNm3aNNxLG3V27NgRa9asiRtuuCHuu++++M53vhPbtm2L1atXD/fSGKBhP+MyduzYqKioiGKx2G97sVg8rsyOqaqqOu7syqvtz8kZyCyOWb9+faxduzY2bNgQEydOHMpljgp5Z9He3h7PPPNMLF68uG9bb29vREScf/758fDDD8fb3/72oV30CDWQ3xdnnXVWnHbaaVFRUdG37bzzzouOjo7o7u6OMWPGDOmaR6qBzOK2226Lyy67rO/y6YQJE+LFF1+M5cuXx+LFi/v9bD2G1on+7O7s7Iw3vvGNJ322JaIEzriMGTMmqquro62trW9bb29vtLW1RV1d3QmPmTJlSmzfvr3ftsceeyymTJkylEsd8QYyi4iIO+64I26//fZYt25d1NbWnoqljnh5Z3HeeefFAw88EFu3bu37mDFjRrznPe+JrVu3xtlnn30qlz+iDOT3xdSpU+PAgQN98RgR8fvf/z7OOuss0fIaDGQWL7300nFxciwoMz+q75QatD+78903PDQefPDBrKamJrvvvvuyffv2ZV/5yleyCy64IOvo6MiyLMu+8IUvZCtWrOjbf+fOndn555+frV+/Ptu3b1/W0tLi7dCDJO8s1qxZk1VXV2cPP/xw9qc//anv4/Dhw8P1EkaMvLP4R95VNHjyzuLZZ5/N6urqshtvvDHbv39/9vOf/zy78MILs9tvv324XsKIkXcWLS0tWV1dXfbDH/4wO3DgQPbLX/4ye9/73pd95jOfGaZXMHIcPnw4e/LJJ7Mnn3wyGz9+fLZhw4bsySefzJ555pksy7JsxYoV2Re+8IW+/Y+9Hfob3/hGtm/fvux73/vegN4OPeyXiiIiZs6cGV1dXdHS0hIdHR0xadKkWLduXd+pv4MHD/Yr5qlTp8aKFSti1apVceutt8Y73vGOWL16dYwfP364XsKIkXcWmzZtir/+9a/x6U9/ut/jLF26NK699tpTuvaRJu8sGDp5Z3HOOefE+vXro7m5OS677LIYN25czJ07N6655prhegkjRt5ZLF68OMrKymLVqlXx3HPPRWVlZVx88cXxuc99brhewojxxBNPxNy5c/t+3dzcHBERl19+eXz961+Pjo6OOHjwYN//f9vb3hZr1qyJ5ubmuPvuu+Pss8+Or33ta3HRRRflet6yLHOuDABIg7+uAQDJEC4AQDKECwCQDOECACRDuAAAyRAuAEAyhAsAkAzhAgAkQ7gAAMkQLgBAMoQLAJCM/wfH+cz+KKMbEwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "mapeList = []\n",
        "epoch = []\n",
        "epochs = 3\n",
        "for k in range(epochs):\n",
        "      epoch.append(k)\n",
        "for i in learningRates:\n",
        "  for e in batchSizes:\n",
        "    model.compile(optimizer=Adam(learning_rate=i), loss=\"mean_squared_error\")\n",
        "    model.fit(x=train_x_normalized, y=train_y_normalized, validation_data=(validation_x_normalized,validation_y_normalized),batch_size = e, epochs=3, verbose=2)\n",
        "\n",
        "    #testing\n",
        "    model.evaluate(test_x_normalized, test_y_normalized)\n",
        "\n",
        "\n",
        "    #MAPE\n",
        "    pred = model.predict(test_x_normalized)\n",
        "    actual = test_y_normalized\n",
        "    mape = MAPE(actual, pred)\n",
        "    mapeList.append(mape)\n",
        "\n",
        "\n",
        "  #finding mape and comparing with results from Project 1\n",
        "  plt.plot(epoch,mapeList)\n",
        "  plt.xlabel('Number of Epochs')\n",
        "  plt.ylabel('MAPE (%) using tanh as Activation Function')\n",
        "  plt.title('MAPE per Batch Epoch - Using TensorFlow and Keras')\n",
        "  plt.show()\n",
        "  print(\"Learning Rate:\", end=\"\")\n",
        "  print(i)\n",
        "  print(\"Batch Size:\", end=\"\")\n",
        "  print(e)\n",
        "  mapeList = []\n",
        "  epoch = []\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "  # plt.plot(np.array(epoch).reshape(1,-1), np.array(mapeList).reshape(1,-1))\n",
        "  # plt.xlabel('Number of Epochs')\n",
        "  # plt.ylabel('MAPE (%) using tanh as Activation Function')\n",
        "  # plt.title('MAPE per Batch Epoch - Using TensorFlow and Keras')\n",
        "  # plt.show()\n",
        "  # print(\"Learning Rate:\", end=\"\")\n",
        "  # print(i)\n",
        "  # print(\"Batch Size:\", end=\"\")\n",
        "  # print(e)\n",
        "  # print(\"\")\n",
        "  # mapeList = []\n",
        "\n",
        "  mapeList"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Un8jJegsWwyz",
        "outputId": "54bad853-98e2-45cb-99a0-1126cbed9de8"
      },
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[5.384121445045871]"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "epoch"
      ],
      "metadata": {
        "id": "wNzEIB5rX0Tq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7dbf1bd1-4890-4ed8-ba21-8c5e749906c3"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Phase 2 - Machine Translation"
      ],
      "metadata": {
        "id": "QpiggUEyZeJD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import string\n",
        "import re\n",
        "from pickle import dump\n",
        "from unicodedata import normalize\n",
        "from numpy import array\n",
        "\n",
        "# load doc into memory\n",
        "def load_doc(filename):\n",
        "\t# open the file as read only\n",
        "\tfile = open(filename, mode='rt', encoding='utf-8')\n",
        "\t# read all text\n",
        "\ttext = file.read()\n",
        "\t# close the file\n",
        "\tfile.close()\n",
        "\treturn text\n",
        "\n",
        "# split a loaded document into sentences\n",
        "def to_pairs(doc):\n",
        "\tlines = doc.strip().split('\\n')\n",
        "\tpairs = [line.split('\\t') for line in  lines]\n",
        "\treturn pairs\n",
        "\n",
        "# clean a list of lines\n",
        "def clean_pairs(lines):\n",
        "\tcleaned = list()\n",
        "\t# prepare regex for char filtering\n",
        "\tre_print = re.compile('[^%s]' % re.escape(string.printable))\n",
        "\t# prepare translation table for removing punctuation\n",
        "\ttable = str.maketrans('', '', string.punctuation)\n",
        "\tfor pair in lines:\n",
        "\t\tclean_pair = list()\n",
        "\t\tfor line in pair:\n",
        "\t\t\t# normalize unicode characters\n",
        "\t\t\tline = normalize('NFD', line).encode('ascii', 'ignore')\n",
        "\t\t\tline = line.decode('UTF-8')\n",
        "\t\t\t# tokenize on white space\n",
        "\t\t\tline = line.split()\n",
        "\t\t\t# convert to lowercase\n",
        "\t\t\tline = [word.lower() for word in line]\n",
        "\t\t\t# remove punctuation from each token\n",
        "\t\t\tline = [word.translate(table) for word in line]\n",
        "\t\t\t# remove non-printable chars form each token\n",
        "\t\t\tline = [re_print.sub('', w) for w in line]\n",
        "\t\t\t# remove tokens with numbers in them\n",
        "\t\t\tline = [word for word in line if word.isalpha()]\n",
        "\t\t\t# store as string\n",
        "\t\t\tclean_pair.append(' '.join(line))\n",
        "\t\tcleaned.append(clean_pair)\n",
        "\treturn array(cleaned)\n",
        "\n",
        "# save a list of clean sentences to file\n",
        "def save_clean_data(sentences, filename):\n",
        "\tdump(sentences, open(filename, 'wb'))\n",
        "\tprint('Saved: %s' % filename)\n",
        "\n",
        "# load dataset\n",
        "filename = 'deu.txt'\n",
        "doc = load_doc(filename)\n",
        "# split into english-german pairs\n",
        "pairs = to_pairs(doc)\n",
        "# clean sentences\n",
        "clean_pairs = clean_pairs(pairs)\n",
        "# save clean pairs to file\n",
        "save_clean_data(clean_pairs, 'english-german.pkl')\n",
        "# spot check\n",
        "for i in range(100):\n",
        "\tprint('[%s] => [%s]' % (clean_pairs[i,0], clean_pairs[i,1]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UjGDvKucVL1t",
        "outputId": "be67f16e-8c4b-4669-97de-5c0c01193f4a"
      },
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved: english-german.pkl\n",
            "[hi] => [hallo]\n",
            "[hi] => [gru gott]\n",
            "[run] => [lauf]\n",
            "[wow] => [potzdonner]\n",
            "[wow] => [donnerwetter]\n",
            "[fire] => [feuer]\n",
            "[help] => [hilfe]\n",
            "[help] => [zu hulf]\n",
            "[stop] => [stopp]\n",
            "[wait] => [warte]\n",
            "[hello] => [hallo]\n",
            "[i try] => [ich probiere es]\n",
            "[i won] => [ich hab gewonnen]\n",
            "[i won] => [ich habe gewonnen]\n",
            "[smile] => [lacheln]\n",
            "[cheers] => [zum wohl]\n",
            "[freeze] => [keine bewegung]\n",
            "[freeze] => [stehenbleiben]\n",
            "[got it] => [verstanden]\n",
            "[got it] => [einverstanden]\n",
            "[he ran] => [er rannte]\n",
            "[he ran] => [er lief]\n",
            "[hop in] => [mach mit]\n",
            "[hug me] => [druck mich]\n",
            "[hug me] => [nimm mich in den arm]\n",
            "[hug me] => [umarme mich]\n",
            "[i fell] => [ich fiel]\n",
            "[i fell] => [ich fiel hin]\n",
            "[i fell] => [ich sturzte]\n",
            "[i fell] => [ich bin hingefallen]\n",
            "[i fell] => [ich bin gesturzt]\n",
            "[i know] => [ich wei]\n",
            "[i lied] => [ich habe gelogen]\n",
            "[i lost] => [ich habe verloren]\n",
            "[im] => [ich bin jahre alt]\n",
            "[im] => [ich bin]\n",
            "[im ok] => [mir gehts gut]\n",
            "[im ok] => [es geht mir gut]\n",
            "[no way] => [unmoglich]\n",
            "[no way] => [das gibts doch nicht]\n",
            "[no way] => [ausgeschlossen]\n",
            "[no way] => [in keinster weise]\n",
            "[really] => [wirklich]\n",
            "[really] => [echt]\n",
            "[really] => [im ernst]\n",
            "[thanks] => [danke]\n",
            "[try it] => [versuchs]\n",
            "[why me] => [warum ich]\n",
            "[ask tom] => [frag tom]\n",
            "[ask tom] => [fragen sie tom]\n",
            "[ask tom] => [fragt tom]\n",
            "[be cool] => [entspann dich]\n",
            "[be fair] => [sei nicht ungerecht]\n",
            "[be fair] => [sei fair]\n",
            "[be nice] => [sei nett]\n",
            "[be nice] => [seien sie nett]\n",
            "[beat it] => [geh weg]\n",
            "[beat it] => [hau ab]\n",
            "[beat it] => [verschwinde]\n",
            "[beat it] => [verdufte]\n",
            "[beat it] => [mach dich fort]\n",
            "[beat it] => [zieh leine]\n",
            "[beat it] => [mach dich vom acker]\n",
            "[beat it] => [verzieh dich]\n",
            "[beat it] => [verkrumele dich]\n",
            "[beat it] => [troll dich]\n",
            "[beat it] => [zisch ab]\n",
            "[beat it] => [pack dich]\n",
            "[beat it] => [mach ne fliege]\n",
            "[beat it] => [schwirr ab]\n",
            "[beat it] => [mach die sause]\n",
            "[beat it] => [scher dich weg]\n",
            "[beat it] => [scher dich fort]\n",
            "[call me] => [ruf mich an]\n",
            "[come in] => [komm herein]\n",
            "[come in] => [herein]\n",
            "[come on] => [komm]\n",
            "[come on] => [kommt]\n",
            "[come on] => [mach schon]\n",
            "[come on] => [macht schon]\n",
            "[get out] => [raus]\n",
            "[go away] => [geh weg]\n",
            "[go away] => [hau ab]\n",
            "[go away] => [verschwinde]\n",
            "[go away] => [verdufte]\n",
            "[go away] => [mach dich fort]\n",
            "[go away] => [zieh leine]\n",
            "[go away] => [mach dich vom acker]\n",
            "[go away] => [verzieh dich]\n",
            "[go away] => [verkrumele dich]\n",
            "[go away] => [troll dich]\n",
            "[go away] => [zisch ab]\n",
            "[go away] => [pack dich]\n",
            "[go away] => [mach ne fliege]\n",
            "[go away] => [schwirr ab]\n",
            "[go away] => [mach die sause]\n",
            "[go away] => [scher dich weg]\n",
            "[go away] => [scher dich fort]\n",
            "[go away] => [geh weg]\n",
            "[go away] => [verpiss dich]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pickle import load\n",
        "from pickle import dump\n",
        "from numpy.random import rand\n",
        "from numpy.random import shuffle\n",
        "\n",
        "# load a clean dataset\n",
        "def load_clean_sentences(filename):\n",
        "\treturn load(open(filename, 'rb'))\n",
        "\n",
        "# save a list of clean sentences to file\n",
        "def save_clean_data(sentences, filename):\n",
        "\tdump(sentences, open(filename, 'wb'))\n",
        "\tprint('Saved: %s' % filename)\n",
        "\n",
        "# load dataset\n",
        "raw_dataset = load_clean_sentences('english-german.pkl')\n",
        "\n",
        "# reduce dataset size\n",
        "n_sentences = 10000\n",
        "dataset = raw_dataset[:n_sentences, :]\n",
        "# random shuffle\n",
        "shuffle(dataset)\n",
        "# split into train/test\n",
        "train, test = dataset[:9000], dataset[9000:]\n",
        "# save\n",
        "save_clean_data(dataset, 'english-german-both.pkl')\n",
        "save_clean_data(train, 'english-german-train.pkl')\n",
        "save_clean_data(test, 'english-german-test.pkl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JwGcZgEYVRds",
        "outputId": "1205573f-2b32-4512-c853-36c0b3ba4c62"
      },
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved: english-german-both.pkl\n",
            "Saved: english-german-train.pkl\n",
            "Saved: english-german-test.pkl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from pickle import load\n",
        "from numpy import array\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from keras.models import Sequential\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Embedding\n",
        "from keras.layers import RepeatVector\n",
        "from keras.layers import TimeDistributed\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "\n",
        "def load_clean_sentences(filename):\n",
        "    return load(open(filename,'rb'))\n",
        "\n",
        "dataset=load_clean_sentences('english-german-both.pkl')\n",
        "train=load_clean_sentences('english-german-train.pkl')\n",
        "test=load_clean_sentences('english-german-test.pkl')\n",
        "\n",
        "def create_tokenizer(lines):\n",
        "    tokenizer=Tokenizer()\n",
        "    tokenizer.fit_on_texts(lines)\n",
        "    return tokenizer\n",
        "\n",
        "\n",
        "def max_length(lines):\n",
        "\treturn max(len(line.split()) for line in lines)\n",
        "\n",
        "\n",
        "eng_tokenizer = create_tokenizer(dataset[:, 0])\n",
        "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n",
        "eng_length = max_length(dataset[:, 0])\n",
        "print('English Vocabulary Size: %d' % eng_vocab_size)\n",
        "print('English Max Length: %d' % (eng_length))\n",
        "\n",
        "ger_tokenizer = create_tokenizer(dataset[:, 1])\n",
        "ger_vocab_size = len(ger_tokenizer.word_index) + 1\n",
        "ger_length = max_length(dataset[:, 1])\n",
        "print('German Vocabulary Size: %d' % ger_vocab_size)\n",
        "print('German Max Length: %d' % (ger_length))\n",
        "\n",
        "def encode_sequences(tokenizer, length, lines):\n",
        "\tX = tokenizer.texts_to_sequences(lines)\n",
        "\tX = pad_sequences(X, maxlen=length, padding='post')\n",
        "\treturn X\n",
        "\n",
        "def encode_output(sequences, vocab_size):\n",
        "\tylist = list()\n",
        "\tfor sequence in sequences:\n",
        "\t\tencoded = to_categorical(sequence, num_classes=vocab_size)\n",
        "\t\tylist.append(encoded)\n",
        "\ty = array(ylist)\n",
        "\ty = y.reshape(sequences.shape[0], sequences.shape[1], vocab_size)\n",
        "\treturn y\n",
        "\n",
        "\n",
        "\n",
        "trainX = encode_sequences(ger_tokenizer, ger_length, train[:, 1])\n",
        "trainY = encode_sequences(eng_tokenizer, eng_length, train[:, 0])\n",
        "trainY = encode_output(trainY, eng_vocab_size)\n",
        "\n",
        "testX = encode_sequences(ger_tokenizer, ger_length, test[:, 1])\n",
        "testY = encode_sequences(eng_tokenizer, eng_length, test[:, 0])\n",
        "testY = encode_output(testY, eng_vocab_size)\n",
        "\n",
        "def define_model(src_vocab, tar_vocab, src_timesteps, tar_timesteps, n_units):\n",
        "\tmodel = Sequential()\n",
        "\tmodel.add(Embedding(src_vocab, n_units, input_length=src_timesteps, mask_zero=True))\n",
        "\tmodel.add(LSTM(n_units))\n",
        "\tmodel.add(RepeatVector(tar_timesteps))\n",
        "\tmodel.add(LSTM(n_units, return_sequences=True))\n",
        "\tmodel.add(TimeDistributed(Dense(tar_vocab, activation='softmax')))\n",
        "\treturn model\n",
        "\n",
        "\n",
        "model = define_model(ger_vocab_size, eng_vocab_size, ger_length, eng_length, 256)\n",
        "model.compile(optimizer='adam', loss='categorical_crossentropy')\n",
        "\n",
        "print(model.summary())\n",
        "plot_model(model, to_file='model.png', show_shapes=True)\n",
        "\n",
        "\n",
        "filename = 'model.h5'\n",
        "checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
        "print(len(trainX))\n",
        "model.fit(trainX, trainY, epochs=30, batch_size=64, validation_data=(testX, testY), callbacks=[checkpoint], verbose=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dX88PJu4WqS6",
        "outputId": "9d1e889c-f752-4582-a89a-8caa2e28b977"
      },
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Vocabulary Size: 2403\n",
            "English Max Length: 5\n",
            "German Vocabulary Size: 3862\n",
            "German Max Length: 10\n",
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 10, 256)           988672    \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 256)               525312    \n",
            "                                                                 \n",
            " repeat_vector (RepeatVecto  (None, 5, 256)            0         \n",
            " r)                                                              \n",
            "                                                                 \n",
            " lstm_1 (LSTM)               (None, 5, 256)            525312    \n",
            "                                                                 \n",
            " time_distributed (TimeDist  (None, 5, 2403)           617571    \n",
            " ributed)                                                        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 2656867 (10.14 MB)\n",
            "Trainable params: 2656867 (10.14 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "None\n",
            "9000\n",
            "Epoch 1/30\n",
            "\n",
            "Epoch 1: val_loss improved from inf to 3.67718, saving model to model.h5\n",
            "141/141 - 17s - loss: 4.4391 - val_loss: 3.6772 - 17s/epoch - 118ms/step\n",
            "Epoch 2/30\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3079: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Epoch 2: val_loss improved from 3.67718 to 3.54192, saving model to model.h5\n",
            "141/141 - 2s - loss: 3.5444 - val_loss: 3.5419 - 2s/epoch - 17ms/step\n",
            "Epoch 3/30\n",
            "\n",
            "Epoch 3: val_loss improved from 3.54192 to 3.46599, saving model to model.h5\n",
            "141/141 - 2s - loss: 3.4040 - val_loss: 3.4660 - 2s/epoch - 13ms/step\n",
            "Epoch 4/30\n",
            "\n",
            "Epoch 4: val_loss improved from 3.46599 to 3.35916, saving model to model.h5\n",
            "141/141 - 2s - loss: 3.2598 - val_loss: 3.3592 - 2s/epoch - 14ms/step\n",
            "Epoch 5/30\n",
            "\n",
            "Epoch 5: val_loss improved from 3.35916 to 3.27665, saving model to model.h5\n",
            "141/141 - 2s - loss: 3.1223 - val_loss: 3.2767 - 2s/epoch - 12ms/step\n",
            "Epoch 6/30\n",
            "\n",
            "Epoch 6: val_loss improved from 3.27665 to 3.18628, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.9981 - val_loss: 3.1863 - 2s/epoch - 11ms/step\n",
            "Epoch 7/30\n",
            "\n",
            "Epoch 7: val_loss improved from 3.18628 to 3.08420, saving model to model.h5\n",
            "141/141 - 3s - loss: 2.8460 - val_loss: 3.0842 - 3s/epoch - 19ms/step\n",
            "Epoch 8/30\n",
            "\n",
            "Epoch 8: val_loss improved from 3.08420 to 2.99317, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.6974 - val_loss: 2.9932 - 2s/epoch - 14ms/step\n",
            "Epoch 9/30\n",
            "\n",
            "Epoch 9: val_loss improved from 2.99317 to 2.88969, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.5484 - val_loss: 2.8897 - 2s/epoch - 11ms/step\n",
            "Epoch 10/30\n",
            "\n",
            "Epoch 10: val_loss improved from 2.88969 to 2.80773, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.3926 - val_loss: 2.8077 - 2s/epoch - 11ms/step\n",
            "Epoch 11/30\n",
            "\n",
            "Epoch 11: val_loss improved from 2.80773 to 2.75613, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.2506 - val_loss: 2.7561 - 2s/epoch - 11ms/step\n",
            "Epoch 12/30\n",
            "\n",
            "Epoch 12: val_loss improved from 2.75613 to 2.67925, saving model to model.h5\n",
            "141/141 - 2s - loss: 2.1131 - val_loss: 2.6793 - 2s/epoch - 11ms/step\n",
            "Epoch 13/30\n",
            "\n",
            "Epoch 13: val_loss improved from 2.67925 to 2.62349, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.9854 - val_loss: 2.6235 - 2s/epoch - 11ms/step\n",
            "Epoch 14/30\n",
            "\n",
            "Epoch 14: val_loss improved from 2.62349 to 2.55860, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.8652 - val_loss: 2.5586 - 2s/epoch - 11ms/step\n",
            "Epoch 15/30\n",
            "\n",
            "Epoch 15: val_loss improved from 2.55860 to 2.50899, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.7534 - val_loss: 2.5090 - 2s/epoch - 17ms/step\n",
            "Epoch 16/30\n",
            "\n",
            "Epoch 16: val_loss improved from 2.50899 to 2.46850, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.6454 - val_loss: 2.4685 - 2s/epoch - 13ms/step\n",
            "Epoch 17/30\n",
            "\n",
            "Epoch 17: val_loss improved from 2.46850 to 2.43468, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.5426 - val_loss: 2.4347 - 2s/epoch - 11ms/step\n",
            "Epoch 18/30\n",
            "\n",
            "Epoch 18: val_loss improved from 2.43468 to 2.39900, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.4435 - val_loss: 2.3990 - 2s/epoch - 12ms/step\n",
            "Epoch 19/30\n",
            "\n",
            "Epoch 19: val_loss improved from 2.39900 to 2.37116, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.3514 - val_loss: 2.3712 - 2s/epoch - 11ms/step\n",
            "Epoch 20/30\n",
            "\n",
            "Epoch 20: val_loss improved from 2.37116 to 2.33448, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.2582 - val_loss: 2.3345 - 2s/epoch - 11ms/step\n",
            "Epoch 21/30\n",
            "\n",
            "Epoch 21: val_loss improved from 2.33448 to 2.31900, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.1726 - val_loss: 2.3190 - 2s/epoch - 11ms/step\n",
            "Epoch 22/30\n",
            "\n",
            "Epoch 22: val_loss improved from 2.31900 to 2.30873, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.0855 - val_loss: 2.3087 - 2s/epoch - 13ms/step\n",
            "Epoch 23/30\n",
            "\n",
            "Epoch 23: val_loss improved from 2.30873 to 2.28782, saving model to model.h5\n",
            "141/141 - 2s - loss: 1.0072 - val_loss: 2.2878 - 2s/epoch - 17ms/step\n",
            "Epoch 24/30\n",
            "\n",
            "Epoch 24: val_loss improved from 2.28782 to 2.27158, saving model to model.h5\n",
            "141/141 - 2s - loss: 0.9294 - val_loss: 2.2716 - 2s/epoch - 11ms/step\n",
            "Epoch 25/30\n",
            "\n",
            "Epoch 25: val_loss improved from 2.27158 to 2.25518, saving model to model.h5\n",
            "141/141 - 2s - loss: 0.8591 - val_loss: 2.2552 - 2s/epoch - 11ms/step\n",
            "Epoch 26/30\n",
            "\n",
            "Epoch 26: val_loss improved from 2.25518 to 2.24712, saving model to model.h5\n",
            "141/141 - 1s - loss: 0.7945 - val_loss: 2.2471 - 1s/epoch - 11ms/step\n",
            "Epoch 27/30\n",
            "\n",
            "Epoch 27: val_loss improved from 2.24712 to 2.23386, saving model to model.h5\n",
            "141/141 - 2s - loss: 0.7279 - val_loss: 2.2339 - 2s/epoch - 11ms/step\n",
            "Epoch 28/30\n",
            "\n",
            "Epoch 28: val_loss improved from 2.23386 to 2.23229, saving model to model.h5\n",
            "141/141 - 2s - loss: 0.6681 - val_loss: 2.2323 - 2s/epoch - 11ms/step\n",
            "Epoch 29/30\n",
            "\n",
            "Epoch 29: val_loss improved from 2.23229 to 2.21738, saving model to model.h5\n",
            "141/141 - 2s - loss: 0.6136 - val_loss: 2.2174 - 2s/epoch - 11ms/step\n",
            "Epoch 30/30\n",
            "\n",
            "Epoch 30: val_loss did not improve from 2.21738\n",
            "141/141 - 2s - loss: 0.5640 - val_loss: 2.2193 - 2s/epoch - 13ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7baec82b3160>"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pickle import load\n",
        "from numpy import array\n",
        "from numpy import argmax\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import load_model\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "\n",
        "# load a clean dataset\n",
        "def load_clean_sentences(filename):\n",
        "\treturn load(open(filename, 'rb'))\n",
        "\n",
        "# fit a tokenizer\n",
        "def create_tokenizer(lines):\n",
        "\ttokenizer = Tokenizer()\n",
        "\ttokenizer.fit_on_texts(lines)\n",
        "\treturn tokenizer\n",
        "\n",
        "# max sentence length\n",
        "def max_length(lines):\n",
        "\treturn max(len(line.split()) for line in lines)\n",
        "\n",
        "# encode and pad sequences\n",
        "def encode_sequences(tokenizer, length, lines):\n",
        "\t# integer encode sequences\n",
        "\tX = tokenizer.texts_to_sequences(lines)\n",
        "\t# pad sequences with 0 values\n",
        "\tX = pad_sequences(X, maxlen=length, padding='post')\n",
        "\treturn X\n",
        "\n",
        "# map an integer to a word\n",
        "def word_for_id(integer, tokenizer):\n",
        "\tfor word, index in tokenizer.word_index.items():\n",
        "\t\tif index == integer:\n",
        "\t\t\treturn word\n",
        "\treturn None\n",
        "\n",
        "# generate target given source sequence\n",
        "def predict_sequence(model, tokenizer, source):\n",
        "\tprediction = model.predict(source, verbose=0)[0]\n",
        "\tintegers = [argmax(vector) for vector in prediction]\n",
        "\ttarget = list()\n",
        "\tfor i in integers:\n",
        "\t\tword = word_for_id(i, tokenizer)\n",
        "\t\tif word is None:\n",
        "\t\t\tbreak\n",
        "\t\ttarget.append(word)\n",
        "\treturn ' '.join(target)\n",
        "\n",
        "# evaluate the skill of the model\n",
        "def evaluate_model(model, tokenizer, sources, raw_dataset):\n",
        "\tactual, predicted = list(), list()\n",
        "\tfor i, source in enumerate(sources):\n",
        "\t\t# translate encoded source text\n",
        "\t\tsource = source.reshape((1, source.shape[0]))\n",
        "\t\ttranslation = predict_sequence(model, eng_tokenizer, source)\n",
        "\t\traw_target, raw_src = raw_dataset[i]\n",
        "\t\tif i < 10:\n",
        "\t\t\tprint('src=[%s], target=[%s], predicted=[%s]' % (raw_src, raw_target, translation))\n",
        "\t\tactual.append([raw_target.split()])\n",
        "\t\tpredicted.append(translation.split())\n",
        "\t# calculate BLEU score\n",
        "\tprint('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))\n",
        "\tprint('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))\n",
        "\tprint('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))\n",
        "\tprint('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))\n",
        "\n",
        "# load datasets\n",
        "dataset = load_clean_sentences('english-german-both.pkl')\n",
        "train = load_clean_sentences('english-german-train.pkl')\n",
        "test = load_clean_sentences('english-german-test.pkl')\n",
        "# prepare english tokenizer\n",
        "eng_tokenizer = create_tokenizer(dataset[:, 0])\n",
        "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n",
        "eng_length = max_length(dataset[:, 0])\n",
        "# prepare german tokenizer\n",
        "ger_tokenizer = create_tokenizer(dataset[:, 1])\n",
        "ger_vocab_size = len(ger_tokenizer.word_index) + 1\n",
        "ger_length = max_length(dataset[:, 1])\n",
        "# prepare data\n",
        "trainX = encode_sequences(ger_tokenizer, ger_length, train[:, 1])\n",
        "testX = encode_sequences(ger_tokenizer, ger_length, test[:, 1])\n",
        "\n",
        "# load model\n",
        "model = load_model('model.h5')\n",
        "# test on some training sequences\n",
        "print('train')\n",
        "evaluate_model(model, eng_tokenizer, trainX, train)\n",
        "# test on some test sequences\n",
        "print('test')\n",
        "evaluate_model(model, eng_tokenizer, testX, test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FsSuWaCjBPac",
        "outputId": "5345d42c-3496-411d-e4db-be521db0e520"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train\n",
            "src=[ich fuhle deinen schmerz], target=[i feel your pain], predicted=[i feel your]\n",
            "src=[ich hab sie verbrannt], target=[i burned them], predicted=[i burned you]\n",
            "src=[hat es euch freude gemacht], target=[did you enjoy it], predicted=[did you enjoy it]\n",
            "src=[verklagen wir tom], target=[lets sue tom], predicted=[lets sue tom]\n",
            "src=[ich liebe ihren sohn], target=[i love your son], predicted=[i love your son]\n",
            "src=[wir sind viel gelaufen], target=[we walked a lot], predicted=[we have a lot]\n",
            "src=[tom kennt uns], target=[tom knows us], predicted=[tom knows us]\n",
            "src=[sei nicht so schuchtern], target=[dont be shy], predicted=[dont be so shy]\n",
            "src=[tom hat eine waffe], target=[tom has a gun], predicted=[tom has a gun]\n",
            "src=[wie romantisch], target=[how romantic], predicted=[how romantic]\n",
            "BLEU-1: 0.827482\n",
            "BLEU-2: 0.758537\n",
            "BLEU-3: 0.673482\n",
            "BLEU-4: 0.425225\n",
            "test\n",
            "src=[sie gab ihm eine ohrfeige], target=[she slapped him], predicted=[she shot a bear]\n",
            "src=[ihr wart tapfer], target=[you were brave], predicted=[you are]\n",
            "src=[ich kann hier nicht sterben], target=[i cant die here], predicted=[i cant go]\n",
            "src=[probier den an], target=[try this on], predicted=[try the on]\n",
            "src=[das wird nicht funktionieren], target=[that wont work], predicted=[it wont work]\n",
            "src=[ich habe ein zuhause], target=[i have a home], predicted=[i went a]\n",
            "src=[das ist unsere schuld], target=[its our fault], predicted=[its my cd]\n",
            "src=[das gefallt mir], target=[i like that], predicted=[i like it]\n",
            "src=[ich wei was ihr meint], target=[i hear you], predicted=[i know you well]\n",
            "src=[du bist der grote], target=[youre the man], predicted=[you are tallest]\n",
            "BLEU-1: 0.483762\n",
            "BLEU-2: 0.350782\n",
            "BLEU-3: 0.275721\n",
            "BLEU-4: 0.142015\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}