abstract_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO7yxwb3rK9z01/tx3iIUwV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/psyuktha/Scrape-ML/blob/PubMed/abstract_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from tensorflow.keras import layers\n"
      ],
      "metadata": {
        "id": "WfyrUTH2n8ld"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igvGu8Isezn4",
        "outputId": "654e9ab5-cf32-4596-8d44-8a9eacbd33e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'pubmed-rct'...\n",
            "remote: Enumerating objects: 39, done.\u001b[K\n",
            "remote: Counting objects: 100% (14/14), done.\u001b[K\n",
            "remote: Compressing objects: 100% (9/9), done.\u001b[K\n",
            "remote: Total 39 (delta 8), reused 5 (delta 5), pack-reused 25\u001b[K\n",
            "Receiving objects: 100% (39/39), 177.08 MiB | 27.07 MiB/s, done.\n",
            "Resolving deltas: 100% (15/15), done.\n",
            "Updating files: 100% (13/13), done.\n",
            "PubMed_200k_RCT\t\t\t\t       PubMed_20k_RCT_numbers_replaced_with_at_sign\n",
            "PubMed_200k_RCT_numbers_replaced_with_at_sign  README.md\n",
            "PubMed_20k_RCT\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/Franck-Dernoncourt/pubmed-rct.git\n",
        "!ls pubmed-rct"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_dir='content/pubmed-rct/PubMed_20k_RCT_numbers_replaced_with_at_sign/'"
      ],
      "metadata": {
        "id": "flLTEmr1fAhr"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# prompt: function to read lines of a document\n",
        "\n",
        "def get_lines(filename):\n",
        "  with open(filename, 'r') as f:\n",
        "    return f.readlines()\n"
      ],
      "metadata": {
        "id": "WvcS42KIfGfh"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train=get_lines('/content/pubmed-rct/PubMed_20k_RCT_numbers_replaced_with_at_sign/train.txt')"
      ],
      "metadata": {
        "id": "XwYq6h3pfD29"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#preprocess the data as dictionary with labels and features\n",
        "def preprocess(filename):\n",
        "  input=get_lines(filename)\n",
        "  abstract=''\n",
        "  abstract_samples=[]\n",
        "  for i in input:\n",
        "    if i.startswith(\"###\"):\n",
        "      astract_id=i\n",
        "      abstract=''\n",
        "    elif i.isspace():\n",
        "      abstract_line_split=abstract.splitlines()\n",
        "      for line_no,line in enumerate(abstract_line_split):\n",
        "        line_data={}\n",
        "        target=line.split(\"\\t\")\n",
        "        line_data['target']=target[0]\n",
        "        line_data['text']=target[1]\n",
        "        line_data['line_no']=line_no\n",
        "        line_data[\"total_lines\"]=len(abstract_line_split)-1\n",
        "        abstract_samples.append(line_data)\n",
        "    else:\n",
        "      abstract+=i\n",
        "\n",
        "  return abstract_samples\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "1pANlfRPfNNQ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_samples=preprocess('/content/pubmed-rct/PubMed_20k_RCT_numbers_replaced_with_at_sign/train.txt')"
      ],
      "metadata": {
        "id": "SIxpz2CRfTd7"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_samples=preprocess('/content/pubmed-rct/PubMed_20k_RCT_numbers_replaced_with_at_sign/test.txt')"
      ],
      "metadata": {
        "id": "U6F62uz3fViy"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "train_df=pd.DataFrame(train_samples)\n",
        "test_df=pd.DataFrame(test_samples)\n"
      ],
      "metadata": {
        "id": "xqL1y7lMfYxK"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_sentences=train_df['text'].tolist()\n",
        "test_sentences=test_df['text'].tolist()\n"
      ],
      "metadata": {
        "id": "Hi91g9Ibfc8W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization"
      ],
      "metadata": {
        "id": "mWO3SowgoFtP"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#finding the output sequence length for the vectorisation layer\n",
        "import matplotlib.pyplot as plt\n",
        "s=[len(sentence.split()) for sentence in train_sentences]\n",
        "plt.hist(s,bins=20)\n",
        "op_len = int(np.percentile(s,95))\n",
        "op_len"
      ],
      "metadata": {
        "id": "K0xWh2ysoaAG",
        "outputId": "f2acff5d-252a-452d-dbdd-93f85f125059",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        }
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "55"
            ]
          },
          "metadata": {},
          "execution_count": 20
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2U0lEQVR4nO3df1BU973/8RegC6jZJf6AlSsqqVal/qqouPl1m8p1NaQTK+mo8SZUSbxa9EZJVEgNGm9arLlp1PrrpukEZxob9U61ESqGYsWbuEHF0KgJNEmxmOqCiYFVoqBwvn/0y6lbMXFVRI7Px8yZkfN5n8/5nM8s2VeO53wMMgzDEAAAgMUEt/UAAAAAWgMhBwAAWBIhBwAAWBIhBwAAWBIhBwAAWBIhBwAAWBIhBwAAWBIhBwAAWFKHth5AW2pqatKJEyd0xx13KCgoqK2HAwAAroJhGDpz5oyio6MVHHzl+zW3dcg5ceKEYmJi2noYAADgGhw/fly9evW6YvttHXLuuOMOSX+fJLvd3sajAQAAV8Pn8ykmJsb8Hr+S2zrkNP8Vld1uJ+QAANDOfN2jJjx4DAAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALImQAwAALKlDWw8Ageubkdcq/R5bntQq/QIA0Ba4kwMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACwpoJDT2Nio5557TrGxsQoPD9c3vvEN/dd//ZcMwzBrDMNQVlaWevbsqfDwcCUmJuqjjz7y6+f06dOaNm2a7Ha7IiIilJqaqrNnz/rVvP/++7rvvvsUFhammJgYrVix4rLxbN26VQMHDlRYWJiGDBmi3//+94FcDgAAsLCAQs7PfvYzrV+/XmvWrNGHH36on/3sZ1qxYoV+8YtfmDUrVqzQ6tWrtWHDBhUXF6tz585yu906f/68WTNt2jQdPXpUBQUFys3N1d69ezVz5kyz3efzady4cerTp49KSkr04osvaunSpXrllVfMmn379mnq1KlKTU3Ve++9p4kTJ2rixIk6cuTI9cwHAACwiCDj0tswX+Ohhx5SVFSUfvWrX5n7kpOTFR4erl//+tcyDEPR0dF6+umn9cwzz0iSamtrFRUVpZycHE2ZMkUffvih4uLidODAAY0cOVKSlJ+frwcffFCffvqpoqOjtX79ev34xz+W1+uVzWaTJGVkZGj79u0qKyuTJE2ePFl1dXXKzc01xzJmzBgNHz5cGzZsuKrr8fl8cjgcqq2tld1uv9ppaHP8K+QAgNvZ1X5/B3Qn5+6771ZhYaH+/Oc/S5L+9Kc/6e2339aECRMkSRUVFfJ6vUpMTDSPcTgcSkhIkMfjkSR5PB5FRESYAUeSEhMTFRwcrOLiYrPm/vvvNwOOJLndbpWXl+uLL74way49T3NN83laUl9fL5/P57cBAABr6hBIcUZGhnw+nwYOHKiQkBA1NjbqJz/5iaZNmyZJ8nq9kqSoqCi/46Kiosw2r9eryMhI/0F06KCuXbv61cTGxl7WR3PbnXfeKa/X+5XnaUl2draef/75QC4ZAAC0UwHdydmyZYtef/11bdq0SYcOHdLGjRv13//939q4cWNrje+GyszMVG1trbkdP368rYcEAABaSUB3chYsWKCMjAxNmTJFkjRkyBD99a9/VXZ2tlJSUuR0OiVJVVVV6tmzp3lcVVWVhg8fLklyOp2qrq726/fixYs6ffq0ebzT6VRVVZVfTfPPX1fT3N6S0NBQhYaGBnLJAACgnQroTs6XX36p4GD/Q0JCQtTU1CRJio2NldPpVGFhodnu8/lUXFwsl8slSXK5XKqpqVFJSYlZs3v3bjU1NSkhIcGs2bt3ry5cuGDWFBQUaMCAAbrzzjvNmkvP01zTfB4AAHB7CyjkfO9739NPfvIT5eXl6dixY9q2bZt+/vOf6/vf/74kKSgoSPPmzdMLL7ygN998U4cPH9bjjz+u6OhoTZw4UZI0aNAgjR8/Xk8++aT279+vd955R3PmzNGUKVMUHR0tSXr00Udls9mUmpqqo0ePavPmzVq1apXS09PNsTz11FPKz8/XSy+9pLKyMi1dulQHDx7UnDlzbtDUAACA9iygv676xS9+oeeee04/+tGPVF1drejoaP3Hf/yHsrKyzJqFCxeqrq5OM2fOVE1Nje69917l5+crLCzMrHn99dc1Z84cjR07VsHBwUpOTtbq1avNdofDobfeektpaWmKj49X9+7dlZWV5beWzt13361NmzZp8eLFevbZZ9W/f39t375dgwcPvp75AAAAFhHQOjlWwzo5/lgnBwDQHrTKOjkAAADtBSEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYEiEHAABYUkAhp2/fvgoKCrpsS0tLkySdP39eaWlp6tatm7p06aLk5GRVVVX59VFZWamkpCR16tRJkZGRWrBggS5evOhXs2fPHo0YMUKhoaHq16+fcnJyLhvL2rVr1bdvX4WFhSkhIUH79+8P8NIBAICVBRRyDhw4oJMnT5pbQUGBJOkHP/iBJGn+/PnasWOHtm7dqqKiIp04cUKTJk0yj29sbFRSUpIaGhq0b98+bdy4UTk5OcrKyjJrKioqlJSUpAceeEClpaWaN2+ennjiCe3atcus2bx5s9LT07VkyRIdOnRIw4YNk9vtVnV19XVNBgAAsI4gwzCMaz143rx5ys3N1UcffSSfz6cePXpo06ZNeuSRRyRJZWVlGjRokDwej8aMGaOdO3fqoYce0okTJxQVFSVJ2rBhgxYtWqRTp07JZrNp0aJFysvL05EjR8zzTJkyRTU1NcrPz5ckJSQkaNSoUVqzZo0kqampSTExMZo7d64yMjKuevw+n08Oh0O1tbWy2+3XOg03Xd+MvFbp99jypFbpFwCAG+lqv7+v+ZmchoYG/frXv9aMGTMUFBSkkpISXbhwQYmJiWbNwIED1bt3b3k8HkmSx+PRkCFDzIAjSW63Wz6fT0ePHjVrLu2juaa5j4aGBpWUlPjVBAcHKzEx0ay5kvr6evl8Pr8NAABY0zWHnO3bt6umpkY//OEPJUler1c2m00RERF+dVFRUfJ6vWbNpQGnub257atqfD6fzp07p88++0yNjY0t1jT3cSXZ2dlyOBzmFhMTE9A1AwCA9uOaQ86vfvUrTZgwQdHR0TdyPK0qMzNTtbW15nb8+PG2HhIAAGglHa7loL/+9a/6wx/+oN/+9rfmPqfTqYaGBtXU1PjdzamqqpLT6TRr/vktqOa3ry6t+ec3sqqqqmS32xUeHq6QkBCFhIS0WNPcx5WEhoYqNDQ0sIsFAADt0jXdyXnttdcUGRmppKR/PKgaHx+vjh07qrCw0NxXXl6uyspKuVwuSZLL5dLhw4f93oIqKCiQ3W5XXFycWXNpH801zX3YbDbFx8f71TQ1NamwsNCsAQAACPhOTlNTk1577TWlpKSoQ4d/HO5wOJSamqr09HR17dpVdrtdc+fOlcvl0pgxYyRJ48aNU1xcnB577DGtWLFCXq9XixcvVlpamnmHZdasWVqzZo0WLlyoGTNmaPfu3dqyZYvy8v7xRlF6erpSUlI0cuRIjR49WitXrlRdXZ2mT59+vfMBAAAsIuCQ84c//EGVlZWaMWPGZW0vv/yygoODlZycrPr6erndbq1bt85sDwkJUW5urmbPni2Xy6XOnTsrJSVFy5YtM2tiY2OVl5en+fPna9WqVerVq5deffVVud1us2by5Mk6deqUsrKy5PV6NXz4cOXn51/2MDIAALh9Xdc6Oe0d6+T4Y50cAEB70Orr5AAAANzKCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSCDkAAMCSAg45f/vb3/Tv//7v6tatm8LDwzVkyBAdPHjQbDcMQ1lZWerZs6fCw8OVmJiojz76yK+P06dPa9q0abLb7YqIiFBqaqrOnj3rV/P+++/rvvvuU1hYmGJiYrRixYrLxrJ161YNHDhQYWFhGjJkiH7/+98HejkAAMCiAgo5X3zxhe655x517NhRO3fu1AcffKCXXnpJd955p1mzYsUKrV69Whs2bFBxcbE6d+4st9ut8+fPmzXTpk3T0aNHVVBQoNzcXO3du1czZ840230+n8aNG6c+ffqopKREL774opYuXapXXnnFrNm3b5+mTp2q1NRUvffee5o4caImTpyoI0eOXM98AAAAiwgyDMO42uKMjAy98847+r//+78W2w3DUHR0tJ5++mk988wzkqTa2lpFRUUpJydHU6ZM0Ycffqi4uDgdOHBAI0eOlCTl5+frwQcf1Keffqro6GitX79eP/7xj+X1emWz2cxzb9++XWVlZZKkyZMnq66uTrm5ueb5x4wZo+HDh2vDhg1XdT0+n08Oh0O1tbWy2+1XOw1trm9GXqv0e2x5Uqv0CwDAjXS1398B3cl58803NXLkSP3gBz9QZGSkvv3tb+uXv/yl2V5RUSGv16vExERzn8PhUEJCgjwejyTJ4/EoIiLCDDiSlJiYqODgYBUXF5s1999/vxlwJMntdqu8vFxffPGFWXPpeZprms/Tkvr6evl8Pr8NAABYU0Ah5y9/+YvWr1+v/v37a9euXZo9e7b+8z//Uxs3bpQkeb1eSVJUVJTfcVFRUWab1+tVZGSkX3uHDh3UtWtXv5qW+rj0HFeqaW5vSXZ2thwOh7nFxMQEcvkAAKAdCSjkNDU1acSIEfrpT3+qb3/725o5c6aefPLJq/7robaWmZmp2tpaczt+/HhbDwkAALSSgEJOz549FRcX57dv0KBBqqyslCQ5nU5JUlVVlV9NVVWV2eZ0OlVdXe3XfvHiRZ0+fdqvpqU+Lj3HlWqa21sSGhoqu93utwEAAGsKKOTcc889Ki8v99v35z//WX369JEkxcbGyul0qrCw0Gz3+XwqLi6Wy+WSJLlcLtXU1KikpMSs2b17t5qampSQkGDW7N27VxcuXDBrCgoKNGDAAPNNLpfL5Xee5prm8wAAgNtbQCFn/vz5evfdd/XTn/5UH3/8sTZt2qRXXnlFaWlpkqSgoCDNmzdPL7zwgt58800dPnxYjz/+uKKjozVx4kRJf7/zM378eD355JPav3+/3nnnHc2ZM0dTpkxRdHS0JOnRRx+VzWZTamqqjh49qs2bN2vVqlVKT083x/LUU08pPz9fL730ksrKyrR06VIdPHhQc+bMuUFTAwAA2rMOgRSPGjVK27ZtU2ZmppYtW6bY2FitXLlS06ZNM2sWLlyouro6zZw5UzU1Nbr33nuVn5+vsLAws+b111/XnDlzNHbsWAUHBys5OVmrV6822x0Oh9566y2lpaUpPj5e3bt3V1ZWlt9aOnfffbc2bdqkxYsX69lnn1X//v21fft2DR48+HrmAwAAWERA6+RYDevk+GOdHABAe9Aq6+QAAAC0F4QcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSYQcAABgSQGFnKVLlyooKMhvGzhwoNl+/vx5paWlqVu3burSpYuSk5NVVVXl10dlZaWSkpLUqVMnRUZGasGCBbp48aJfzZ49ezRixAiFhoaqX79+ysnJuWwsa9euVd++fRUWFqaEhATt378/kEsBAAAWF/CdnG9961s6efKkub399ttm2/z587Vjxw5t3bpVRUVFOnHihCZNmmS2NzY2KikpSQ0NDdq3b582btyonJwcZWVlmTUVFRVKSkrSAw88oNLSUs2bN09PPPGEdu3aZdZs3rxZ6enpWrJkiQ4dOqRhw4bJ7Xarurr6WucBAABYTJBhGMbVFi9dulTbt29XaWnpZW21tbXq0aOHNm3apEceeUSSVFZWpkGDBsnj8WjMmDHauXOnHnroIZ04cUJRUVGSpA0bNmjRokU6deqUbDabFi1apLy8PB05csTse8qUKaqpqVF+fr4kKSEhQaNGjdKaNWskSU1NTYqJidHcuXOVkZFx1Rfv8/nkcDhUW1sru91+1ce1tb4Zea3S77HlSa3SLwAAN9LVfn8HfCfno48+UnR0tO666y5NmzZNlZWVkqSSkhJduHBBiYmJZu3AgQPVu3dveTweSZLH49GQIUPMgCNJbrdbPp9PR48eNWsu7aO5prmPhoYGlZSU+NUEBwcrMTHRrLmS+vp6+Xw+vw0AAFhTQCEnISFBOTk5ys/P1/r161VRUaH77rtPZ86ckdfrlc1mU0REhN8xUVFR8nq9kiSv1+sXcJrbm9u+qsbn8+ncuXP67LPP1NjY2GJNcx9Xkp2dLYfDYW4xMTGBXD4AAGhHOgRSPGHCBPPPQ4cOVUJCgvr06aMtW7YoPDz8hg/uRsvMzFR6err5s8/nI+gAAGBR1/UKeUREhL75zW/q448/ltPpVENDg2pqavxqqqqq5HQ6JUlOp/Oyt62af/66GrvdrvDwcHXv3l0hISEt1jT3cSWhoaGy2+1+GwAAsKbrCjlnz57VJ598op49eyo+Pl4dO3ZUYWGh2V5eXq7Kykq5XC5Jksvl0uHDh/3egiooKJDdbldcXJxZc2kfzTXNfdhsNsXHx/vVNDU1qbCw0KwBAAAIKOQ888wzKioq0rFjx7Rv3z59//vfV0hIiKZOnSqHw6HU1FSlp6frj3/8o0pKSjR9+nS5XC6NGTNGkjRu3DjFxcXpscce05/+9Cft2rVLixcvVlpamkJDQyVJs2bN0l/+8hctXLhQZWVlWrdunbZs2aL58+eb40hPT9cvf/lLbdy4UR9++KFmz56turo6TZ8+/QZODQAAaM8Ceibn008/1dSpU/X555+rR48euvfee/Xuu++qR48ekqSXX35ZwcHBSk5OVn19vdxut9atW2ceHxISotzcXM2ePVsul0udO3dWSkqKli1bZtbExsYqLy9P8+fP16pVq9SrVy+9+uqrcrvdZs3kyZN16tQpZWVlyev1avjw4crPz7/sYWQAAHD7CmidHKthnRx/rJMDAGgPWm2dHAAAgPaAkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACyJkAMAACypQ1sPwKr6ZuS19RAAALitcScHAABYEiEHAABYEiEHAABYEiEHAABY0nWFnOXLlysoKEjz5s0z950/f15paWnq1q2bunTpouTkZFVVVfkdV1lZqaSkJHXq1EmRkZFasGCBLl686FezZ88ejRgxQqGhoerXr59ycnIuO//atWvVt29fhYWFKSEhQfv377+eywEAABZyzSHnwIED+p//+R8NHTrUb//8+fO1Y8cObd26VUVFRTpx4oQmTZpktjc2NiopKUkNDQ3at2+fNm7cqJycHGVlZZk1FRUVSkpK0gMPPKDS0lLNmzdPTzzxhHbt2mXWbN68Wenp6VqyZIkOHTqkYcOGye12q7q6+lovCQAAWEiQYRhGoAedPXtWI0aM0Lp16/TCCy9o+PDhWrlypWpra9WjRw9t2rRJjzzyiCSprKxMgwYNksfj0ZgxY7Rz50499NBDOnHihKKioiRJGzZs0KJFi3Tq1CnZbDYtWrRIeXl5OnLkiHnOKVOmqKamRvn5+ZKkhIQEjRo1SmvWrJEkNTU1KSYmRnPnzlVGRsZVXYfP55PD4VBtba3sdnug0/CV2uMr5MeWJ7X1EAAA+FpX+/19TXdy0tLSlJSUpMTERL/9JSUlunDhgt/+gQMHqnfv3vJ4PJIkj8ejIUOGmAFHktxut3w+n44ePWrW/HPfbrfb7KOhoUElJSV+NcHBwUpMTDRrWlJfXy+fz+e3AQAAawp4McA33nhDhw4d0oEDBy5r83q9stlsioiI8NsfFRUlr9dr1lwacJrbm9u+qsbn8+ncuXP64osv1NjY2GJNWVnZFceenZ2t559//uouFAAAtGsB3ck5fvy4nnrqKb3++usKCwtrrTG1mszMTNXW1prb8ePH23pIAACglQQUckpKSlRdXa0RI0aoQ4cO6tChg4qKirR69Wp16NBBUVFRamhoUE1Njd9xVVVVcjqdkiSn03nZ21bNP39djd1uV3h4uLp3766QkJAWa5r7aEloaKjsdrvfBgAArCmgkDN27FgdPnxYpaWl5jZy5EhNmzbN/HPHjh1VWFhoHlNeXq7Kykq5XC5Jksvl0uHDh/3egiooKJDdbldcXJxZc2kfzTXNfdhsNsXHx/vVNDU1qbCw0KwBAAC3t4Ceybnjjjs0ePBgv32dO3dWt27dzP2pqalKT09X165dZbfbNXfuXLlcLo0ZM0aSNG7cOMXFxemxxx7TihUr5PV6tXjxYqWlpSk0NFSSNGvWLK1Zs0YLFy7UjBkztHv3bm3ZskV5ef94Yyk9PV0pKSkaOXKkRo8erZUrV6qurk7Tp0+/rgkBAADWcMP/FfKXX35ZwcHBSk5OVn19vdxut9atW2e2h4SEKDc3V7Nnz5bL5VLnzp2VkpKiZcuWmTWxsbHKy8vT/PnztWrVKvXq1Uuvvvqq3G63WTN58mSdOnVKWVlZ8nq9Gj58uPLz8y97GBkAANyermmdHKtgnRx/rJMDAGgPWnWdHAAAgFsdIQcAAFgSIQcAAFgSIQcAAFgSIQcAAFgSIQcAAFjSDV8nB+1Xa772zuvpAICbjTs5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgg5AADAkgIKOevXr9fQoUNlt9tlt9vlcrm0c+dOs/38+fNKS0tTt27d1KVLFyUnJ6uqqsqvj8rKSiUlJalTp06KjIzUggULdPHiRb+aPXv2aMSIEQoNDVW/fv2Uk5Nz2VjWrl2rvn37KiwsTAkJCdq/f38glwIAACwuoJDTq1cvLV++XCUlJTp48KC++93v6uGHH9bRo0clSfPnz9eOHTu0detWFRUV6cSJE5o0aZJ5fGNjo5KSktTQ0KB9+/Zp48aNysnJUVZWlllTUVGhpKQkPfDAAyotLdW8efP0xBNPaNeuXWbN5s2blZ6eriVLlujQoUMaNmyY3G63qqurr3c+AACARQQZhmFcTwddu3bViy++qEceeUQ9evTQpk2b9Mgjj0iSysrKNGjQIHk8Ho0ZM0Y7d+7UQw89pBMnTigqKkqStGHDBi1atEinTp2SzWbTokWLlJeXpyNHjpjnmDJlimpqapSfny9JSkhI0KhRo7RmzRpJUlNTk2JiYjR37lxlZGRc9dh9Pp8cDodqa2tlt9uvZxou0zcj74b2194dW57U1kMAAFjE1X5/X/MzOY2NjXrjjTdUV1cnl8ulkpISXbhwQYmJiWbNwIED1bt3b3k8HkmSx+PRkCFDzIAjSW63Wz6fz7wb5PF4/Ppormnuo6GhQSUlJX41wcHBSkxMNGuupL6+Xj6fz28DAADWFHDIOXz4sLp06aLQ0FDNmjVL27ZtU1xcnLxer2w2myIiIvzqo6Ki5PV6JUler9cv4DS3N7d9VY3P59O5c+f02WefqbGxscWa5j6uJDs7Ww6Hw9xiYmICvXwAANBOBBxyBgwYoNLSUhUXF2v27NlKSUnRBx980Bpju+EyMzNVW1trbsePH2/rIQEAgFbSIdADbDab+vXrJ0mKj4/XgQMHtGrVKk2ePFkNDQ2qqanxu5tTVVUlp9MpSXI6nZe9BdX89tWlNf/8RlZVVZXsdrvCw8MVEhKikJCQFmua+7iS0NBQhYaGBnrJAACgHbrudXKamppUX1+v+Ph4dezYUYWFhWZbeXm5Kisr5XK5JEkul0uHDx/2ewuqoKBAdrtdcXFxZs2lfTTXNPdhs9kUHx/vV9PU1KTCwkKzBgAAIKA7OZmZmZowYYJ69+6tM2fOaNOmTdqzZ4927dolh8Oh1NRUpaenq2vXrrLb7Zo7d65cLpfGjBkjSRo3bpzi4uL02GOPacWKFfJ6vVq8eLHS0tLMOyyzZs3SmjVrtHDhQs2YMUO7d+/Wli1blJf3j7eV0tPTlZKSopEjR2r06NFauXKl6urqNH369Bs4NQAAoD0LKORUV1fr8ccf18mTJ+VwODR06FDt2rVL//Zv/yZJevnllxUcHKzk5GTV19fL7XZr3bp15vEhISHKzc3V7Nmz5XK51LlzZ6WkpGjZsmVmTWxsrPLy8jR//nytWrVKvXr10quvviq3223WTJ48WadOnVJWVpa8Xq+GDx+u/Pz8yx5GBgAAt6/rXienPWOdnJuHdXIAADdKq6+TAwAAcCsj5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsi5AAAAEsKKORkZ2dr1KhRuuOOOxQZGamJEyeqvLzcr+b8+fNKS0tTt27d1KVLFyUnJ6uqqsqvprKyUklJSerUqZMiIyO1YMECXbx40a9mz549GjFihEJDQ9WvXz/l5ORcNp61a9eqb9++CgsLU0JCgvbv3x/I5QAAAAsLKOQUFRUpLS1N7777rgoKCnThwgWNGzdOdXV1Zs38+fO1Y8cObd26VUVFRTpx4oQmTZpktjc2NiopKUkNDQ3at2+fNm7cqJycHGVlZZk1FRUVSkpK0gMPPKDS0lLNmzdPTzzxhHbt2mXWbN68Wenp6VqyZIkOHTqkYcOGye12q7q6+nrmAwAAWESQYRjGtR586tQpRUZGqqioSPfff79qa2vVo0cPbdq0SY888ogkqaysTIMGDZLH49GYMWO0c+dOPfTQQzpx4oSioqIkSRs2bNCiRYt06tQp2Ww2LVq0SHl5eTpy5Ih5rilTpqimpkb5+fmSpISEBI0aNUpr1qyRJDU1NSkmJkZz585VRkbGVY3f5/PJ4XCotrZWdrv9WqehRX0z8m5of+3dseVJbT0EAIBFXO3393U9k1NbWytJ6tq1qySppKREFy5cUGJiolkzcOBA9e7dWx6PR5Lk8Xg0ZMgQM+BIktvtls/n09GjR82aS/tormnuo6GhQSUlJX41wcHBSkxMNGtaUl9fL5/P57cBAABruuaQ09TUpHnz5umee+7R4MGDJUler1c2m00RERF+tVFRUfJ6vWbNpQGnub257atqfD6fzp07p88++0yNjY0t1jT30ZLs7Gw5HA5zi4mJCfzCAQBAu3DNISctLU1HjhzRG2+8cSPH06oyMzNVW1trbsePH2/rIQEAgFbS4VoOmjNnjnJzc7V371716tXL3O90OtXQ0KCamhq/uzlVVVVyOp1mzT+/BdX89tWlNf/8RlZVVZXsdrvCw8MVEhKikJCQFmua+2hJaGioQkNDA79gAADQ7gR0J8cwDM2ZM0fbtm3T7t27FRsb69ceHx+vjh07qrCw0NxXXl6uyspKuVwuSZLL5dLhw4f93oIqKCiQ3W5XXFycWXNpH801zX3YbDbFx8f71TQ1NamwsNCsAQAAt7eA7uSkpaVp06ZN+t3vfqc77rjDfP7F4XAoPDxcDodDqampSk9PV9euXWW32zV37ly5XC6NGTNGkjRu3DjFxcXpscce04oVK+T1erV48WKlpaWZd1lmzZqlNWvWaOHChZoxY4Z2796tLVu2KC/vH28spaenKyUlRSNHjtTo0aO1cuVK1dXVafr06TdqbgAAQDsWUMhZv369JOk73/mO3/7XXntNP/zhDyVJL7/8soKDg5WcnKz6+nq53W6tW7fOrA0JCVFubq5mz54tl8ulzp07KyUlRcuWLTNrYmNjlZeXp/nz52vVqlXq1auXXn31VbndbrNm8uTJOnXqlLKysuT1ejV8+HDl5+df9jAyAAC4PV3XOjntHevk3DyskwMAuFFuyjo5AAAAtypCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsCRCDgAAsKQOgR6wd+9evfjiiyopKdHJkye1bds2TZw40Ww3DENLlizRL3/5S9XU1Oiee+7R+vXr1b9/f7Pm9OnTmjt3rnbs2KHg4GAlJydr1apV6tKli1nz/vvvKy0tTQcOHFCPHj00d+5cLVy40G8sW7du1XPPPadjx46pf//++tnPfqYHH3zwGqYBra1vRl6r9HtseVKr9AsAaP8CvpNTV1enYcOGae3atS22r1ixQqtXr9aGDRtUXFyszp07y+126/z582bNtGnTdPToURUUFCg3N1d79+7VzJkzzXafz6dx48apT58+Kikp0YsvvqilS5fqlVdeMWv27dunqVOnKjU1Ve+9954mTpyoiRMn6siRI4FeEgAAsKAgwzCMaz44KMjvTo5hGIqOjtbTTz+tZ555RpJUW1urqKgo5eTkaMqUKfrwww8VFxenAwcOaOTIkZKk/Px8Pfjgg/r0008VHR2t9evX68c//rG8Xq9sNpskKSMjQ9u3b1dZWZkkafLkyaqrq1Nubq45njFjxmj48OHasGHDVY3f5/PJ4XCotrZWdrv9WqehRa115wL+uJMDALefq/3+vqHP5FRUVMjr9SoxMdHc53A4lJCQII/HI0nyeDyKiIgwA44kJSYmKjg4WMXFxWbN/fffbwYcSXK73SovL9cXX3xh1lx6nuaa5vO0pL6+Xj6fz28DAADWdENDjtfrlSRFRUX57Y+KijLbvF6vIiMj/do7dOigrl27+tW01Mel57hSTXN7S7Kzs+VwOMwtJiYm0EsEAADtxG31dlVmZqZqa2vN7fjx4209JAAA0EpuaMhxOp2SpKqqKr/9VVVVZpvT6VR1dbVf+8WLF3X69Gm/mpb6uPQcV6ppbm9JaGio7Ha73wYAAKzphoac2NhYOZ1OFRYWmvt8Pp+Ki4vlcrkkSS6XSzU1NSopKTFrdu/eraamJiUkJJg1e/fu1YULF8yagoICDRgwQHfeeadZc+l5mmuazwMAAG5vAYecs2fPqrS0VKWlpZL+/rBxaWmpKisrFRQUpHnz5umFF17Qm2++qcOHD+vxxx9XdHS0+QbWoEGDNH78eD355JPav3+/3nnnHc2ZM0dTpkxRdHS0JOnRRx+VzWZTamqqjh49qs2bN2vVqlVKT083x/HUU08pPz9fL730ksrKyrR06VIdPHhQc+bMuf5ZAQAA7V7AiwEePHhQDzzwgPlzc/BISUlRTk6OFi5cqLq6Os2cOVM1NTW69957lZ+fr7CwMPOY119/XXPmzNHYsWPNxQBXr15ttjscDr311ltKS0tTfHy8unfvrqysLL+1dO6++25t2rRJixcv1rPPPqv+/ftr+/btGjx48DVNBAAAsJbrWienvWOdnPaPdXIA4PbTJuvkAAAA3CoIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJIIOQAAwJI6tPUAgOvRNyOv1fo+tjyp1foGALQ+7uQAAABLIuQAAABLIuQAAABLIuQAAABLIuQAAABLIuQAAABLIuQAAABLIuQAAABLIuQAAABLYsVj4ApaazVlVlIGgJuDOzkAAMCS2n3IWbt2rfr27auwsDAlJCRo//79bT0kAABwC2jXIWfz5s1KT0/XkiVLdOjQIQ0bNkxut1vV1dVtPTQAANDGggzDMNp6ENcqISFBo0aN0po1ayRJTU1NiomJ0dy5c5WRkfG1x/t8PjkcDtXW1sput9/QsbXmv44NXAnP+wC4HVzt93e7ffC4oaFBJSUlyszMNPcFBwcrMTFRHo+nxWPq6+tVX19v/lxbWyvp75N1ozXVf3nD+wS+Tu/5W1ut7yPPu1utbwAIRPP39tfdp2m3Ieezzz5TY2OjoqKi/PZHRUWprKysxWOys7P1/PPPX7Y/JiamVcYIWIljZVuPAAD8nTlzRg6H44rt7TbkXIvMzEylp6ebPzc1Nen06dPq1q2bgoKCbsg5fD6fYmJidPz48Rv+V2BWxHwFhvkKHHMWGOYrcMxZYG7EfBmGoTNnzig6Ovor69ptyOnevbtCQkJUVVXlt7+qqkpOp7PFY0JDQxUaGuq3LyIiolXGZ7fb+bAHgPkKDPMVOOYsMMxX4JizwFzvfH3VHZxm7fbtKpvNpvj4eBUWFpr7mpqaVFhYKJfL1YYjAwAAt4J2eydHktLT05WSkqKRI0dq9OjRWrlyperq6jR9+vS2HhoAAGhj7TrkTJ48WadOnVJWVpa8Xq+GDx+u/Pz8yx5GvplCQ0O1ZMmSy/5aDC1jvgLDfAWOOQsM8xU45iwwN3O+2vU6OQAAAFfSbp/JAQAA+CqEHAAAYEmEHAAAYEmEHAAAYEmEnBto7dq16tu3r8LCwpSQkKD9+/e39ZBuCUuXLlVQUJDfNnDgQLP9/PnzSktLU7du3dSlSxclJydftsij1e3du1ff+973FB0draCgIG3fvt2v3TAMZWVlqWfPngoPD1diYqI++ugjv5rTp09r2rRpstvtioiIUGpqqs6ePXsTr+Lm+br5+uEPf3jZZ278+PF+NbfTfGVnZ2vUqFG64447FBkZqYkTJ6q8vNyv5mp+DysrK5WUlKROnTopMjJSCxYs0MWLF2/mpdw0VzNn3/nOdy77nM2aNcuv5naZs/Xr12vo0KHmAn8ul0s7d+4029vq80XIuUE2b96s9PR0LVmyRIcOHdKwYcPkdrtVXV3d1kO7JXzrW9/SyZMnze3tt9822+bPn68dO3Zo69atKioq0okTJzRp0qQ2HO3NV1dXp2HDhmnt2rUttq9YsUKrV6/Whg0bVFxcrM6dO8vtduv8+fNmzbRp03T06FEVFBQoNzdXe/fu1cyZM2/WJdxUXzdfkjR+/Hi/z9xvfvMbv/bbab6KioqUlpamd999VwUFBbpw4YLGjRunuro6s+brfg8bGxuVlJSkhoYG7du3Txs3blROTo6ysrLa4pJa3dXMmSQ9+eSTfp+zFStWmG2305z16tVLy5cvV0lJiQ4ePKjvfve7evjhh3X06FFJbfj5MnBDjB492khLSzN/bmxsNKKjo43s7Ow2HNWtYcmSJcawYcNabKupqTE6duxobN261dz34YcfGpIMj8dzk0Z4a5FkbNu2zfy5qanJcDqdxosvvmjuq6mpMUJDQ43f/OY3hmEYxgcffGBIMg4cOGDW7Ny50wgKCjL+9re/3bSxt4V/ni/DMIyUlBTj4YcfvuIxt/N8GYZhVFdXG5KMoqIiwzCu7vfw97//vREcHGx4vV6zZv369Ybdbjfq6+tv7gW0gX+eM8MwjH/91381nnrqqSsec7vP2Z133mm8+uqrbfr54k7ODdDQ0KCSkhIlJiaa+4KDg5WYmCiPx9OGI7t1fPTRR4qOjtZdd92ladOmqbKyUpJUUlKiCxcu+M3dwIED1bt3b+bu/6uoqJDX6/WbI4fDoYSEBHOOPB6PIiIiNHLkSLMmMTFRwcHBKi4uvuljvhXs2bNHkZGRGjBggGbPnq3PP//cbLvd56u2tlaS1LVrV0lX93vo8Xg0ZMgQv8VW3W63fD6f+X/rVvbPc9bs9ddfV/fu3TV48GBlZmbqyy+/NNtu1zlrbGzUG2+8obq6Orlcrjb9fLXrFY9vFZ999pkaGxsvW2k5KipKZWVlbTSqW0dCQoJycnI0YMAAnTx5Us8//7zuu+8+HTlyRF6vVzab7bJ/KDUqKkper7dtBnyLaZ6Hlj5fzW1er1eRkZF+7R06dFDXrl1vy3kcP368Jk2apNjYWH3yySd69tlnNWHCBHk8HoWEhNzW89XU1KR58+bpnnvu0eDBgyXpqn4PvV5vi5/B5jYra2nOJOnRRx9Vnz59FB0drffff1+LFi1SeXm5fvvb30q6/ebs8OHDcrlcOn/+vLp06aJt27YpLi5OpaWlbfb5IuSg1U2YMMH889ChQ5WQkKA+ffpoy5YtCg8Pb8ORwaqmTJli/nnIkCEaOnSovvGNb2jPnj0aO3ZsG46s7aWlpenIkSN+z8Xhq11pzi59hmvIkCHq2bOnxo4dq08++UTf+MY3bvYw29yAAQNUWlqq2tpa/e///q9SUlJUVFTUpmPir6tugO7duyskJOSyJ8WrqqrkdDrbaFS3roiICH3zm9/Uxx9/LKfTqYaGBtXU1PjVMHf/0DwPX/X5cjqdlz3kfvHiRZ0+fZp5lHTXXXepe/fu+vjjjyXdvvM1Z84c5ebm6o9//KN69epl7r+a30On09niZ7C5zaquNGctSUhIkCS/z9ntNGc2m039+vVTfHy8srOzNWzYMK1atapNP1+EnBvAZrMpPj5ehYWF5r6mpiYVFhbK5XK14chuTWfPntUnn3yinj17Kj4+Xh07dvSbu/LyclVWVjJ3/19sbKycTqffHPl8PhUXF5tz5HK5VFNTo5KSErNm9+7dampqMv/Dezv79NNP9fnnn6tnz56Sbr/5MgxDc+bM0bZt27R7927Fxsb6tV/N76HL5dLhw4f9wmFBQYHsdrvi4uJuzoXcRF83Zy0pLS2VJL/P2e00Z/+sqalJ9fX1bfv5uuZHluHnjTfeMEJDQ42cnBzjgw8+MGbOnGlERET4PSl+u3r66aeNPXv2GBUVFcY777xjJCYmGt27dzeqq6sNwzCMWbNmGb179zZ2795tHDx40HC5XIbL5WrjUd9cZ86cMd577z3jvffeMyQZP//5z4333nvP+Otf/2oYhmEsX77ciIiIMH73u98Z77//vvHwww8bsbGxxrlz58w+xo8fb3z72982iouLjbffftvo37+/MXXq1La6pFb1VfN15swZ45lnnjE8Ho9RUVFh/OEPfzBGjBhh9O/f3zh//rzZx+00X7NnzzYcDoexZ88e4+TJk+b25ZdfmjVf93t48eJFY/Dgwca4ceOM0tJSIz8/3+jRo4eRmZnZFpfU6r5uzj7++GNj2bJlxsGDB42Kigrjd7/7nXHXXXcZ999/v9nH7TRnGRkZRlFRkVFRUWG8//77RkZGhhEUFGS89dZbhmG03eeLkHMD/eIXvzB69+5t2Gw2Y/To0ca7777b1kO6JUyePNno2bOnYbPZjH/5l38xJk+ebHz88cdm+7lz54wf/ehHxp133ml06tTJ+P73v2+cPHmyDUd88/3xj380JF22paSkGIbx99fIn3vuOSMqKsoIDQ01xo4da5SXl/v18fnnnxtTp041unTpYtjtdmP69OnGmTNn2uBqWt9XzdeXX35pjBs3zujRo4fRsWNHo0+fPsaTTz552f9w3E7z1dJcSTJee+01s+Zqfg+PHTtmTJgwwQgPDze6d+9uPP3008aFCxdu8tXcHF83Z5WVlcb9999vdO3a1QgNDTX69etnLFiwwKitrfXr53aZsxkzZhh9+vQxbDab0aNHD2Ps2LFmwDGMtvt8BRmGYVz7fSAAAIBbE8/kAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAASyLkAAAAS/p/rJ383OqkIf0AAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_vectorizer = TextVectorization( max_tokens = 68000, output_sequence_length=op_len)"
      ],
      "metadata": {
        "id": "_QiDNRCjoYBb"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_vectorizer.adapt(train_sentences)"
      ],
      "metadata": {
        "id": "WVvsubd2ol1V"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_20 = text_vectorizer.get_vocabulary()\n",
        "print(f\"no of words {len(vocab_20)}\")\n",
        "print(f\"common words {vocab_20[:5]}\")\n",
        "print(f\"least common words {vocab_20[-5:]}\")\n"
      ],
      "metadata": {
        "id": "miJt2lnUpLwQ",
        "outputId": "142f2308-fe68-4cb7-d355-4d7116c48c0f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "no of words 64841\n",
            "common words ['', '[UNK]', 'the', 'and', 'of']\n",
            "least common words ['aainduced', 'aaigroup', 'aachener', 'aachen', 'aaacp']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "token_embed = layers.Embedding(input_dim=len(vocab_20),\n",
        "                                             output_dim=128,\n",
        "                                             mask_zero= True,\n",
        "                                             name='token_embedding')"
      ],
      "metadata": {
        "id": "k6pD8uQqo5lQ"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#creating labels\n",
        "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
        "one_hot=OneHotEncoder(sparse=False)\n",
        "train_label_hot=one_hot.fit_transform(tf.expand_dims(train_df['target'],axis=-1))\n",
        "test_label_hot=one_hot.fit_transform(tf.expand_dims(test_df['target'],axis=-1))\n",
        "\n"
      ],
      "metadata": {
        "id": "6bingpCIpQtf",
        "outputId": "8af6b719-827e-49f8-b9bb-7763a8f832ae",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#creating datasets\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((train_sentences,train_label_hot))\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((test_sentences,test_label_hot))\n",
        "train_dataset = train_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
        "test_dataset = test_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n"
      ],
      "metadata": {
        "id": "t9tyunF1o9cr"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def split_chars(text):\n",
        "  return ' '.join(list(text))"
      ],
      "metadata": {
        "id": "1CLoopb8pcG1"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_chars=[split_chars(i) for i in train_sentences]\n",
        "test_chars=[split_chars(i) for i in test_sentences]\n",
        "\n"
      ],
      "metadata": {
        "id": "VwLSyewIqBIc"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#character level tokenisation\n",
        "char_lens = [len(i) for i in train_sentences]\n",
        "plt.hist(char_lens,bins=25)\n",
        "perc_len=int(np.percentile(char_lens,95))\n",
        "perc_len"
      ],
      "metadata": {
        "id": "os0QFumXqEmx",
        "outputId": "ccee9af4-d283-4cb8-f230-b6f11e8cd65c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        }
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "290"
            ]
          },
          "metadata": {},
          "execution_count": 32
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArCUlEQVR4nO3de3TU5Z3H8U8uzCRcZiKXJEYSwy6uEEGQAGG87bpmmdrY1oq7wLKYItYDG6ghFgKrDeq2GxbPruBya+tZwzkr5XKO0kokbDYI1GUECUYJCrUrGmycBIvJAIUkZJ79oye/MnKRQEjMk/frnDmH/J7vPPM835NkPufH/H6JMsYYAQAAWCa6qxcAAABwLRByAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWiu3qBXSlcDis2tpa9evXT1FRUV29HAAAcBmMMTpx4oRSUlIUHX3x8zU9OuTU1tYqNTW1q5cBAACuwNGjRzV48OCLjvfokNOvXz9Jf2ySx+Pp4tUAAIDLEQqFlJqa6ryPX0yPDjlt/0Xl8XgIOQAAdDNf9VETPngMAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYKXYrl4AOkf6wtIOmefjJTkdMg8AANcaZ3IAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsFK7Qs7TTz+tqKioiMewYcOc8TNnzigvL08DBgxQ3759NWnSJNXV1UXMUVNTo5ycHPXu3VuJiYmaP3++zp49G1GzY8cOjRkzRm63W0OHDlVJScl5a1m5cqXS09MVFxenrKws7d27tz1bAQAAlmv3mZxbbrlFn332mfN48803nbF58+bptdde06ZNm7Rz507V1tbqwQcfdMZbW1uVk5Oj5uZm7d69W2vXrlVJSYmKioqcmiNHjignJ0f33HOPqqqqlJ+fr0cffVTbtm1zajZs2KCCggItXrxY+/fv16hRo+T3+1VfX3+lfQAAAJaJMsaYyy1++umntXnzZlVVVZ031tjYqEGDBmndunV66KGHJEmHDh3S8OHDFQgENGHCBG3dulX333+/amtrlZSUJElas2aNCgsLdezYMblcLhUWFqq0tFTV1dXO3FOmTFFDQ4PKysokSVlZWRo3bpxWrFghSQqHw0pNTdXcuXO1cOHCy958KBSS1+tVY2OjPB7PZT+vO0pfWNoh83y8JKdD5gEA4Epd7vt3bHsn/vDDD5WSkqK4uDj5fD4VFxcrLS1NlZWVamlpUXZ2tlM7bNgwpaWlOSEnEAho5MiRTsCRJL/fr9mzZ+vgwYO67bbbFAgEIuZoq8nPz5ckNTc3q7KyUosWLXLGo6OjlZ2drUAgcMm1NzU1qampyfk6FAq1d/udrqPCCQAAPU27/rsqKytLJSUlKisr0+rVq3XkyBHdddddOnHihILBoFwulxISEiKek5SUpGAwKEkKBoMRAadtvG3sUjWhUEinT5/W559/rtbW1gvWtM1xMcXFxfJ6vc4jNTW1PdsHAADdSLvO5Nx3333Ov2+99VZlZWXpxhtv1MaNGxUfH9/hi+toixYtUkFBgfN1KBQi6AAAYKmruoQ8ISFBf/EXf6Hf/va3Sk5OVnNzsxoaGiJq6urqlJycLElKTk4+72qrtq+/qsbj8Sg+Pl4DBw5UTEzMBWva5rgYt9stj8cT8QAAAHa6qpBz8uRJ/d///Z+uv/56ZWZmqlevXqqoqHDGDx8+rJqaGvl8PkmSz+fTgQMHIq6CKi8vl8fjUUZGhlNz7hxtNW1zuFwuZWZmRtSEw2FVVFQ4NQAAAO0KOT/84Q+1c+dOffzxx9q9e7e++93vKiYmRlOnTpXX69XMmTNVUFCgN954Q5WVlZoxY4Z8Pp8mTJggSZo4caIyMjI0ffp0vfvuu9q2bZueeuop5eXlye12S5JmzZqljz76SAsWLNChQ4e0atUqbdy4UfPmzXPWUVBQoJ///Odau3atPvjgA82ePVunTp3SjBkzOrA1AACgO2vXZ3I+/fRTTZ06Vb///e81aNAg3XnnnXrrrbc0aNAgSdLzzz+v6OhoTZo0SU1NTfL7/Vq1apXz/JiYGG3ZskWzZ8+Wz+dTnz59lJubq2effdapGTJkiEpLSzVv3jwtX75cgwcP1osvvii/3+/UTJ48WceOHVNRUZGCwaBGjx6tsrKy8z6MDAAAeq523SfHNt3hPjlft0vIuU8OAKCrXe77N3+7CgAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAVrqqkLNkyRJFRUUpPz/fOXbmzBnl5eVpwIAB6tu3ryZNmqS6urqI59XU1CgnJ0e9e/dWYmKi5s+fr7Nnz0bU7NixQ2PGjJHb7dbQoUNVUlJy3uuvXLlS6enpiouLU1ZWlvbu3Xs12wEAABa54pDz9ttv66c//aluvfXWiOPz5s3Ta6+9pk2bNmnnzp2qra3Vgw8+6Iy3trYqJydHzc3N2r17t9auXauSkhIVFRU5NUeOHFFOTo7uueceVVVVKT8/X48++qi2bdvm1GzYsEEFBQVavHix9u/fr1GjRsnv96u+vv5KtwQAACwSZYwx7X3SyZMnNWbMGK1atUo//vGPNXr0aC1btkyNjY0aNGiQ1q1bp4ceekiSdOjQIQ0fPlyBQEATJkzQ1q1bdf/996u2tlZJSUmSpDVr1qiwsFDHjh2Ty+VSYWGhSktLVV1d7bzmlClT1NDQoLKyMklSVlaWxo0bpxUrVkiSwuGwUlNTNXfuXC1cuPCy9hEKheT1etXY2CiPx9PeNnSK9IWlXb2ECB8vyenqJQAAerjLff++ojM5eXl5ysnJUXZ2dsTxyspKtbS0RBwfNmyY0tLSFAgEJEmBQEAjR450Ao4k+f1+hUIhHTx40Kn58tx+v9+Zo7m5WZWVlRE10dHRys7OdmoupKmpSaFQKOIBAADsFNveJ6xfv1779+/X22+/fd5YMBiUy+VSQkJCxPGkpCQFg0Gn5tyA0zbeNnapmlAopNOnT+uLL75Qa2vrBWsOHTp00bUXFxfrmWeeubyNAgCAbq1dZ3KOHj2qxx9/XC+//LLi4uKu1ZqumUWLFqmxsdF5HD16tKuXBAAArpF2hZzKykrV19drzJgxio2NVWxsrHbu3KkXXnhBsbGxSkpKUnNzsxoaGiKeV1dXp+TkZElScnLyeVdbtX39VTUej0fx8fEaOHCgYmJiLljTNseFuN1ueTyeiAcAALBTu0LOvffeqwMHDqiqqsp5jB07VtOmTXP+3atXL1VUVDjPOXz4sGpqauTz+SRJPp9PBw4ciLgKqry8XB6PRxkZGU7NuXO01bTN4XK5lJmZGVETDodVUVHh1AAAgJ6tXZ/J6devn0aMGBFxrE+fPhowYIBzfObMmSooKFD//v3l8Xg0d+5c+Xw+TZgwQZI0ceJEZWRkaPr06Vq6dKmCwaCeeuop5eXlye12S5JmzZqlFStWaMGCBXrkkUe0fft2bdy4UaWlf7rSqKCgQLm5uRo7dqzGjx+vZcuW6dSpU5oxY8ZVNQQAANih3R88/irPP/+8oqOjNWnSJDU1Ncnv92vVqlXOeExMjLZs2aLZs2fL5/OpT58+ys3N1bPPPuvUDBkyRKWlpZo3b56WL1+uwYMH68UXX5Tf73dqJk+erGPHjqmoqEjBYFCjR49WWVnZeR9GBgAAPdMV3SfHFtwnp/24Tw4AoKtd0/vkAAAAfN0RcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYKXYrl4Aupf0haUdMs/HS3I6ZB4AAC6GMzkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWalfIWb16tW699VZ5PB55PB75fD5t3brVGT9z5ozy8vI0YMAA9e3bV5MmTVJdXV3EHDU1NcrJyVHv3r2VmJio+fPn6+zZsxE1O3bs0JgxY+R2uzV06FCVlJSct5aVK1cqPT1dcXFxysrK0t69e9uzFQAAYLl2hZzBgwdryZIlqqys1L59+/TXf/3X+s53vqODBw9KkubNm6fXXntNmzZt0s6dO1VbW6sHH3zQeX5ra6tycnLU3Nys3bt3a+3atSopKVFRUZFTc+TIEeXk5Oiee+5RVVWV8vPz9eijj2rbtm1OzYYNG1RQUKDFixdr//79GjVqlPx+v+rr66+2HwAAwBJRxhhzNRP0799fzz33nB566CENGjRI69at00MPPSRJOnTokIYPH65AIKAJEyZo69atuv/++1VbW6ukpCRJ0po1a1RYWKhjx47J5XKpsLBQpaWlqq6udl5jypQpamhoUFlZmSQpKytL48aN04oVKyRJ4XBYqampmjt3rhYuXHjZaw+FQvJ6vWpsbJTH47maNlwz6QtLu3oJ18THS3K6egkAgG7qct+/r/gzOa2trVq/fr1OnToln8+nyspKtbS0KDs726kZNmyY0tLSFAgEJEmBQEAjR450Ao4k+f1+hUIh52xQIBCImKOtpm2O5uZmVVZWRtRER0crOzvbqbmYpqYmhUKhiAcAALBTu0POgQMH1LdvX7ndbs2aNUuvvvqqMjIyFAwG5XK5lJCQEFGflJSkYDAoSQoGgxEBp228bexSNaFQSKdPn9bnn3+u1tbWC9a0zXExxcXF8nq9ziM1NbW92wcAAN1Eu0POzTffrKqqKu3Zs0ezZ89Wbm6u3n///Wuxtg63aNEiNTY2Oo+jR4929ZIAAMA1EtveJ7hcLg0dOlSSlJmZqbffflvLly/X5MmT1dzcrIaGhoizOXV1dUpOTpYkJScnn3cVVNvVV+fWfPmKrLq6Onk8HsXHxysmJkYxMTEXrGmb42Lcbrfcbnd7twwAALqhq75PTjgcVlNTkzIzM9WrVy9VVFQ4Y4cPH1ZNTY18Pp8kyefz6cCBAxFXQZWXl8vj8SgjI8OpOXeOtpq2OVwulzIzMyNqwuGwKioqnBoAAIB2nclZtGiR7rvvPqWlpenEiRNat26dduzYoW3btsnr9WrmzJkqKChQ//795fF4NHfuXPl8Pk2YMEGSNHHiRGVkZGj69OlaunSpgsGgnnrqKeXl5TlnWGbNmqUVK1ZowYIFeuSRR7R9+3Zt3LhRpaV/usqooKBAubm5Gjt2rMaPH69ly5bp1KlTmjFjRge2BgAAdGftCjn19fV6+OGH9dlnn8nr9erWW2/Vtm3b9Dd/8zeSpOeff17R0dGaNGmSmpqa5Pf7tWrVKuf5MTEx2rJli2bPni2fz6c+ffooNzdXzz77rFMzZMgQlZaWat68eVq+fLkGDx6sF198UX6/36mZPHmyjh07pqKiIgWDQY0ePVplZWXnfRgZAAD0XFd9n5zujPvkdB3ukwMAuFLX/D45AAAAX2eEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWKldIae4uFjjxo1Tv379lJiYqAceeECHDx+OqDlz5ozy8vI0YMAA9e3bV5MmTVJdXV1ETU1NjXJyctS7d28lJiZq/vz5Onv2bETNjh07NGbMGLndbg0dOlQlJSXnrWflypVKT09XXFycsrKytHfv3vZsBwAAWKxdIWfnzp3Ky8vTW2+9pfLycrW0tGjixIk6deqUUzNv3jy99tpr2rRpk3bu3Kna2lo9+OCDznhra6tycnLU3Nys3bt3a+3atSopKVFRUZFTc+TIEeXk5Oiee+5RVVWV8vPz9eijj2rbtm1OzYYNG1RQUKDFixdr//79GjVqlPx+v+rr66+mHwAAwBJRxhhzpU8+duyYEhMTtXPnTt19991qbGzUoEGDtG7dOj300EOSpEOHDmn48OEKBAKaMGGCtm7dqvvvv1+1tbVKSkqSJK1Zs0aFhYU6duyYXC6XCgsLVVpaqurqaue1pkyZooaGBpWVlUmSsrKyNG7cOK1YsUKSFA6HlZqaqrlz52rhwoWXtf5QKCSv16vGxkZ5PJ4rbcM1lb6wtKuXcE18vCSnq5cAAOimLvf9+6o+k9PY2ChJ6t+/vySpsrJSLS0tys7OdmqGDRumtLQ0BQIBSVIgENDIkSOdgCNJfr9foVBIBw8edGrOnaOtpm2O5uZmVVZWRtRER0crOzvbqbmQpqYmhUKhiAcAALDTFYeccDis/Px83XHHHRoxYoQkKRgMyuVyKSEhIaI2KSlJwWDQqTk34LSNt41dqiYUCun06dP6/PPP1draesGatjkupLi4WF6v13mkpqa2f+MAAKBbuOKQk5eXp+rqaq1fv74j13NNLVq0SI2Njc7j6NGjXb0kAABwjcReyZPmzJmjLVu2aNeuXRo8eLBzPDk5Wc3NzWpoaIg4m1NXV6fk5GSn5stXQbVdfXVuzZevyKqrq5PH41F8fLxiYmIUExNzwZq2OS7E7XbL7Xa3f8MAAKDbadeZHGOM5syZo1dffVXbt2/XkCFDIsYzMzPVq1cvVVRUOMcOHz6smpoa+Xw+SZLP59OBAwciroIqLy+Xx+NRRkaGU3PuHG01bXO4XC5lZmZG1ITDYVVUVDg1AACgZ2vXmZy8vDytW7dOv/zlL9WvXz/n8y9er1fx8fHyer2aOXOmCgoK1L9/f3k8Hs2dO1c+n08TJkyQJE2cOFEZGRmaPn26li5dqmAwqKeeekp5eXnOWZZZs2ZpxYoVWrBggR555BFt375dGzduVGnpn640KigoUG5ursaOHavx48dr2bJlOnXqlGbMmNFRvQEAAN1Yu0LO6tWrJUl/9Vd/FXH8pZde0ve+9z1J0vPPP6/o6GhNmjRJTU1N8vv9WrVqlVMbExOjLVu2aPbs2fL5fOrTp49yc3P17LPPOjVDhgxRaWmp5s2bp+XLl2vw4MF68cUX5ff7nZrJkyfr2LFjKioqUjAY1OjRo1VWVnbeh5EBAEDPdFX3yenuuE9O1+E+OQCAK9Up98kBAAD4uiLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJViu3oB6JnSF5Z2yDwfL8npkHkAAPbhTA4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWiu3qBdgqfWFpVy8BAIAejTM5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGCldoecXbt26Vvf+pZSUlIUFRWlzZs3R4wbY1RUVKTrr79e8fHxys7O1ocffhhRc/z4cU2bNk0ej0cJCQmaOXOmTp48GVHz3nvv6a677lJcXJxSU1O1dOnS89ayadMmDRs2THFxcRo5cqRef/319m4HAABYqt0h59SpUxo1apRWrlx5wfGlS5fqhRde0Jo1a7Rnzx716dNHfr9fZ86ccWqmTZumgwcPqry8XFu2bNGuXbv02GOPOeOhUEgTJ07UjTfeqMrKSj333HN6+umn9bOf/cyp2b17t6ZOnaqZM2fqnXfe0QMPPKAHHnhA1dXV7d0SAACwUJQxxlzxk6Oi9Oqrr+qBBx6Q9MezOCkpKXriiSf0wx/+UJLU2NiopKQklZSUaMqUKfrggw+UkZGht99+W2PHjpUklZWV6Zvf/KY+/fRTpaSkaPXq1XryyScVDAblcrkkSQsXLtTmzZt16NAhSdLkyZN16tQpbdmyxVnPhAkTNHr0aK1Zs+ay1h8KheT1etXY2CiPx3Olbbgg/kBn5/h4SU5XLwEA0Mku9/27Qz+Tc+TIEQWDQWVnZzvHvF6vsrKyFAgEJEmBQEAJCQlOwJGk7OxsRUdHa8+ePU7N3Xff7QQcSfL7/Tp8+LC++OILp+bc12mraXudC2lqalIoFIp4AAAAO3VoyAkGg5KkpKSkiONJSUnOWDAYVGJiYsR4bGys+vfvH1FzoTnOfY2L1bSNX0hxcbG8Xq/zSE1Nbe8WAQBAN9Gjrq5atGiRGhsbncfRo0e7ekkAAOAa6dCQk5ycLEmqq6uLOF5XV+eMJScnq76+PmL87NmzOn78eETNheY49zUuVtM2fiFut1sejyfiAQAA7NShIWfIkCFKTk5WRUWFcywUCmnPnj3y+XySJJ/Pp4aGBlVWVjo127dvVzgcVlZWllOza9cutbS0ODXl5eW6+eabdd111zk1575OW03b6wAAgJ6t3SHn5MmTqqqqUlVVlaQ/fti4qqpKNTU1ioqKUn5+vn784x/rV7/6lQ4cOKCHH35YKSkpzhVYw4cP1ze+8Q19//vf1969e/W///u/mjNnjqZMmaKUlBRJ0t///d/L5XJp5syZOnjwoDZs2KDly5eroKDAWcfjjz+usrIy/du//ZsOHTqkp59+Wvv27dOcOXOuvisAAKDbi23vE/bt26d77rnH+boteOTm5qqkpEQLFizQqVOn9Nhjj6mhoUF33nmnysrKFBcX5zzn5Zdf1pw5c3TvvfcqOjpakyZN0gsvvOCMe71e/fd//7fy8vKUmZmpgQMHqqioKOJeOrfffrvWrVunp556Sv/0T/+km266SZs3b9aIESOuqBEAAMAuV3WfnO6O++R0f9wnBwB6ni65Tw4AAMDXBSEHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJViu3oBwNVIX1jaIfN8vCSnQ+YBAHx9cCYHAABYiZADAACsRMgBAABWIuQAAAArEXIAAICVCDkAAMBKhBwAAGAlQg4AALASIQcAAFiJkAMAAKxEyAEAAFYi5AAAACsRcgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYKbarFwB8HaQvLO2QeT5ektMh8wAArh5ncgAAgJUIOQAAwEqEHAAAYCVCDgAAsBIhBwAAWImQAwAArETIAQAAVuI+OUAH4n47APD1wZkcAABgJUIOAACwUrcPOStXrlR6erri4uKUlZWlvXv3dvWSAADA10C3DjkbNmxQQUGBFi9erP3792vUqFHy+/2qr6/v6qUBAIAuFmWMMV29iCuVlZWlcePGacWKFZKkcDis1NRUzZ07VwsXLvzK54dCIXm9XjU2Nsrj8XTo2jrqA6jA1eADzABsdLnv39326qrm5mZVVlZq0aJFzrHo6GhlZ2crEAhc8DlNTU1qampyvm5sbJT0x2Z1tHDTHzp8TqC90uZt6rC5qp/xd9hcAHA12t63v+o8TbcNOZ9//rlaW1uVlJQUcTwpKUmHDh264HOKi4v1zDPPnHc8NTX1mqwRsIl3WVevAAAinThxQl6v96Lj3TbkXIlFixapoKDA+TocDuv48eMaMGCAoqKiOux1QqGQUlNTdfTo0Q7/b7Dugh7QA4keSPRAogcSPWjTUX0wxujEiRNKSUm5ZF23DTkDBw5UTEyM6urqIo7X1dUpOTn5gs9xu91yu90RxxISEq7VEuXxeHr0N7NEDyR6INEDiR5I9ECiB206og+XOoPTptteXeVyuZSZmamKigrnWDgcVkVFhXw+XxeuDAAAfB102zM5klRQUKDc3FyNHTtW48eP17Jly3Tq1CnNmDGjq5cGAAC6WLcOOZMnT9axY8dUVFSkYDCo0aNHq6ys7LwPI3c2t9utxYsXn/dfYz0JPaAHEj2Q6IFEDyR60Kaz+9Ct75MDAABwMd32MzkAAACXQsgBAABWIuQAAAArEXIAAICVCDkdbOXKlUpPT1dcXJyysrK0d+/erl5ShykuLta4cePUr18/JSYm6oEHHtDhw4cjas6cOaO8vDwNGDBAffv21aRJk867YWNNTY1ycnLUu3dvJSYmav78+Tp79mxnbqXDLFmyRFFRUcrPz3eO9YQe/O53v9M//MM/aMCAAYqPj9fIkSO1b98+Z9wYo6KiIl1//fWKj49Xdna2Pvzww4g5jh8/rmnTpsnj8SghIUEzZ87UyZMnO3srV6S1tVU/+tGPNGTIEMXHx+vP//zP9c///M8Rf0fHth7s2rVL3/rWt5SSkqKoqCht3rw5Yryj9vvee+/prrvuUlxcnFJTU7V06dJrvbXLdqketLS0qLCwUCNHjlSfPn2UkpKihx9+WLW1tRFzdPceSF/9vXCuWbNmKSoqSsuWLYs43ml9MOgw69evNy6Xy/znf/6nOXjwoPn+979vEhISTF1dXVcvrUP4/X7z0ksvmerqalNVVWW++c1vmrS0NHPy5EmnZtasWSY1NdVUVFSYffv2mQkTJpjbb7/dGT979qwZMWKEyc7ONu+88455/fXXzcCBA82iRYu6YktXZe/evSY9Pd3ceuut5vHHH3eO296D48ePmxtvvNF873vfM3v27DEfffSR2bZtm/ntb3/r1CxZssR4vV6zefNm8+6775pvf/vbZsiQIeb06dNOzTe+8Q0zatQo89Zbb5lf//rXZujQoWbq1KldsaV2+8lPfmIGDBhgtmzZYo4cOWI2bdpk+vbta5YvX+7U2NaD119/3Tz55JPmlVdeMZLMq6++GjHeEfttbGw0SUlJZtq0aaa6utr84he/MPHx8eanP/1pZ23zki7Vg4aGBpOdnW02bNhgDh06ZAKBgBk/frzJzMyMmKO798CYr/5eaPPKK6+YUaNGmZSUFPP8889HjHVWHwg5HWj8+PEmLy/P+bq1tdWkpKSY4uLiLlzVtVNfX28kmZ07dxpj/vhD3qtXL7Np0yan5oMPPjCSTCAQMMb88YcjOjraBINBp2b16tXG4/GYpqamzt3AVThx4oS56aabTHl5ufnLv/xLJ+T0hB4UFhaaO++886Lj4XDYJCcnm+eee8451tDQYNxut/nFL35hjDHm/fffN5LM22+/7dRs3brVREVFmd/97nfXbvEdJCcnxzzyyCMRxx588EEzbdo0Y4z9PfjyG1tH7XfVqlXmuuuui/g5KCwsNDfffPM13lH7XerNvc3evXuNJPPJJ58YY+zrgTEX78Onn35qbrjhBlNdXW1uvPHGiJDTmX3gv6s6SHNzsyorK5Wdne0ci46OVnZ2tgKBQBeu7NppbGyUJPXv31+SVFlZqZaWlogeDBs2TGlpaU4PAoGARo4cGXHDRr/fr1AopIMHD3bi6q9OXl6ecnJyIvYq9Ywe/OpXv9LYsWP1t3/7t0pMTNRtt92mn//85874kSNHFAwGI3rg9XqVlZUV0YOEhASNHTvWqcnOzlZ0dLT27NnTeZu5QrfffrsqKir0m9/8RpL07rvv6s0339R9990nqWf04Fwdtd9AIKC7775bLpfLqfH7/Tp8+LC++OKLTtpNx2lsbFRUVJTzNxJ7Sg/C4bCmT5+u+fPn65ZbbjlvvDP7QMjpIJ9//rlaW1vPu9tyUlKSgsFgF63q2gmHw8rPz9cdd9yhESNGSJKCwaBcLtd5f/T03B4Eg8EL9qhtrDtYv3699u/fr+Li4vPGekIPPvroI61evVo33XSTtm3bptmzZ+sHP/iB1q5dK+lPe7jUz0IwGFRiYmLEeGxsrPr3798terBw4UJNmTJFw4YNU69evXTbbbcpPz9f06ZNk9QzenCujtpvd//ZONeZM2dUWFioqVOnOn+Isqf04F//9V8VGxurH/zgBxcc78w+dOs/64Cuk5eXp+rqar355ptdvZROdfToUT3++OMqLy9XXFxcVy+nS4TDYY0dO1b/8i//Ikm67bbbVF1drTVr1ig3N7eLV9c5Nm7cqJdfflnr1q3TLbfcoqqqKuXn5yslJaXH9AAX19LSor/7u7+TMUarV6/u6uV0qsrKSi1fvlz79+9XVFRUVy+HMzkdZeDAgYqJiTnvKpq6ujolJyd30aqujTlz5mjLli164403NHjwYOd4cnKympub1dDQEFF/bg+Sk5Mv2KO2sa+7yspK1dfXa8yYMYqNjVVsbKx27typF154QbGxsUpKSrK+B9dff70yMjIijg0fPlw1NTWS/rSHS/0sJCcnq76+PmL87NmzOn78eLfowfz5852zOSNHjtT06dM1b9485+xeT+jBuTpqv939Z0P6U8D55JNPVF5e7pzFkXpGD37961+rvr5eaWlpzu/ITz75RE888YTS09MldW4fCDkdxOVyKTMzUxUVFc6xcDisiooK+Xy+LlxZxzHGaM6cOXr11Ve1fft2DRkyJGI8MzNTvXr1iujB4cOHVVNT4/TA5/PpwIEDEd/gbb8IvvzG+XV077336sCBA6qqqnIeY8eO1bRp05x/296DO+6447xbB/zmN7/RjTfeKEkaMmSIkpOTI3oQCoW0Z8+eiB40NDSosrLSqdm+fbvC4bCysrI6YRdX5w9/+IOioyN/fcbExCgcDkvqGT04V0ft1+fzadeuXWppaXFqysvLdfPNN+u6667rpN1cubaA8+GHH+p//ud/NGDAgIjxntCD6dOn67333ov4HZmSkqL58+dr27Ztkjq5D+36mDIuaf369cbtdpuSkhLz/vvvm8cee8wkJCREXEXTnc2ePdt4vV6zY8cO89lnnzmPP/zhD07NrFmzTFpamtm+fbvZt2+f8fl8xufzOeNtl09PnDjRVFVVmbKyMjNo0KBuc/n0hZx7dZUx9vdg7969JjY21vzkJz8xH374oXn55ZdN7969zX/91385NUuWLDEJCQnml7/8pXnvvffMd77znQteTnzbbbeZPXv2mDfffNPcdNNNX9vLp78sNzfX3HDDDc4l5K+88ooZOHCgWbBggVNjWw9OnDhh3nnnHfPOO+8YSebf//3fzTvvvONcOdQR+21oaDBJSUlm+vTpprq62qxfv9707t37a3P59KV60NzcbL797W+bwYMHm6qqqojfkedeIdTde2DMV38vfNmXr64ypvP6QMjpYP/xH/9h0tLSjMvlMuPHjzdvvfVWVy+pw0i64OOll15yak6fPm3+8R//0Vx33XWmd+/e5rvf/a757LPPIub5+OOPzX333Wfi4+PNwIEDzRNPPGFaWlo6eTcd58shpyf04LXXXjMjRowwbrfbDBs2zPzsZz+LGA+Hw+ZHP/qRSUpKMm6329x7773m8OHDETW///3vzdSpU03fvn2Nx+MxM2bMMCdOnOjMbVyxUChkHn/8cZOWlmbi4uLMn/3Zn5knn3wy4s3Mth688cYbF/z5z83NNcZ03H7fffddc+eddxq3221uuOEGs2TJks7a4le6VA+OHDly0d+Rb7zxhjNHd++BMV/9vfBlFwo5ndWHKGPOuUUnAACAJfhMDgAAsBIhBwAAWImQAwAArETIAQAAViLkAAAAKxFyAACAlQg5AADASoQcAABgJUIOAACwEiEHAABYiZADAACsRMgBAABW+n82IA5XYmZ/0QAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#get all keyboard characters\n",
        "import string\n",
        "alpha = string.ascii_lowercase + string.digits +string.punctuation"
      ],
      "metadata": {
        "id": "ciu9QFycqJr_"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#create char level tokenisation\n",
        "num_char_tokens= len(alpha)+2 #space and oov(out of vocab,[UNK])\n",
        "char_vectorizer=TextVectorization(max_tokens=num_char_tokens,\n",
        "                                  output_sequence_length=perc_len,\n",
        "                                  name='char_vectorizer')\n",
        "#set standardization to None if to include ounctuation marks"
      ],
      "metadata": {
        "id": "ktc2PHvBqNkW"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "char_vectorizer.adapt(train_chars)"
      ],
      "metadata": {
        "id": "Nh-_JXJ_qPtB"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "char_vocab = char_vectorizer.get_vocabulary()\n",
        "len(char_vocab)"
      ],
      "metadata": {
        "id": "u2B_7aXxqR15",
        "outputId": "dc31e48d-6fff-4046-e97e-392a8ab31635",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "char_embed = layers.Embedding(input_dim=len(char_vocab),\n",
        "                                             output_dim=25,\n",
        "                                             mask_zero= True,\n",
        "                                             name='char_embedding')"
      ],
      "metadata": {
        "id": "RRVAI958qTxY"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_char_dataset=tf.data.Dataset.from_tensor_slices((train_chars,train_label_hot)).batch(32).prefetch(tf.data.AUTOTUNE)\n",
        "test_char_dataset=tf.data.Dataset.from_tensor_slices((test_chars,test_label_hot)).batch(32).prefetch(tf.data.AUTOTUNE)"
      ],
      "metadata": {
        "id": "NBFHc-VrqWmH"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# multi modal model with words vectorisation and character vectorisation\n",
        "token_inputs = layers.Input(shape=(1,),dtype=tf.string, name='token_input')\n",
        "token_vectors=text_vectorizer(token_inputs)\n",
        "token_embeddings=token_embed(token_vectors)\n",
        "token_output=layers.Dense(128,activation='relu')(token_embeddings)\n",
        "token_output=layers.Flatten()(token_output)\n",
        "token_model=tf.keras.Model(inputs=token_inputs,\n",
        "                           outputs=token_output)\n",
        "\n",
        "char_inputs = layers.Input(shape=(1,),dtype=tf.string, name='char_input')\n",
        "char_vectors=char_vectorizer(char_inputs)\n",
        "char_embeddings=char_embed(char_vectors)\n",
        "char_bi_lstm=layers.Bidirectional(layers.LSTM(25))(char_embeddings)\n",
        "char_model=tf.keras.Model(inputs=char_inputs,\n",
        "                           outputs=char_bi_lstm)\n",
        "\n",
        "token_char_concat=tf.keras.layers.Concatenate(name='token_char_hybrid')([token_model.output,\n",
        "                                                       char_model.output])\n",
        "\n",
        "combined_dropout = layers.Dropout(0.5)(token_char_concat)\n",
        "combined_dense = layers.Dense(128,activation='relu')(combined_dropout)\n",
        "final_dropout=layers.Dropout(0.5)(combined_dense)\n",
        "output_layer=layers.Dense(5,activation=\"softmax\")(final_dropout)\n",
        "\n",
        "model=tf.keras.Model(inputs=[token_model.input, char_model.input],\n",
        "                       outputs=output_layer,\n",
        "                       name='hybrid_model')\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "F7r1jWGmqa3b"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "metrics=[\"accuracy\"],\n",
        "                optimizer=tf.keras.optimizers.Adam()\n",
        "               )\n"
      ],
      "metadata": {
        "id": "K6D9wvpHqjNo"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "id": "PYLMbjmeqsaR",
        "outputId": "c1dca168-66b0-47ec-8d7a-4b7712b83946",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"hybrid_model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " token_input (InputLayer)    [(None, 1)]                  0         []                            \n",
            "                                                                                                  \n",
            " text_vectorization (TextVe  (None, 55)                   0         ['token_input[0][0]']         \n",
            " ctorization)                                                                                     \n",
            "                                                                                                  \n",
            " char_input (InputLayer)     [(None, 1)]                  0         []                            \n",
            "                                                                                                  \n",
            " token_embedding (Embedding  (None, 55, 128)              8299648   ['text_vectorization[2][0]']  \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " char_vectorizer (TextVecto  (None, 290)                  0         ['char_input[0][0]']          \n",
            " rization)                                                                                        \n",
            "                                                                                                  \n",
            " dense_5 (Dense)             (None, 55, 128)              16512     ['token_embedding[2][0]']     \n",
            "                                                                                                  \n",
            " char_embedding (Embedding)  (None, 290, 25)              700       ['char_vectorizer[2][0]']     \n",
            "                                                                                                  \n",
            " flatten_2 (Flatten)         (None, 7040)                 0         ['dense_5[0][0]']             \n",
            "                                                                                                  \n",
            " bidirectional_2 (Bidirecti  (None, 50)                   10200     ['char_embedding[2][0]']      \n",
            " onal)                                                                                            \n",
            "                                                                                                  \n",
            " token_char_hybrid (Concate  (None, 7090)                 0         ['flatten_2[0][0]',           \n",
            " nate)                                                               'bidirectional_2[0][0]']     \n",
            "                                                                                                  \n",
            " dropout_4 (Dropout)         (None, 7090)                 0         ['token_char_hybrid[0][0]']   \n",
            "                                                                                                  \n",
            " dense_6 (Dense)             (None, 128)                  907648    ['dropout_4[0][0]']           \n",
            "                                                                                                  \n",
            " dropout_5 (Dropout)         (None, 128)                  0         ['dense_6[0][0]']             \n",
            "                                                                                                  \n",
            " dense_7 (Dense)             (None, 5)                    645       ['dropout_5[0][0]']           \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 9235353 (35.23 MB)\n",
            "Trainable params: 9235353 (35.23 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combine chars and tokens into a dataset\n",
        "train_char_token_data=tf.data.Dataset.from_tensor_slices((train_sentences,train_chars))#make data\n",
        "train_char_token_label=tf.data.Dataset.from_tensor_slices(train_label_hot)#make labels\n",
        "train_char_token_dataset=tf.data.Dataset.zip((train_char_token_data,train_char_token_label))\n",
        "\n",
        "train_char_token_dataset=train_char_token_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
        "\n",
        "test_char_token_data=tf.data.Dataset.from_tensor_slices((test_sentences,test_chars))#make data\n",
        "test_char_token_label=tf.data.Dataset.from_tensor_slices(test_label_hot)#make labels\n",
        "test_char_token_dataset=tf.data.Dataset.zip((test_char_token_data,test_char_token_label))\n",
        "\n",
        "test_char_token_dataset=test_char_token_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
        "\n",
        "train_char_token_dataset,test_char_token_dataset\n"
      ],
      "metadata": {
        "id": "iaDTdbC1q0Wv",
        "outputId": "bafaac07-9c79-4e5f-9ef4-a7f88045e695",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<_PrefetchDataset element_spec=((TensorSpec(shape=(None,), dtype=tf.string, name=None), TensorSpec(shape=(None,), dtype=tf.string, name=None)), TensorSpec(shape=(None, 5), dtype=tf.float64, name=None))>,\n",
              " <_PrefetchDataset element_spec=((TensorSpec(shape=(None,), dtype=tf.string, name=None), TensorSpec(shape=(None,), dtype=tf.string, name=None)), TensorSpec(shape=(None, 5), dtype=tf.float64, name=None))>)"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "history=model.fit(train_char_token_dataset,\n",
        "                      steps_per_epoch=int(0.1*len(train_char_token_dataset)),\n",
        "                                              epochs=3,\n",
        "                      validation_data=test_char_token_dataset,\n",
        "                      validation_steps = int(0.1*len(test_char_token_dataset))\n",
        "                                              )\n"
      ],
      "metadata": {
        "id": "JP-vpAkprTvt",
        "outputId": "4129022e-e96c-4b9f-a86e-57d48af57b46",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/3\n",
            "562/562 [==============================] - 353s 605ms/step - loss: 0.9282 - accuracy: 0.6257 - val_loss: 0.6267 - val_accuracy: 0.7573\n",
            "Epoch 2/3\n",
            "562/562 [==============================] - 294s 523ms/step - loss: 0.6715 - accuracy: 0.7506 - val_loss: 0.5989 - val_accuracy: 0.7683\n",
            "Epoch 3/3\n",
            "562/562 [==============================] - 299s 532ms/step - loss: 0.6425 - accuracy: 0.7649 - val_loss: 0.5655 - val_accuracy: 0.7786\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.evaluate(test_char_token_dataset)"
      ],
      "metadata": {
        "id": "ETmpgCf8rehA",
        "outputId": "9dadbc33-99cf-47d6-d1e9-d2e6e472e52e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "942/942 [==============================] - 68s 72ms/step - loss: 0.5783 - accuracy: 0.7863\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.5782727003097534, 0.786261796951294]"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "megcoS_JvTKH"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}