Text Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2SRQKz3E-hTK",
    "outputId": "444a552d-7396-4fa4-e1b7-28173fce535c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-12-12 11:07:03--  https://doc-0o-18-docs.googleusercontent.com/docs/securesc/e2th8t97ida4c7qm0bld5ql54cdmkla7/cqcecil34esb4cs6aet8nn4e4v3gms78/1607771175000/00484516897554883881/08403420218720776473/1rxD15nyeIPIAZ-J2VYPrDRZI66-TBWvM?e=download&authuser=0\n",
      "Resolving doc-0o-18-docs.googleusercontent.com (doc-0o-18-docs.googleusercontent.com)... 142.250.73.193, 2607:f8b0:4004:829::2001\n",
      "Connecting to doc-0o-18-docs.googleusercontent.com (doc-0o-18-docs.googleusercontent.com)|142.250.73.193|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: unspecified [application/rar]\n",
      "Saving to: ‘documents.rar’\n",
      "\n",
      "documents.rar           [  <=>               ]  18.16M  76.8MB/s    in 0.2s    \n",
      "\n",
      "2020-12-12 11:07:03 (76.8 MB/s) - ‘documents.rar’ saved [19038123]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "! wget --header=\"Host: doc-0o-18-docs.googleusercontent.com\" --header=\"User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36\" --header=\"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\" --header=\"Accept-Language: en-US,en;q=0.9,so;q=0.8\" --header=\"Referer: https://drive.google.com/\" --header=\"Cookie: AUTH_em2qh84qf50cu7hgm8cc1jk92s9uldlr=08403420218720776473|1607771025000|6anudosqero0mh30bjfqsaarqoics1pq\" --header=\"Connection: keep-alive\" \"https://doc-0o-18-docs.googleusercontent.com/docs/securesc/e2th8t97ida4c7qm0bld5ql54cdmkla7/cqcecil34esb4cs6aet8nn4e4v3gms78/1607771175000/00484516897554883881/08403420218720776473/1rxD15nyeIPIAZ-J2VYPrDRZI66-TBWvM?e=download&authuser=0\" -c -O 'documents.rar'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hgQPu7w0oGmc",
    "outputId": "dcef1987-8402-4789-c95e-576a8e0503dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-12-12 16:48:21--  https://doc-0s-18-docs.googleusercontent.com/docs/securesc/e2th8t97ida4c7qm0bld5ql54cdmkla7/8rchgttr0dsm9luj10p4ip8m2d0ttibh/1607791650000/00484516897554883881/08403420218720776473/1zJbDcbwvM2ueudqJHPpOb3Z9V2QrGd2r?e=download&authuser=0&nonce=kolsrco0ehfgg&user=08403420218720776473&hash=6hch0mih10gunlcrrer0h19m6ph2gp56\n",
      "Resolving doc-0s-18-docs.googleusercontent.com (doc-0s-18-docs.googleusercontent.com)... 142.250.73.225, 2607:f8b0:4004:82a::2001\n",
      "Connecting to doc-0s-18-docs.googleusercontent.com (doc-0s-18-docs.googleusercontent.com)|142.250.73.225|:443... connected.\n",
      "HTTP request sent, awaiting response... 416 Requested range not satisfiable\n",
      "\n",
      "    The file is already fully retrieved; nothing to do.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "! wget --header=\"Host: doc-0s-18-docs.googleusercontent.com\" --header=\"User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36\" --header=\"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\" --header=\"Accept-Language: en-US,en;q=0.9,so;q=0.8\" --header=\"Cookie: AUTH_em2qh84qf50cu7hgm8cc1jk92s9uldlr_nonce=kolsrco0ehfgg\" --header=\"Connection: keep-alive\" \"https://doc-0s-18-docs.googleusercontent.com/docs/securesc/e2th8t97ida4c7qm0bld5ql54cdmkla7/8rchgttr0dsm9luj10p4ip8m2d0ttibh/1607791650000/00484516897554883881/08403420218720776473/1zJbDcbwvM2ueudqJHPpOb3Z9V2QrGd2r?e=download&authuser=0&nonce=kolsrco0ehfgg&user=08403420218720776473&hash=6hch0mih10gunlcrrer0h19m6ph2gp56\" -c -O 'glove_vectors'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MrJoCZAg_jNS"
   },
   "outputs": [],
   "source": [
    "!mkdir document\n",
    "!unrar e documents.rar /document/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SpGLFY2vpVis"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "871fgTgQuKXe",
    "outputId": "4717e4c4-e56f-4cd1-d9d6-ede798c0840c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "984"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "KEjWVwEC_SBn"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk import ne_chunk, pos_tag, word_tokenize\n",
    "from nltk.tree import Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "CUtHJ8hfEJx_"
   },
   "outputs": [],
   "source": [
    "def preprocessing(vat):\n",
    "  vat = vat[2:]\n",
    "  emails = re.findall(\"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+)\", vat)\n",
    "  listt = []\n",
    "  listt2=[]\n",
    "  # Extracting Emails\n",
    "  for email in emails:\n",
    "    variable= email.split('@')[1]\n",
    "    variable2=variable.split(\".\")\n",
    "    for item in variable2:\n",
    "      if (len(item)> 2 and (str(item) !=\"com\")):\n",
    "        listt2.append(item)\n",
    "\n",
    "  vat = re.sub(\"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+)\",\" \",vat)\n",
    "  vat = re.sub('\\\\\\\\n','@',vat)\n",
    "\n",
    "  listt=[]\n",
    "  string2 = re.search('Subject',vat)\n",
    "  startI=string2.span()[0]\n",
    "\n",
    "  for word in (vat[startI:]):\n",
    "    if word !='@':\n",
    "      listt.append(word)\n",
    "    else:\n",
    "      break\n",
    "\n",
    "\n",
    "  total = startI\n",
    "  total = total + 9\n",
    "  listt = listt[9:]\n",
    "  count=0\n",
    "  for ch in listt:\n",
    "    count=count+1\n",
    "    if ch == ':' :\n",
    "      break\n",
    "  total = total +count\n",
    "\n",
    "  listt = listt[count+1:]\n",
    "\n",
    "  count=0\n",
    "  subject = []\n",
    "  new = \"\"\n",
    "  for x in listt:\n",
    "    count=count+1\n",
    "    if x ==\" \":\n",
    "      subject.append(new)\n",
    "      new = \"\"\n",
    "    else:\n",
    "      new = new+x\n",
    "  subject.append(new)\n",
    "  total = total + count\n",
    "\n",
    "\n",
    "  substring=vat[startI:total] \n",
    "  vat=vat.replace(substring,\" \")\n",
    "\n",
    "  var1= re.search('From:.*',vat)\n",
    "  var2 = re.search('Write to:.*',vat)\n",
    "  if (var1 !=None):\n",
    "    var1 = var1.span()[0]\n",
    "\n",
    "  if (var2 != None):\n",
    "    var2 = var2.span()[0]\n",
    "\n",
    "  count=0\n",
    "  for sym in (vat[var1:]):\n",
    "    count=count+1\n",
    "    if sym == '@':\n",
    "      break\n",
    "  substring=vat[var1:var1 + count] \n",
    "  vat=vat.replace(substring,\" \")\n",
    "\n",
    "  count=0\n",
    "  for sym in (vat[var1:]):\n",
    "    count=count+1\n",
    "    if sym == '@':\n",
    "      break\n",
    "  substring=vat[var1:var1 + count] \n",
    "  vat=vat.replace(substring,\" \")\n",
    "\n",
    "\n",
    "\n",
    "  vat = re.sub(r'\\([^)]*\\)', '', vat)\n",
    "  vat = re.sub('\\s+',' ',vat)\n",
    "  vat = re.sub(r'\\w+:\\s?','',vat)\n",
    "\n",
    "\n",
    "  vat = re.sub(r\"won't\", \"will not\", vat)\n",
    "  vat = re.sub(r\"can\\'t\", \"can not\", vat)\n",
    "  vat = re.sub(r\"n\\'t\", \" not\", vat)\n",
    "  vat = re.sub(r\"\\'re\", \" are\", vat)\n",
    "  vat = re.sub(r\"\\'s\", \" is\", vat)\n",
    "  vat = re.sub(r\"\\'d\", \" would\", vat)\n",
    "  vat = re.sub(r\"\\'ll\", \" will\", vat)\n",
    "  vat= re.sub(r\"\\'t\", \" not\", vat)\n",
    "  vat = re.sub(r\"\\'ve\", \" have\", vat)\n",
    "  vat = re.sub(r\"\\'m\", \" am\", vat)\n",
    "  \n",
    "\n",
    "  vat = vat.replace(\"@\",\" \")\n",
    "  vat = vat.replace(\"\\\\\",\" \")\n",
    "  vat = vat.replace(\"/\",\" \")\n",
    "  vat = vat.replace(\"< >\",\"\")\n",
    "  vat = vat.replace(\"$\",\"\")\n",
    "  vat = vat.replace(\"#\",\"\")\n",
    "  pattern = r'[0-9]'\n",
    "  vat = re.sub(pattern, '', vat)\n",
    "\n",
    "  vat = vat.replace(\"-\",\" \")\n",
    "  vat = vat.replace(\"_\",\" \")\n",
    "  vat = vat.replace(\"  \",\" \")\n",
    "  vat=vat.replace(\"\\\"\",\"\")\n",
    "  vat = vat.replace(\",\",\" \")\n",
    "\n",
    "  word=word_tokenize(vat)\n",
    "  tagg = pos_tag(word)\n",
    "  chunkk=list(ne_chunk(tagg))\n",
    "\n",
    "  new = \"\"\n",
    "  for i in range(0,len(list(chunkk))):\n",
    "    if (type(chunkk[i]) == nltk.tree.Tree):\n",
    "      if (chunkk[i].label() == 'GPE'):\n",
    "        for j in range(0,len(chunkk[i])):\n",
    "          new = new + chunkk[i][j][0]\n",
    "          if (j != (len(chunkk[i])-1)):\n",
    "            new = new + \"_\"\n",
    "        else:\n",
    "          new = new + \" \"\n",
    "\n",
    "      elif (chunkk[i].label() == 'PERSON'):\n",
    "        for j in range(0,len(chunkk[i])):\n",
    "          new = new + \"\"\n",
    "    \n",
    "    else:\n",
    "      new = new + chunkk[i][0]\n",
    "      new = new +\" \"\n",
    "\n",
    "\n",
    "\n",
    "  new = re.sub('\\w+\\_[^a-zA-Z]','',new)\n",
    "  new = re.sub('(?:\\s)_[^, ]*', '', new)\n",
    "  new = new.lower()\n",
    "  new = re.sub(r'\\b\\w{1,3}\\b','', new)\n",
    "  new = re.sub(\"  \",\" \",new)\n",
    "  new = re.sub(\"   \",\" \",new)\n",
    "\n",
    "  listt2.extend(subject)\n",
    "\n",
    "  listToStr = ','.join(map(str, listt2))\n",
    "  new = listToStr + new\n",
    "  return(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 66,
     "referenced_widgets": [
      "b97d5a3c20cd434b9a3cb2062013d29a",
      "e9e838e5631149c297fde4b76a0b2d12",
      "49ad9f4bf1e249f8b2112b327e4f400c",
      "0df33cb58ed64423a1885ec7abbd463d",
      "e18a73000a0740488b9a05ac80980af9",
      "dd959e0d61bc4891a1e9aa4abbea0231",
      "9246c619538f4ffa91a2d0869679d54f",
      "c70888f3d1784f9285f7b8de2c60e61a"
     ]
    },
    "id": "P5TnK792_3b8",
    "outputId": "24df7308-3fde-4b74-92c5-5c5faa4b797d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b97d5a3c20cd434b9a3cb2062013d29a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=18828.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "df = pd.DataFrame(columns=['Text','label'])   \n",
    "import os\n",
    "import re\n",
    "count=0\n",
    "for files in tqdm(os.listdir('/document')):\n",
    "  count=count+1\n",
    "  label=os.path.splitext(files)[0]\n",
    "  file = open('/document/'+files,\"rb\")\n",
    "  vat = file.read()\n",
    "  vat = str(vat)\n",
    "  label = re.sub('[0-9]','',label)\n",
    "  value = preprocessing(vat)\n",
    "  df = df.append({'Text':value,'label':label},ignore_index=True)\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "DAbTBZeUGdDK"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "y = df['label'].values\n",
    "\n",
    "df.drop('label',axis = 1, inplace = True)\n",
    "X = df\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.25,stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "rna90IFg-GS9"
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "y_train = np.array(y_train)\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "\n",
    "label_encoder = LabelEncoder()\n",
    "y_train = label_encoder.fit_transform(y_train)\n",
    "\n",
    "label_encoder = LabelEncoder()\n",
    "y_test = label_encoder.fit_transform(y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "0h-vVrlO_-lr"
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "\n",
    "y_train = np_utils.to_categorical(y_train, 20) \n",
    "y_test = np_utils.to_categorical(y_test, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wUjYF2MUTC-6"
   },
   "source": [
    "Building CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "mZNxWIIpTGkE"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "tokenizer=Tokenizer(\n",
    "    num_words= 8000,\n",
    "    filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^`{|}~\\t\\n')\n",
    "tokenizer.fit_on_texts(X_train['Text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GN8RtyD7UNFT",
    "outputId": "dc1702cb-3c7f-4eab-d679-b82d629c0a31"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "X_train['Texttoken']  = tokenizer.texts_to_sequences(X_train['Text'].values)\n",
    "X_test['Texttoken']   = tokenizer.texts_to_sequences(X_test['Text'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "KNT9xqnhUkcI"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "max_text_length = 100\n",
    "X_train_pad = sequence.pad_sequences(X_train['Texttoken'].values,maxlen = max_text_length,padding='post' )\n",
    "X_test_pad = sequence.pad_sequences(X_test['Texttoken'].values,maxlen = max_text_length,padding='post' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "HxieV2jAm3vv"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('glove_vectors', 'rb') as f:\n",
    "  glove = pickle.load(f)\n",
    "  glove_words =  set(glove.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "THo8lB8Dvehg",
    "outputId": "966af6cd-9f35-4ce6-b6ea-6d9b8706e981"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1339,  694,  852,  124,    5, 7123, 7876,    5, 1448, 3523, 5108,\n",
       "        500, 5030,   15,  752,   13,  123,    1,    3, 1186, 1812, 1131,\n",
       "         22,  724,  151, 1344,  448,  271,  527,    1,  350,  271,  718,\n",
       "       1555, 2521, 3311, 1058,  221,  382,   19, 5885, 1061, 1812, 1080,\n",
       "        382, 4090,   98,   19,  851,  221, 2474, 1080,  184, 1061, 1812,\n",
       "        200,  212,   10,   17,   18, 3080,  328,  125,   11,   54,  328,\n",
       "         14,  380,  382,   10,   14,  655,   91,  380,   30, 1812,  537,\n",
       "        328,   11,  556,   58,    2,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0], dtype=int32)"
      ]
     },
     "execution_count": 39,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_pad[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "Z0FG_hO4m8Qu"
   },
   "outputs": [],
   "source": [
    "from numpy import zeros\n",
    "max_vocabulary = len(tokenizer.word_index)\n",
    "embedding_matrix = zeros((max_vocabulary+1,300))\n",
    "for word, i in tokenizer.word_index.items():\n",
    "  if word in glove_words:\n",
    "    embedding_vector = glove[word]\n",
    "    embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "dYnlKatw2ppu"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "id": "q9oKjxRzv9S6"
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import concatenate\n",
    "from tensorflow.keras.layers import MaxPool1D\n",
    "from tensorflow.keras.layers import Dropout,Flatten\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "from keras.layers import Embedding,Dense\n",
    "from tensorflow.keras import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "WwX6HVr0x9xr"
   },
   "outputs": [],
   "source": [
    "inputs = keras.layers.Input(shape=(max_text_length,))\n",
    "X = Embedding(output_dim=300, input_dim= max_vocabulary+1, input_length=100, weights=[embedding_matrix])(inputs)\n",
    "first = Conv1D(10, 2, activation=\"relu\")(X)\n",
    "second = Conv1D(11,2,activation=\"relu\")(X)\n",
    "third = Conv1D(12,2,activation=\"relu\")(X)\n",
    "\n",
    "x = concatenate([first, second, third])\n",
    "\n",
    "max1 = MaxPool1D(pool_size=2,strides=1, padding='valid')(x)\n",
    "max1 = BatchNormalization()(max1)\n",
    "\n",
    "\n",
    "\n",
    "first2 = Conv1D(12, 2, activation=\"relu\",kernel_initializer=\"glorot_normal\")(max1)\n",
    "second2 = Conv1D(10,2,activation=\"relu\",kernel_initializer=\"glorot_normal\")(max1)\n",
    "third2 = Conv1D(8,2,activation=\"relu\",kernel_initializer=\"glorot_normal\")(max1)\n",
    "\n",
    "x2 = concatenate([first2, second2, third2])\n",
    "\n",
    "max2 = MaxPool1D(pool_size=2,strides=1, padding='valid')(x2)\n",
    "max2 = BatchNormalization()(max2)\n",
    "\n",
    "\n",
    "first3 = Conv1D(12,2,activation = \"relu\",kernel_initializer=\"glorot_normal\")(max2)\n",
    "\n",
    "second3 = Flatten()(first3)\n",
    "dropout1 = Dropout(0.3)(second3)\n",
    "\n",
    "third3 = Dense(21,activation=\"relu\",kernel_initializer=\"glorot_normal\")(dropout1)\n",
    "third3 = BatchNormalization()(third3)\n",
    "third3 = Dropout(0.3)(third3)\n",
    "\n",
    "\n",
    "output = Dense(20,activation=\"softmax\",name = \"output\")(third3)\n",
    "\n",
    "model = Model(inputs= inputs, outputs= output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GiE5h7NB7rRZ",
    "outputId": "8b7f340c-0c81-4e92-a372-7721b51fe493"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_7\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_8 (InputLayer)            [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 100, 300)     21488100    input_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_35 (Conv1D)              (None, 99, 10)       6010        embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_36 (Conv1D)              (None, 99, 11)       6611        embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_37 (Conv1D)              (None, 99, 12)       7212        embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 99, 33)       0           conv1d_35[0][0]                  \n",
      "                                                                 conv1d_36[0][0]                  \n",
      "                                                                 conv1d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_10 (MaxPooling1D) (None, 98, 33)       0           concatenate_10[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 98, 33)       132         max_pooling1d_10[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_38 (Conv1D)              (None, 97, 12)       804         batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_39 (Conv1D)              (None, 97, 10)       670         batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_40 (Conv1D)              (None, 97, 8)        536         batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)    (None, 97, 30)       0           conv1d_38[0][0]                  \n",
      "                                                                 conv1d_39[0][0]                  \n",
      "                                                                 conv1d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling1D) (None, 96, 30)       0           concatenate_11[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 96, 30)       120         max_pooling1d_11[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_41 (Conv1D)              (None, 95, 12)       732         batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 1140)         0           conv1d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 1140)         0           flatten_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 21)           23961       dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 21)           84          dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 21)           0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 20)           440         dropout_10[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 21,535,412\n",
      "Trainable params: 21,535,244\n",
      "Non-trainable params: 168\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "id": "mIaXfLC6RDqq"
   },
   "outputs": [],
   "source": [
    "def decrease(epoch, lr):    \n",
    "  if (epoch%3 == 0 and epoch!=0):    \n",
    "    print(epoch)\n",
    "    lr = lr - (0.1 * lr)\n",
    "    return lr\n",
    "  else:\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "id": "_uqT1q6o7s0D"
   },
   "outputs": [],
   "source": [
    "DynamicLR = ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto',min_delta=0, cooldown=0, min_lr=0.0001)\n",
    "adaptiveLR =  LearningRateScheduler(decrease)\n",
    "adam = tf.keras.optimizers.Adam(0.001)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3fy0sKsM7vpZ",
    "outputId": "31d7e6a5-b348-4c5c-fdc3-057ca85ad557"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "142/142 [==============================] - 29s 202ms/step - loss: 3.0720 - accuracy: 0.0952 - val_loss: 2.6887 - val_accuracy: 0.1717\n",
      "Epoch 2/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 2.3832 - accuracy: 0.2488 - val_loss: 2.1517 - val_accuracy: 0.3193\n",
      "Epoch 3/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 2.0079 - accuracy: 0.3580 - val_loss: 1.7943 - val_accuracy: 0.4406\n",
      "3\n",
      "Epoch 4/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 1.6778 - accuracy: 0.4707 - val_loss: 1.5208 - val_accuracy: 0.5377\n",
      "Epoch 5/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 1.4143 - accuracy: 0.5557 - val_loss: 1.3653 - val_accuracy: 0.5749\n",
      "Epoch 6/10\n",
      "142/142 [==============================] - 28s 197ms/step - loss: 1.1938 - accuracy: 0.6227 - val_loss: 1.1768 - val_accuracy: 0.6473\n",
      "6\n",
      "Epoch 7/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 0.9973 - accuracy: 0.6980 - val_loss: 1.0771 - val_accuracy: 0.6707\n",
      "Epoch 8/10\n",
      "142/142 [==============================] - 28s 197ms/step - loss: 0.8511 - accuracy: 0.7438 - val_loss: 1.0171 - val_accuracy: 0.6851\n",
      "Epoch 9/10\n",
      "142/142 [==============================] - ETA: 0s - loss: 0.7235 - accuracy: 0.7856\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0007290000503417104.\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 0.7235 - accuracy: 0.7856 - val_loss: 1.0372 - val_accuracy: 0.6805\n",
      "9\n",
      "Epoch 10/10\n",
      "142/142 [==============================] - 28s 198ms/step - loss: 0.6181 - accuracy: 0.8227 - val_loss: 0.9538 - val_accuracy: 0.7130\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_pad,y_train,batch_size=100, epochs=10, verbose=1, validation_data = (X_test_pad,y_test),callbacks = [DynamicLR,adaptiveLR])"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "24.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0df33cb58ed64423a1885ec7abbd463d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c70888f3d1784f9285f7b8de2c60e61a",
      "placeholder": "​",
      "style": "IPY_MODEL_9246c619538f4ffa91a2d0869679d54f",
      "value": " 18828/18828 [31:05&lt;00:00, 10.09it/s]"
     }
    },
    "49ad9f4bf1e249f8b2112b327e4f400c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dd959e0d61bc4891a1e9aa4abbea0231",
      "max": 18828,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e18a73000a0740488b9a05ac80980af9",
      "value": 18828
     }
    },
    "9246c619538f4ffa91a2d0869679d54f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b97d5a3c20cd434b9a3cb2062013d29a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_49ad9f4bf1e249f8b2112b327e4f400c",
       "IPY_MODEL_0df33cb58ed64423a1885ec7abbd463d"
      ],
      "layout": "IPY_MODEL_e9e838e5631149c297fde4b76a0b2d12"
     }
    },
    "c70888f3d1784f9285f7b8de2c60e61a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dd959e0d61bc4891a1e9aa4abbea0231": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e18a73000a0740488b9a05ac80980af9": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e9e838e5631149c297fde4b76a0b2d12": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}