nlp_twitter_sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "hLDpUczwsSxe"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "import nltk\n",
        "import spacy\n",
        "import string\n",
        "import warnings"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data=pd.read_csv(\"https://raw.githubusercontent.com/BhavanishDhamnaskar/twitterdata/main/twitter_training.csv\",names=[\"Tweet_ID\",\"entity\",\"sentiment\",\"Tweet_content\"])"
      ],
      "metadata": {
        "id": "q86uOhSrsbnB"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "y48DyClyuk7Y",
        "outputId": "c8b523f9-767c-41d1-80f9-46138378117a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID       entity sentiment  \\\n",
              "0      2401  Borderlands  Positive   \n",
              "1      2401  Borderlands  Positive   \n",
              "2      2401  Borderlands  Positive   \n",
              "3      2401  Borderlands  Positive   \n",
              "4      2401  Borderlands  Positive   \n",
              "\n",
              "                                       Tweet_content  \n",
              "0  im getting on borderlands and i will murder yo...  \n",
              "1  I am coming to the borders and I will kill you...  \n",
              "2  im getting on borderlands and i will kill you ...  \n",
              "3  im coming on borderlands and i will murder you...  \n",
              "4  im getting on borderlands 2 and i will murder ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-086b5118-1e68-4077-a397-fec8b908923d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>I am coming to the borders and I will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands 2 and i will murder ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-086b5118-1e68-4077-a397-fec8b908923d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-086b5118-1e68-4077-a397-fec8b908923d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-086b5118-1e68-4077-a397-fec8b908923d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9feaa90d-b30c-4bcf-a1c6-9cb854b2a9ce\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9feaa90d-b30c-4bcf-a1c6-9cb854b2a9ce')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9feaa90d-b30c-4bcf-a1c6-9cb854b2a9ce button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P2CXSHk8uk-b",
        "outputId": "a4abf80f-7df7-46ce-eb01-eccee76b7c46"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(74682, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_data=pd.read_csv(\"https://raw.githubusercontent.com/BhavanishDhamnaskar/twitterdata/main/twitter_validation.csv\",names=[\"Tweet_ID\",\"entity\",\"sentiment\",\"Tweet_content\"])\n"
      ],
      "metadata": {
        "id": "UR2hR5DBulBD"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "-WUZkjrculED",
        "outputId": "6ba98876-5a8b-4f8e-cbee-37bd5015318f"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID     entity   sentiment  \\\n",
              "0      3364   Facebook  Irrelevant   \n",
              "1       352     Amazon     Neutral   \n",
              "2      8312  Microsoft    Negative   \n",
              "3      4371      CS-GO    Negative   \n",
              "4      4433     Google     Neutral   \n",
              "\n",
              "                                       Tweet_content  \n",
              "0  I mentioned on Facebook that I was struggling ...  \n",
              "1  BBC News - Amazon boss Jeff Bezos rejects clai...  \n",
              "2  @Microsoft Why do I pay for WORD when it funct...  \n",
              "3  CSGO matchmaking is so full of closet hacking,...  \n",
              "4  Now the President is slapping Americans in the...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c8921cc6-3d8a-4599-b901-6f59187ebc60\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3364</td>\n",
              "      <td>Facebook</td>\n",
              "      <td>Irrelevant</td>\n",
              "      <td>I mentioned on Facebook that I was struggling ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>352</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>BBC News - Amazon boss Jeff Bezos rejects clai...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8312</td>\n",
              "      <td>Microsoft</td>\n",
              "      <td>Negative</td>\n",
              "      <td>@Microsoft Why do I pay for WORD when it funct...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4371</td>\n",
              "      <td>CS-GO</td>\n",
              "      <td>Negative</td>\n",
              "      <td>CSGO matchmaking is so full of closet hacking,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4433</td>\n",
              "      <td>Google</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>Now the President is slapping Americans in the...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c8921cc6-3d8a-4599-b901-6f59187ebc60')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c8921cc6-3d8a-4599-b901-6f59187ebc60 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c8921cc6-3d8a-4599-b901-6f59187ebc60');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-89497a15-9fb2-4afb-8873-b6012e6f7a4d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-89497a15-9fb2-4afb-8873-b6012e6f7a4d')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-89497a15-9fb2-4afb-8873-b6012e6f7a4d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.shape\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MTBvfUZ7ulG5",
        "outputId": "343a46ca-3439-474d-bf05-461025cdc8cf"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Let’s merge train and test dataset.**\n"
      ],
      "metadata": {
        "id": "nkL_ps1mu__I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.concat([train_data,test_data],axis=0)\n"
      ],
      "metadata": {
        "id": "YkziV8r2ulJh"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1=df.copy()"
      ],
      "metadata": {
        "id": "gmpceYU1ulML"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **EDA**"
      ],
      "metadata": {
        "id": "7JQKjvKqvNpQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p49M4hT2vMTw",
        "outputId": "1625d408-d43f-4ba9-a6aa-0993bff1efdd"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(75682, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NgBQDDZ4ulQ0",
        "outputId": "b01e355b-a88c-46c9-ae2e-168ed31a8fa5"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 75682 entries, 0 to 999\n",
            "Data columns (total 4 columns):\n",
            " #   Column         Non-Null Count  Dtype \n",
            "---  ------         --------------  ----- \n",
            " 0   Tweet_ID       75682 non-null  int64 \n",
            " 1   entity         75682 non-null  object\n",
            " 2   sentiment      75682 non-null  object\n",
            " 3   Tweet_content  74996 non-null  object\n",
            "dtypes: int64(1), object(3)\n",
            "memory usage: 2.9+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.isnull().sum()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-IAIPARAulTz",
        "outputId": "feefc333-5b2c-40dc-b51d-7c376c5f1d07"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Tweet_ID           0\n",
              "entity             0\n",
              "sentiment          0\n",
              "Tweet_content    686\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.duplicated().sum()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FSytRjmHulWo",
        "outputId": "e8e34e62-4b2f-491f-9b5b-72bf98971ee2"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3217"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in df.columns:\n",
        "  print(i,\"has\",df[i].nunique(),\"unique values\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ynSyDPOVulZv",
        "outputId": "cb7b54ef-b048-4b70-8b6b-07512ca9b9af"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tweet_ID has 12447 unique values\n",
            "entity has 32 unique values\n",
            "sentiment has 4 unique values\n",
            "Tweet_content has 69974 unique values\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['entity'].unique()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2PXKf3uzuldK",
        "outputId": "8e436c5e-c1e0-4746-cdea-a037e22276fa"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Borderlands', 'CallOfDutyBlackopsColdWar', 'Amazon', 'Overwatch',\n",
              "       'Xbox(Xseries)', 'NBA2K', 'Dota2', 'PlayStation5(PS5)',\n",
              "       'WorldOfCraft', 'CS-GO', 'Google', 'AssassinsCreed', 'ApexLegends',\n",
              "       'LeagueOfLegends', 'Fortnite', 'Microsoft', 'Hearthstone',\n",
              "       'Battlefield', 'PlayerUnknownsBattlegrounds(PUBG)', 'Verizon',\n",
              "       'HomeDepot', 'FIFA', 'RedDeadRedemption(RDR)', 'CallOfDuty',\n",
              "       'TomClancysRainbowSix', 'Facebook', 'GrandTheftAuto(GTA)',\n",
              "       'MaddenNFL', 'johnson&johnson', 'Cyberpunk2077',\n",
              "       'TomClancysGhostRecon', 'Nvidia'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['sentiment'].unique()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JiM8c3znuljS",
        "outputId": "dcf5b7c2-e063-41eb-b8c3-a85789c0ff88"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['Positive', 'Neutral', 'Negative', 'Irrelevant'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "LhLiwzWjvjOc",
        "outputId": "44bd0aa3-9bf5-494d-b851-5ce536b7bd39"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID       entity sentiment  \\\n",
              "0      2401  Borderlands  Positive   \n",
              "1      2401  Borderlands  Positive   \n",
              "2      2401  Borderlands  Positive   \n",
              "3      2401  Borderlands  Positive   \n",
              "4      2401  Borderlands  Positive   \n",
              "\n",
              "                                       Tweet_content  \n",
              "0  im getting on borderlands and i will murder yo...  \n",
              "1  I am coming to the borders and I will kill you...  \n",
              "2  im getting on borderlands and i will kill you ...  \n",
              "3  im coming on borderlands and i will murder you...  \n",
              "4  im getting on borderlands 2 and i will murder ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b1eafd0e-3a6a-4879-94a3-e0f6136e3117\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>I am coming to the borders and I will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands 2 and i will murder ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b1eafd0e-3a6a-4879-94a3-e0f6136e3117')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b1eafd0e-3a6a-4879-94a3-e0f6136e3117 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b1eafd0e-3a6a-4879-94a3-e0f6136e3117');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4441a32c-85db-45d5-960b-ffe2465226b7\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4441a32c-85db-45d5-960b-ffe2465226b7')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4441a32c-85db-45d5-960b-ffe2465226b7 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.dtypes\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KJtSAxh-vjRQ",
        "outputId": "ea8eb495-57b1-4f8a-b396-8d4afcad9828"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Tweet_ID          int64\n",
              "entity           object\n",
              "sentiment        object\n",
              "Tweet_content    object\n",
              "dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "All datatypes are correct and in accordance with data defination"
      ],
      "metadata": {
        "id": "0ce03W0Gvt-4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Dropping the null values"
      ],
      "metadata": {
        "id": "LelgWUcnvynx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.dropna(axis=0, how='any')\n"
      ],
      "metadata": {
        "id": "5MePLuhpvjUn"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.isnull().sum()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fbUNsrskvjXt",
        "outputId": "7d738412-f77b-4e10-b5e9-c77e890fb6d2"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Tweet_ID         0\n",
              "entity           0\n",
              "sentiment        0\n",
              "Tweet_content    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Dropping the duplicate values"
      ],
      "metadata": {
        "id": "k5MxnpRov-9I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.duplicated().sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IrF93XjKvjat",
        "outputId": "e91537c9-78ed-451a-e45c-9c9994b090fc"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2857"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Dropping the duplicates in the data.\n",
        "df.drop_duplicates(inplace=True)\n"
      ],
      "metadata": {
        "id": "WJ58w7S3vjdL"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.duplicated().sum()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UD63n7fyvjf5",
        "outputId": "aad6c8bd-a4e7-4842-872e-3f9651995511"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "YHI9IG6ivji_",
        "outputId": "ea72a2ba-4f5c-4f8c-f300-66617fffc4e6"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           Tweet_ID\n",
              "count  72139.000000\n",
              "mean    6435.469593\n",
              "std     3743.598918\n",
              "min        1.000000\n",
              "25%     3195.000000\n",
              "50%     6432.000000\n",
              "75%     9607.000000\n",
              "max    13200.000000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8610a6b2-e611-42a1-8b0c-22836d07c17f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>72139.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>6435.469593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>3743.598918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>3195.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>6432.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>9607.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>13200.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8610a6b2-e611-42a1-8b0c-22836d07c17f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8610a6b2-e611-42a1-8b0c-22836d07c17f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8610a6b2-e611-42a1-8b0c-22836d07c17f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4b6918d0-c9d9-4275-b918-8001b0d3dee1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4b6918d0-c9d9-4275-b918-8001b0d3dee1')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4b6918d0-c9d9-4275-b918-8001b0d3dee1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **TEXT PRE-PROCESSING**\n",
        "\n",
        " Lower Casing"
      ],
      "metadata": {
        "id": "KgkAshzLwNCV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " df['Tweet_content'].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cri0zzhbwMgN",
        "outputId": "c4b20848-da80-4993-8c23-d377bcc18f84"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    im getting on borderlands and i will murder yo...\n",
              "1    I am coming to the borders and I will kill you...\n",
              "2    im getting on borderlands and i will kill you ...\n",
              "3    im coming on borderlands and i will murder you...\n",
              "4    im getting on borderlands 2 and i will murder ...\n",
              "Name: Tweet_content, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"Tweet_content\"] = df[\"Tweet_content\"].str.lower()"
      ],
      "metadata": {
        "id": "CuCArH76vjl6"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content'].head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jLn1w5hXvjot",
        "outputId": "6c60f9c0-c542-4bc5-b04d-5e9fa16344ea"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    im getting on borderlands and i will murder yo...\n",
              "1    i am coming to the borders and i will kill you...\n",
              "2    im getting on borderlands and i will kill you ...\n",
              "3    im coming on borderlands and i will murder you...\n",
              "4    im getting on borderlands 2 and i will murder ...\n",
              "Name: Tweet_content, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of HTML Tags"
      ],
      "metadata": {
        "id": "8NwApwRhwe88"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Remove html tags\n",
        "import re\n",
        "def remove_html_tags(text):\n",
        "  pattern = re.compile('<.*?>')\n",
        "  return pattern.sub(r'', text)"
      ],
      "metadata": {
        "id": "jyt6mewOvjsI"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content']=df['Tweet_content'].apply(remove_html_tags)\n"
      ],
      "metadata": {
        "id": "xJdHAz40vjur"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of URLs"
      ],
      "metadata": {
        "id": "jasTp7Tuw0Gs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "##remove url\n",
        "def remove_url(text):\n",
        "  pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "  return pattern.sub(r'', text)\n"
      ],
      "metadata": {
        "id": "F5KjlBjOvjw7"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content']=df['Tweet_content'].apply(remove_url)\n"
      ],
      "metadata": {
        "id": "2O8htSgyw1x9"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of Punctuations"
      ],
      "metadata": {
        "id": "_vFVrxU9w8zu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## remove punctuations -\n",
        "import string,time\n",
        "string.punctuation\n",
        "exclude = string.punctuation\n",
        "def remove_punc(text):\n",
        "  for char in exclude:\n",
        "    text = text.replace(char,'')\n",
        "  return text\n"
      ],
      "metadata": {
        "id": "zp1d81FUw102"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content']=df['Tweet_content'].apply(remove_punc)\n"
      ],
      "metadata": {
        "id": "-lJsFI0nw13r"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chat Words Conversion\n",
        "\n",
        "This is an important text preprocessing step if we are dealing with chat data. People do use a\n",
        "lot of abbreviated words in chat and so it might be helpful to expand those words for our analysis\n",
        "purposes."
      ],
      "metadata": {
        "id": "VZvXoLm9xNMZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " chat_words = {\n",
        "\"AFAIK\": \"As Far As I Know\",\n",
        "\"AFK\": \"Away From Keyboard\",\n",
        "\"ASAP\": \"As Soon As Possible\",\n",
        "\"ATK\": \"At The Keyboard\",\n",
        "\"ATM\": \"At The Moment\",\n",
        "\"A3\": \"Anytime, Anywhere, Anyplace\",\n",
        "\"BAK\": \"Back At Keyboard\",\n",
        "\"BBL\": \"Be Back Later\",\n",
        "\"BBS\": \"Be Back Soon\",\n",
        "\"BFN\": \"Bye For Now\",\n",
        "\"B4N\": \"Bye For Now\",\n",
        "\"BRB\": \"Be Right Back\",\n",
        "\"BRT\": \"Be Right There\",\n",
        "\"BTW\": \"By The Way\",\n",
        "\"B4\": \"Before\",\n",
        "\"CU\": \"See You\",\n",
        "\"CUL8R\": \"See You Later\",\n",
        "\"CYA\": \"See You\",\n",
        "\"FAQ\": \"Frequently Asked Questions\",\n",
        "\"FC\": \"Fingers Crossed\",\n",
        "\"FWIW\": \"For What It's Worth\",\n",
        "\"FYI\": \"For Your Information\",\n",
        "\"GAL\": \"Get A Life\",\n",
        "\"GG\": \"Good Game\",\n",
        "\"GN\": \"Good Night\",\n",
        "\"GMTA\": \"Great Minds Think Alike\",\n",
        "\"GR8\": \"Great!\",\n",
        "\"G9\": \"Genius\",\n",
        "\"IC\": \"I See\",\n",
        "\"ICQ\": \"I Seek you (also a chat program)\",\n",
        "\"ILU\": \"I Love You\",\n",
        "\"IMHO\": \"In My Honest/Humble Opinion\",\n",
        "\"IMO\": \"In My Opinion\",\n",
        "\"IOW\": \"In Other Words\",\n",
        "\"IRL\": \"In Real Life\",\n",
        "\"KISS\": \"Keep It Simple, Stupid\",\n",
        "\"LDR\": \"Long Distance Relationship\",\n",
        "\"LMAO\": \"Laugh My A** Off\",\n",
        "\"LOL\": \"Laughing Out Loud\",\n",
        "\"LTNS\": \"Long Time No See\",\n",
        "\"L8R\": \"Later\",\n",
        "\"MTE\": \"My Thoughts Exactly\",\n",
        "\"M8\": \"Mate\",\n",
        "\"NRN\": \"No Reply Necessary\",\n",
        "\"OIC\": \"Oh I See\",\n",
        "\"PITA\": \"Pain In The A**\",\n",
        "\"PRT\": \"Party\",\n",
        "\"PRW\": \"Parents Are Watching\",\n",
        "\"QPSA?\": \"Que Pasa?\",\n",
        "\"ROFL\": \"Rolling On The Floor Laughing\",\n",
        "\"ROFLOL\": \"Rolling On The Floor Laughing Out Loud\",\n",
        "\"ROTFLMAO\": \"Rolling On The Floor Laughing My A** Off\",\n",
        "\"SK8\": \"Skate\",\n",
        "\"STATS\": \"Your sex and age\",\n",
        "\"ASL\": \"Age, Sex, Location\",\n",
        "\"THX\": \"Thank You\",\n",
        "\"TTFN\": \"Ta-Ta For Now!\",\n",
        "\"TTYL\": \"Talk To You Later\",\n",
        "\"U\": \"You\",\n",
        "\"U2\": \"You Too\",\n",
        "\"U4E\": \"Yours For Ever\",\n",
        "\"WB\": \"Welcome Back\",\n",
        "\"WTF\": \"What The F***\",\n",
        "\"WTG\": \"Way To Go!\",\n",
        "\"WUF\": \"Where Are You From?\",\n",
        "\"W8\": \"Wait...\",\n",
        "\"7K\": \"Sick:-D Laughter\",\n",
        "\"TFW\": \"That Feeling When\",\n",
        "\"MFW\": \"My Face When\",\n",
        "\"MRW\": \"My Reaction When\",\n",
        "\"IFYP\": \"I Feel Your Pain\",\n",
        "\"LOL\": \"Laughing Out Loud\",\n",
        "\"TNTL\": \"Trying Not To Laugh\",\n",
        "\"JK\": \"Just Kidding\",\n",
        "\"IDC\": \"I Don't Care\",\n",
        "\"ILY\": \"I Love You\",\n",
        "\"IMU\": \"I Miss You\",\n",
        "\"ADIH\": \"Another Day In Hell\",\n",
        "\"ZZZ\": \"Sleeping, Bored, Tired\",\n",
        "\"WYWH\": \"Wish You Were Here\",\n",
        "\"TIME\": \"Tears In My Eyes\",\n",
        "\"BAE\": \"Before Anyone Else\",\n",
        "\"FIMH\": \"Forever In My Heart\",\n",
        "\"BSAAW\": \"Big Smile And A Wink\",\n",
        "\"BWL\": \"Bursting With Laughter\",\n",
        "\"LMAO\": \"Laughing My A** Off\",\n",
        "\"BFF\": \"Best Friends Forever\",\n",
        "\"CSL\": \"Can't Stop Laughing\"\n",
        "}"
      ],
      "metadata": {
        "id": "LWkVkw3Pw16p"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def chat_conversion(text):\n",
        "  new_text = []\n",
        "  for w in text.split():\n",
        "    if w.upper() in chat_words:\n",
        "      new_text.append(chat_words[w.upper()])\n",
        "    else:\n",
        "      new_text.append(w)\n",
        "  return \" \".join(new_text)"
      ],
      "metadata": {
        "id": "RyhV8yMcw19Z"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        " df['Tweet_content']=df['Tweet_content'].apply(chat_conversion)\n"
      ],
      "metadata": {
        "id": "GjjWq_4Sw1_1"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of Emojis\n"
      ],
      "metadata": {
        "id": "kWkTI965xy2y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "def remove_emoji(text):\n",
        "  emoji_pattern = re.compile(\"[\"\n",
        "                        u\"\\U0001F600-\\U0001F64F\" # emoticons\n",
        "                        u\"\\U0001F300-\\U0001F5FF\" # symbols & pictographs\n",
        "                        u\"\\U0001F680-\\U0001F6FF\" # transport & map symbols\n",
        "                        u\"\\U0001F1E0-\\U0001F1FF\" # flags (iOS)\n",
        "                        u\"\\U00002702-\\U000027B0\"\n",
        "                        u\"\\U000024C2-\\U0001F251\"\n",
        "                        \"]+\", flags=re.UNICODE)\n",
        "  return emoji_pattern.sub(r'', text)"
      ],
      "metadata": {
        "id": "CyjxkZQixvPH"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "7czcKYYjxvTB",
        "outputId": "75a8e9f1-d960-440b-cd71-fc1d76d84a3e"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Tweet_ID               entity   sentiment  \\\n",
              "0        2401          Borderlands    Positive   \n",
              "1        2401          Borderlands    Positive   \n",
              "2        2401          Borderlands    Positive   \n",
              "3        2401          Borderlands    Positive   \n",
              "4        2401          Borderlands    Positive   \n",
              "..        ...                  ...         ...   \n",
              "988      7516      LeagueOfLegends     Neutral   \n",
              "989      5708            HomeDepot    Positive   \n",
              "991      2165           CallOfDuty     Neutral   \n",
              "995      4891  GrandTheftAuto(GTA)  Irrelevant   \n",
              "996      4359                CS-GO  Irrelevant   \n",
              "\n",
              "                                         Tweet_content  \n",
              "0    im getting on borderlands and i will murder yo...  \n",
              "1    i am coming to the borders and i will kill you...  \n",
              "2    im getting on borderlands and i will kill you all  \n",
              "3    im coming on borderlands and i will murder you...  \n",
              "4    im getting on borderlands 2 and i will murder ...  \n",
              "..                                                 ...  \n",
              "988  ♥️ suikoden 2 1️⃣ alex kidd in miracle world 😢...  \n",
              "989  thank you to matching funds home depot rw paym...  \n",
              "991  late night stream with the boys come watch som...  \n",
              "995  ⭐️ toronto is the arts and culture capital of ...  \n",
              "996  this is actually a good move tot bring more vi...  \n",
              "\n",
              "[72139 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8983edda-dde8-4408-ac41-e96edb49482b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>i am coming to the borders and i will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands 2 and i will murder ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>988</th>\n",
              "      <td>7516</td>\n",
              "      <td>LeagueOfLegends</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>♥️ suikoden 2 1️⃣ alex kidd in miracle world 😢...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>989</th>\n",
              "      <td>5708</td>\n",
              "      <td>HomeDepot</td>\n",
              "      <td>Positive</td>\n",
              "      <td>thank you to matching funds home depot rw paym...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>991</th>\n",
              "      <td>2165</td>\n",
              "      <td>CallOfDuty</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>late night stream with the boys come watch som...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>4891</td>\n",
              "      <td>GrandTheftAuto(GTA)</td>\n",
              "      <td>Irrelevant</td>\n",
              "      <td>⭐️ toronto is the arts and culture capital of ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>4359</td>\n",
              "      <td>CS-GO</td>\n",
              "      <td>Irrelevant</td>\n",
              "      <td>this is actually a good move tot bring more vi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>72139 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8983edda-dde8-4408-ac41-e96edb49482b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8983edda-dde8-4408-ac41-e96edb49482b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8983edda-dde8-4408-ac41-e96edb49482b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-428e9e94-86a7-4b6a-a5d0-0c44af7be37c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-428e9e94-86a7-4b6a-a5d0-0c44af7be37c')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-428e9e94-86a7-4b6a-a5d0-0c44af7be37c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content']=df['Tweet_content'].apply(remove_emoji)\n"
      ],
      "metadata": {
        "id": "0QU-QodfxvWs"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "3AbYW-1sxvaO",
        "outputId": "a257b752-f6c4-4678-bf00-ed4f4a46cecc"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Tweet_ID               entity   sentiment  \\\n",
              "0        2401          Borderlands    Positive   \n",
              "1        2401          Borderlands    Positive   \n",
              "2        2401          Borderlands    Positive   \n",
              "3        2401          Borderlands    Positive   \n",
              "4        2401          Borderlands    Positive   \n",
              "..        ...                  ...         ...   \n",
              "988      7516      LeagueOfLegends     Neutral   \n",
              "989      5708            HomeDepot    Positive   \n",
              "991      2165           CallOfDuty     Neutral   \n",
              "995      4891  GrandTheftAuto(GTA)  Irrelevant   \n",
              "996      4359                CS-GO  Irrelevant   \n",
              "\n",
              "                                         Tweet_content  \n",
              "0    im getting on borderlands and i will murder yo...  \n",
              "1    i am coming to the borders and i will kill you...  \n",
              "2    im getting on borderlands and i will kill you all  \n",
              "3    im coming on borderlands and i will murder you...  \n",
              "4    im getting on borderlands 2 and i will murder ...  \n",
              "..                                                 ...  \n",
              "988   suikoden 2 1⃣ alex kidd in miracle world  per...  \n",
              "989  thank you to matching funds home depot rw paym...  \n",
              "991  late night stream with the boys come watch som...  \n",
              "995   toronto is the arts and culture capital of ca...  \n",
              "996  this is actually a good move tot bring more vi...  \n",
              "\n",
              "[72139 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-88d958b7-9356-4366-97bd-1378e7cd797f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>i am coming to the borders and i will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands 2 and i will murder ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>988</th>\n",
              "      <td>7516</td>\n",
              "      <td>LeagueOfLegends</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>suikoden 2 1⃣ alex kidd in miracle world  per...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>989</th>\n",
              "      <td>5708</td>\n",
              "      <td>HomeDepot</td>\n",
              "      <td>Positive</td>\n",
              "      <td>thank you to matching funds home depot rw paym...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>991</th>\n",
              "      <td>2165</td>\n",
              "      <td>CallOfDuty</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>late night stream with the boys come watch som...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>4891</td>\n",
              "      <td>GrandTheftAuto(GTA)</td>\n",
              "      <td>Irrelevant</td>\n",
              "      <td>toronto is the arts and culture capital of ca...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>4359</td>\n",
              "      <td>CS-GO</td>\n",
              "      <td>Irrelevant</td>\n",
              "      <td>this is actually a good move tot bring more vi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>72139 rows × 4 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-88d958b7-9356-4366-97bd-1378e7cd797f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-88d958b7-9356-4366-97bd-1378e7cd797f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-88d958b7-9356-4366-97bd-1378e7cd797f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ef12506b-8c77-428d-a6e1-fafe02222f0e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ef12506b-8c77-428d-a6e1-fafe02222f0e')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ef12506b-8c77-428d-a6e1-fafe02222f0e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Spelling Correction\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "R2Lmu-yvycu5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " from textblob import TextBlob"
      ],
      "metadata": {
        "id": "fLUncy95xvdP"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        " def correct_spellings(text):\n",
        "  textBlb = TextBlob(text)\n",
        "  correct_text=textBlb.correct().string\n",
        "  return correct_text"
      ],
      "metadata": {
        "id": "hxW1LJcExvgl"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        " !pip install pyspellchecker"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "birdYQDmyo1x",
        "outputId": "1411bb73-95e3-4998-dd3e-5e04b759c311"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspellchecker\n",
            "  Downloading pyspellchecker-0.7.2-py3-none-any.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m30.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pyspellchecker\n",
            "Successfully installed pyspellchecker-0.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " from spellchecker import SpellChecker\n",
        "\n",
        "spell = SpellChecker()\n",
        "def correct_spelling(text):\n",
        "  corrected_text = []\n",
        "  misspelled_words = spell.unknown(text.split())\n",
        "  for word in text.split():\n",
        "    if word in misspelled_words:\n",
        "        corrected_text.append(spell.correction(word))\n",
        "    else:\n",
        "        corrected_text.append(word)\n",
        "    return \" \".join(corrected_text)\n",
        "text = \"speling correctin\"\n",
        "correct_spellings(text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "IE9U1fvvyo5Q",
        "outputId": "07daeed1-b4c0-4833-8062-484a7ebbf283"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'spelling correction'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of stopwords\n"
      ],
      "metadata": {
        "id": "JNqRwhXXzGVF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n"
      ],
      "metadata": {
        "id": "KkFNZbm4yo_W"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "STOPWORDS = set(stopwords.words('english'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7hj4uQq5UJqY",
        "outputId": "315f7248-9d77-475a-c962-aa77bd800213"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stopwords_list=stopwords.words('english')\n"
      ],
      "metadata": {
        "id": "P2zifNg3ypCV"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_stopwords(text):\n",
        "  new_text = []\n",
        "\n",
        "  for word in text.split():\n",
        "    if word in stopwords.words('english'):\n",
        "        new_text.append('')\n",
        "    else:\n",
        "        new_text.append(word)\n",
        "  x = new_text[:]\n",
        "  new_text.clear()\n",
        "  return \" \".join(x)"
      ],
      "metadata": {
        "id": "H8d65WVUypFD"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data['Tweet_content']=test_data['Tweet_content'].apply(remove_stopwords)"
      ],
      "metadata": {
        "id": "RgupmjdaUQMJ"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of Frequent words"
      ],
      "metadata": {
        "id": "zVBULZ8vUVVv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "cnt = Counter()\n",
        "for text in test_data[\"Tweet_content\"].values:\n",
        "  for word in text.split():\n",
        "    cnt[word] += 1\n",
        "\n",
        "cnt.most_common(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gBO2VDf3UYnx",
        "outputId": "703026a7-49f4-4cc9-96f8-05258cd368e9"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('I', 321),\n",
              " ('game', 79),\n",
              " ('like', 73),\n",
              " ('The', 69),\n",
              " ('get', 58),\n",
              " ('Johnson', 58),\n",
              " ('2', 57),\n",
              " ('&', 51),\n",
              " ('This', 46),\n",
              " (\"I'm\", 42)]"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cnt.most_common(15)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4xvJcGnEU9wZ",
        "outputId": "3e656900-2f68-4ad8-9829-4963921d6176"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('I', 321),\n",
              " ('game', 79),\n",
              " ('like', 73),\n",
              " ('The', 69),\n",
              " ('get', 58),\n",
              " ('Johnson', 58),\n",
              " ('2', 57),\n",
              " ('&', 51),\n",
              " ('This', 46),\n",
              " (\"I'm\", 42),\n",
              " ('-', 41),\n",
              " ('love', 41),\n",
              " ('one', 41),\n",
              " ('play', 36),\n",
              " ('good', 33)]"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "FREQWORDS = set([w for (w, wc) in cnt.most_common(10)])\n",
        "def remove_freqwords(text):\n",
        "  \"\"\"custom function to remove the frequent words\"\"\"\n",
        "  return \" \".join([word for word in str(text).split() if word not in\n",
        "  FREQWORDS])\n"
      ],
      "metadata": {
        "id": "EOiWatzHVDT8"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content'] = df['Tweet_content'].apply(lambda text:\n",
        "  remove_freqwords(text))\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "DjRpVSh6VDbm",
        "outputId": "515058f6-f2e7-4f03-852b-aa73a29266b1"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID       entity sentiment  \\\n",
              "0      2401  Borderlands  Positive   \n",
              "1      2401  Borderlands  Positive   \n",
              "2      2401  Borderlands  Positive   \n",
              "3      2401  Borderlands  Positive   \n",
              "4      2401  Borderlands  Positive   \n",
              "\n",
              "                                       Tweet_content  \n",
              "0  im getting on borderlands and i will murder yo...  \n",
              "1  i am coming to the borders and i will kill you...  \n",
              "2  im getting on borderlands and i will kill you all  \n",
              "3  im coming on borderlands and i will murder you...  \n",
              "4  im getting on borderlands and i will murder yo...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-56d1594a-4c17-4831-b863-0a5dde6a34fb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>i am coming to the borders and i will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-56d1594a-4c17-4831-b863-0a5dde6a34fb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-56d1594a-4c17-4831-b863-0a5dde6a34fb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-56d1594a-4c17-4831-b863-0a5dde6a34fb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a2b15274-46ee-4cf6-ba52-f9177fc3ef5f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a2b15274-46ee-4cf6-ba52-f9177fc3ef5f')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a2b15274-46ee-4cf6-ba52-f9177fc3ef5f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of Rare words"
      ],
      "metadata": {
        "id": "or6RYhaiVeEP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n_rare_words = 10\n",
        "RAREWORDS = set([w for (w, wc) in cnt.most_common()[:-n_rare_words-1:-1]])\n",
        "def remove_rarewords(text):\n",
        "    \"\"\"custom function to remove the rare words\"\"\"\n",
        "    return \" \".join([word for word in str(text).split() if word not in\n",
        "  RAREWORDS])\n",
        "\n",
        "df['Tweet_content'] = df['Tweet_content'].apply(lambda text:\n",
        "  remove_rarewords(text))\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Nc1paqizVafY",
        "outputId": "7b9b8fdc-55b9-4aa1-f8f6-0a57e4f00b5d"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID       entity sentiment  \\\n",
              "0      2401  Borderlands  Positive   \n",
              "1      2401  Borderlands  Positive   \n",
              "2      2401  Borderlands  Positive   \n",
              "3      2401  Borderlands  Positive   \n",
              "4      2401  Borderlands  Positive   \n",
              "\n",
              "                                       Tweet_content  \n",
              "0  im getting on borderlands and i will murder yo...  \n",
              "1  i am coming to the borders and i will kill you...  \n",
              "2  im getting on borderlands and i will kill you all  \n",
              "3  im coming on borderlands and i will murder you...  \n",
              "4  im getting on borderlands and i will murder yo...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-faabe72f-41ef-4869-8d69-16bd54d860a7\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>i am coming to the borders and i will kill you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im coming on borderlands and i will murder you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im getting on borderlands and i will murder yo...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-faabe72f-41ef-4869-8d69-16bd54d860a7')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-faabe72f-41ef-4869-8d69-16bd54d860a7 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-faabe72f-41ef-4869-8d69-16bd54d860a7');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-37ed094c-93c0-48d7-9f27-974ed198fe8b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-37ed094c-93c0-48d7-9f27-974ed198fe8b')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-37ed094c-93c0-48d7-9f27-974ed198fe8b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removal of Emoticons"
      ],
      "metadata": {
        "id": "mvFEz7eHVzMt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "EMOTICONS = {\n",
        "  u\":‑\\)\":\"Happy face or smiley\",\n",
        "  u\":\\)\":\"Happy face or smiley\",\n",
        "  u\":-\\]\":\"Happy face or smiley\",\n",
        "  u\":\\]\":\"Happy face or smiley\",\n",
        "  u\":-3\":\"Happy face smiley\",\n",
        "  u\":3\":\"Happy face smiley\",\n",
        "  u\":->\":\"Happy face smiley\",\n",
        "  u\":>\":\"Happy face smiley\",\n",
        "  u\"8-\\)\":\"Happy face smiley\",\n",
        "  u\":o\\)\":\"Happy face smiley\",\n",
        "  u\":-\\}\":\"Happy face smiley\",\n",
        "  u\":\\}\":\"Happy face smiley\",\n",
        "  u\":-\\)\":\"Happy face smiley\",\n",
        "  u\":c\\)\":\"Happy face smiley\",\n",
        "  u\":\\^\\)\":\"Happy face smiley\",\n",
        "  u\"=\\]\":\"Happy face smiley\",\n",
        "  u\"=\\)\":\"Happy face smiley\",\n",
        "  u\":‑D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\":D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"8‑D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"8D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"X‑D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"XD\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"=D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"=3\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\"B\\^D\":\"Laughing, big grin or laugh with glasses\",\n",
        "  u\":-\\)\\)\":\"Very happy\",\n",
        "  u\":‑\\(\":\"Frown, sad, andry or pouting\",\n",
        "  u\":-\\(\":\"Frown, sad, andry or pouting\",\n",
        "  u\":\\(\":\"Frown, sad, andry or pouting\",\n",
        "  u\":‑c\":\"Frown, sad, andry or pouting\",\n",
        "  u\":c\":\"Frown, sad, andry or pouting\",\n",
        "  u\":‑<\":\"Frown, sad, andry or pouting\",\n",
        "  u\":<\":\"Frown, sad, andry or pouting\",\n",
        "  u\":‑\\[\":\"Frown, sad, andry or pouting\",\n",
        "  u\":\\[\":\"Frown, sad, andry or pouting\",\n",
        "  u\":-\\|\\|\":\"Frown, sad, andry or pouting\",\n",
        "  u\">:\\[\":\"Frown, sad, andry or pouting\",\n",
        "  u\":\\{\":\"Frown, sad, andry or pouting\",\n",
        "  u\":@\":\"Frown, sad, andry or pouting\",\n",
        "  u\">:\\(\":\"Frown, sad, andry or pouting\",\n",
        "  u\":'‑\\(\":\"Crying\",\n",
        "  u\":'\\(\":\"Crying\",\n",
        "  u\":'‑\\)\":\"Tears of happiness\",\n",
        "  u\":'\\)\":\"Tears of happiness\",\n",
        "  u\"D‑':\":\"Horror\",\n",
        "  u\"D:<\":\"Disgust\",\n",
        "  u\"D:\":\"Sadness\",\n",
        "  u\"D8\":\"Great dismay\",\n",
        "  u\"D;\":\"Great dismay\",\n",
        "  u\"D=\":\"Great dismay\",\n",
        "  u\"DX\":\"Great dismay\",\n",
        "  u\":‑O\":\"Surprise\",\n",
        "  u\":O\":\"Surprise\",\n",
        "  u\":‑o\":\"Surprise\",\n",
        "  u\":o\":\"Surprise\",\n",
        "  u\":-0\":\"Shock\",\n",
        "  u\"8‑0\":\"Yawn\",\n",
        "  u\">:O\":\"Yawn\",\n",
        "  u\":-\\*\":\"Kiss\",\n",
        "  u\":\\*\":\"Kiss\",\n",
        "  u\":X\":\"Kiss\",\n",
        "  u\";‑\\)\":\"Wink or smirk\",\n",
        "  u\";\\)\":\"Wink or smirk\",\n",
        "  u\"\\*-\\)\":\"Wink or smirk\",\n",
        "  u\"\\*\\)\":\"Wink or smirk\",\n",
        "  u\";‑\\]\":\"Wink or smirk\",\n",
        "  u\";\\]\":\"Wink or smirk\",\n",
        "  u\";\\^\\)\":\"Wink or smirk\",\n",
        "  u\":‑,\":\"Wink or smirk\",\n",
        "  u\";D\":\"Wink or smirk\",\n",
        "  u\":‑P\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\":P\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\"X‑P\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\"XP\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\":‑Þ\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\":Þ\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\":b\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\"d:\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\"=p\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\">:P\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\":‑/\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":/\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":-[.]\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\">:[(\\\\\\)]\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\">:/\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":[(\\\\\\)]\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\"=/\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\"=[(\\\\\\)]\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":L\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\"=L\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":S\":\"Skeptical, annoyed, undecided, uneasy or hesitant\",\n",
        "  u\":‑\\|\":\"Straight face\",\n",
        "  u\":\\|\":\"Straight face\",\n",
        "  u\":$\":\"Embarrassed or blushing\",\n",
        "  u\":‑x\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\":x\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\":‑#\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\":#\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\":‑&\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\":&\":\"Sealed lips or wearing braces or tongue-tied\",\n",
        "  u\"O:‑\\)\":\"Angel, saint or innocent\",\n",
        "  u\"O:\\)\":\"Angel, saint or innocent\",\n",
        "  u\"0:‑3\":\"Angel, saint or innocent\",\n",
        "  u\"0:3\":\"Angel, saint or innocent\",\n",
        "  u\"0:‑\\)\":\"Angel, saint or innocent\",\n",
        "  u\"0:\\)\":\"Angel, saint or innocent\",\n",
        "  u\":‑b\":\"Tongue sticking out, cheeky, playful or blowing a raspberry\",\n",
        "  u\"0;\\^\\)\":\"Angel, saint or innocent\",\n",
        "  u\">:‑\\)\":\"Evil or devilish\",\n",
        "  u\">:\\)\":\"Evil or devilish\",\n",
        "  u\"\\}:‑\\)\":\"Evil or devilish\",\n",
        "  u\"\\}:\\)\":\"Evil or devilish\",\n",
        "  u\"3:‑\\)\":\"Evil or devilish\",\n",
        "  u\"3:\\)\":\"Evil or devilish\",\n",
        "  u\">;\\)\":\"Evil or devilish\",\n",
        "  u\"\\|;‑\\)\":\"Cool\",\n",
        "  u\"\\|‑O\":\"Bored\",\n",
        "  u\":‑J\":\"Tongue-in-cheek\",\n",
        "  u\"#‑\\)\":\"Party all night\",\n",
        "  u\"%‑\\)\":\"Drunk or confused\",\n",
        "  u\"%\\)\":\"Drunk or confused\",\n",
        "  u\":-###..\":\"Being sick\",\n",
        "  u\":###..\":\"Being sick\",\n",
        "  u\"<:‑\\|\":\"Dump\",\n",
        "  u\"\\(>_<\\)\":\"Troubled\",\n",
        "  u\"\\(>_<\\)>\":\"Troubled\",\n",
        "  u\"\\(';'\\)\":\"Baby\",\n",
        "  u\"\\(\\^\\^>``\":\"Nervous or Embarrassed or Troubled or Shy or Sweat drop\",\n",
        "  u\"\\(\\^_\\^;\\)\":\"Nervous or Embarrassed or Troubled or Shy or Sweat drop\",\n",
        "  u\"\\(-_-;\\)\":\"Nervous or Embarrassed or Troubled or Shy or Sweat drop\",\n",
        "  u\"\\(~_~;\\) \\(\\.;\\)\":\"Nervous or Embarrassed or Troubled or Shy or Sweat drop\",\n",
        "  u\"\\(-_-\\)zzz\":\"Sleeping\",\n",
        "  u\"\\(\\^_-\\)\":\"Wink\",\n",
        "  u\"\\(\\(\\+_\\+\\)\\)\":\"Confused\",\n",
        "  u\"\\(\\+o\\+\\)\":\"Confused\",\n",
        "  u\"\\(o\\|o\\)\":\"Ultraman\",\n",
        "  u\"\\^_\\^\":\"Joyful\",\n",
        "  u\"\\(\\^_\\^\\)/\":\"Joyful\",\n",
        "  u\"\\(\\^O\\^\\)\":\"Joyful\",\n",
        "  u\"\\(\\^o\\^\\)\":\"Joyful\",\n",
        "  u\"\\(__\\)\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"_\\(\\._\\.\\)_\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"<\\(_ _\\)>\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"<m\\(__\\)m>\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"m\\(__\\)m\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"m\\(_ _\\)m\":\"Kowtow as a sign of respect, or dogeza for apology\",\n",
        "  u\"\\('_'\\)\":\"Sad or Crying\",\n",
        "  u\"\\(/_;\\)\":\"Sad or Crying\",\n",
        "  u\"\\(T_T\\) \\(;_;\\)\":\"Sad or Crying\",\n",
        "  u\"\\(;_;\":\"Sad of Crying\",\n",
        "  u\"\\(;_:\\)\":\"Sad or Crying\",\n",
        "  u\"\\(;O;\\)\":\"Sad or Crying\",\n",
        "  u\"\\(:_;\\)\":\"Sad or Crying\",\n",
        "  u\"\\(ToT\\)\":\"Sad or Crying\",\n",
        "  u\";_;\":\"Sad or Crying\",\n",
        "  u\";-;\":\"Sad or Crying\",\n",
        "  u\";n;\":\"Sad or Crying\",\n",
        "  u\";;\":\"Sad or Crying\",\n",
        "  u\"Q\\.Q\":\"Sad or Crying\",\n",
        "  u\"T\\.T\":\"Sad or Crying\",\n",
        "  u\"QQ\":\"Sad or Crying\",\n",
        "  u\"Q_Q\":\"Sad or Crying\",\n",
        "  u\"\\(-\\.-\\)\":\"Shame\",\n",
        "  u\"\\(-_-\\)\":\"Shame\",\n",
        "  u\"\\(\\)\":\"Shame\",\n",
        "  u\"\\(_\\)\":\"Shame\",\n",
        "  u\"\\(=_=\\)\":\"Tired\",\n",
        "  u\"\\(=\\^\\·\\^=\\)\":\"cat\",\n",
        "  u\"\\(=\\^\\·\\·\\^=\\)\":\"cat\",\n",
        "  u\"=_\\^= \":\"cat\",\n",
        "  u\"\\(\\.\\.\\)\":\"Looking down\",\n",
        "  u\"\\(\\._\\.\\)\":\"Looking down\",\n",
        "  u\"\\^m\\^\":\"Giggling with hand covering mouth\",\n",
        "  u\"\\(\\ \\?\":\"Confusion\",\n",
        "  u\"\\(?_?\\)\":\"Confusion\",\n",
        "  u\">\\^_\\^<\":\"Normal Laugh\",\n",
        "  u\"<\\^!\\^>\":\"Normal Laugh\",\n",
        "  u\"\\^/\\^\":\"Normal Laugh\",\n",
        "  u\"\\ \\*\\^_\\^\\*\" :\"Normal Laugh\",\n",
        "  u\"\\(\\^<\\^\\) \\(\\^\\.\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(^\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^\\.\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^_\\^\\.\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^_\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^J\\^\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\*\\^\\.\\^\\*\\)\":\"Normal Laugh\",\n",
        "  u\"\\(\\^—\\^\\ \":\"Normal Laugh\",\n",
        "  u\"\\(#\\^\\.\\^#\\)\":\"Normal Laugh\",\n",
        "  u\"\\ \\^—\\^\\ \":\"Waving\",\n",
        "  u\"\\(;_;\\)/~~~\":\"Waving\",\n",
        "  u\"\\(\\^\\.\\^\\)/~~~\":\"Waving\",\n",
        "  u\"\\(-_-\\)/~~~ \\($\\·\\·\\)/~~~\":\"Waving\",\n",
        "  u\"\\(T_T\\)/~~~\":\"Waving\",\n",
        "  u\"\\(ToT\\)/~~~\":\"Waving\",\n",
        "  u\"\\(\\*\\^0\\^\\*\\)\":\"Excited\",\n",
        "  u\"\\(\\*_\\*\\)\":\"Amazed\",\n",
        "  u\"\\(\\*_\\*;\":\"Amazed\",\n",
        "  u\"\\(\\+_\\+\\) \\(@_@\\)\":\"Amazed\",\n",
        "  u\"\\(\\*\\^\\^\\)v\":\"Laughing,Cheerful\",\n",
        "  u\"\\(\\^_\\^\\)v\":\"Laughing,Cheerful\",\n",
        "  u\"\\(\\(d[-_-]b\\)\\)\":\"Headphones,Listening to music\",\n",
        "  u'\\(-\"-\\)':\"Worried\",\n",
        "  u\"\\(;\\)\":\"Worried\",\n",
        "  u\"\\(\\^0_0\\^\\)\":\"Eyeglasses\",\n",
        "  u\"\\(\\\\ \\)\":\"Happy\",\n",
        "  u\"\\(\\\\ \\)\":\"Happy\",\n",
        "  u\"\\(\\^\\)o\\(\\^\\)\":\"Happy\",\n",
        "  u\"\\(\\^O\\^\\)\":\"Happy\",\n",
        "  u\"\\(\\^o\\^\\)\":\"Happy\",\n",
        "  u\"\\)\\^o\\^\\(\":\"Happy\",\n",
        "  u\":O o_O\":\"Surprised\",\n",
        "  u\"o_0\":\"Surprised\",\n",
        "  u\"o\\.O\":\"Surpised\",\n",
        "  u\"\\(o\\.o\\)\":\"Surprised\",\n",
        "  u\"oO\":\"Surprised\",\n",
        "  u\"\\(\\*m\\)\":\"Dissatisfied\",\n",
        "  u\"\\(‘A`\\)\":\"Snubbed or Deflated\"\n",
        "}"
      ],
      "metadata": {
        "id": "ACnGbdtwVxLI"
      },
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_emoticons(text):\n",
        "  emoticon_pattern = re.compile(u'(' + u'|'.join(k for k in EMOTICONS) + u')')\n",
        "  return emoticon_pattern.sub(r'', text)"
      ],
      "metadata": {
        "id": "FTNfUxPJWZ2W"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        " df['Tweet_content']=df['Tweet_content'].apply(remove_emoticons)"
      ],
      "metadata": {
        "id": "5pgM2TCvWq2-"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "# Conversion of Emoticon words"
      ],
      "metadata": {
        "id": "LfmzDurLW5YA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_emoticons(text):\n",
        "  for emot in EMOTICONS:\n",
        "    text = re.sub(u'('+emot+')', \"_\".join(EMOTICONS[emot].replace(\",\",\"\").\n",
        "  split()), text)\n",
        "    return text\n"
      ],
      "metadata": {
        "id": "0bCQQnVzWq7d"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Stemming"
      ],
      "metadata": {
        "id": "eMKrWpJyXIrO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem.porter import PorterStemmer"
      ],
      "metadata": {
        "id": "bmSkvSkFXJnG"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ps = PorterStemmer()\n",
        "def stem_words(text):\n",
        "  return \" \".join([ps.stem(word) for word in text.split()])\n"
      ],
      "metadata": {
        "id": "hawhRuR-XNtE"
      },
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Tweet_content']=df['Tweet_content'].apply(stem_words)"
      ],
      "metadata": {
        "id": "x3VkrSjMXNwT"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# after Stemming\n",
        "df['Tweet_content'].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C3OIlbwvXNz8",
        "outputId": "54859771-537f-4789-b69b-fcf4c254c03f"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       im get on borderland and i will murder you all\n",
              "1      i am come to the border and i will kill you all\n",
              "2         im get on borderland and i will kill you all\n",
              "3      im come on borderland and i will murder you all\n",
              "4    im get on borderland and i will murder you me all\n",
              "Name: Tweet_content, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Before stemming\n",
        "df1[\"Tweet_content\"].head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qtnV-7YTXN5B",
        "outputId": "c5572340-18d7-4a35-bff7-75d88357a287"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    im getting on borderlands and i will murder yo...\n",
              "1    I am coming to the borders and I will kill you...\n",
              "2    im getting on borderlands and i will kill you ...\n",
              "3    im coming on borderlands and i will murder you...\n",
              "4    im getting on borderlands 2 and i will murder ...\n",
              "Name: Tweet_content, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lemmatization"
      ],
      "metadata": {
        "id": "f1v30MXBXZnY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('all')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O2vUnumpXyuv",
        "outputId": "96e0551a-89bf-4dda-af4b-4c8cf52fc3d5"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/alpino.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n",
            "[nltk_data]    | Downloading package bcp47 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/crubadan.zip.\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dolch.zip.\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n",
            "[nltk_data]    | Downloading package extended_omw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/floresta.zip.\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/indian.zip.\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/kimmo.zip.\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/moses_sample.zip.\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/names.zip.\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package omw-1.4 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/paradigms.zip.\n",
            "[nltk_data]    | Downloading package pe08 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pe08.zip.\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/perluniprops.zip.\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pil.zip.\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pl196x.zip.\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ptb.zip.\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/qc.zip.\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/rslp.zip.\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/rte.zip.\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/smultron.zip.\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Package stopwords is already up-to-date!\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/switchboard.zip.\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet.zip.\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet2021 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet2022 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet2022.zip.\n",
            "[nltk_data]    | Downloading package wordnet31 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/words.zip.\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ycoe.zip.\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "def lemmatize_words(text):\n",
        "    return \" \".join([lemmatizer.lemmatize(word) for word in text.split()])\n",
        "df[\"text_lemmatized\"] = df[\"Tweet_content\"].apply(lambda text:\n",
        "  lemmatize_words(text))\n",
        "df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "IIlqdbw3XN8i",
        "outputId": "b32a44a7-e815-4124-b155-91e341f07691"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Tweet_ID       entity sentiment  \\\n",
              "0      2401  Borderlands  Positive   \n",
              "1      2401  Borderlands  Positive   \n",
              "2      2401  Borderlands  Positive   \n",
              "3      2401  Borderlands  Positive   \n",
              "4      2401  Borderlands  Positive   \n",
              "\n",
              "                                       Tweet_content  \\\n",
              "0     im get on borderland and i will murder you all   \n",
              "1    i am come to the border and i will kill you all   \n",
              "2       im get on borderland and i will kill you all   \n",
              "3    im come on borderland and i will murder you all   \n",
              "4  im get on borderland and i will murder you me all   \n",
              "\n",
              "                                     text_lemmatized  \n",
              "0     im get on borderland and i will murder you all  \n",
              "1    i am come to the border and i will kill you all  \n",
              "2       im get on borderland and i will kill you all  \n",
              "3    im come on borderland and i will murder you all  \n",
              "4  im get on borderland and i will murder you me all  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-53e0e74d-7566-4cc5-8bf6-cae2ba8d84c2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tweet_ID</th>\n",
              "      <th>entity</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>Tweet_content</th>\n",
              "      <th>text_lemmatized</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im get on borderland and i will murder you all</td>\n",
              "      <td>im get on borderland and i will murder you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>i am come to the border and i will kill you all</td>\n",
              "      <td>i am come to the border and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im get on borderland and i will kill you all</td>\n",
              "      <td>im get on borderland and i will kill you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im come on borderland and i will murder you all</td>\n",
              "      <td>im come on borderland and i will murder you all</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2401</td>\n",
              "      <td>Borderlands</td>\n",
              "      <td>Positive</td>\n",
              "      <td>im get on borderland and i will murder you me all</td>\n",
              "      <td>im get on borderland and i will murder you me all</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-53e0e74d-7566-4cc5-8bf6-cae2ba8d84c2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-53e0e74d-7566-4cc5-8bf6-cae2ba8d84c2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-53e0e74d-7566-4cc5-8bf6-cae2ba8d84c2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-80c9879e-b216-4a81-a1a7-aec9babc1b7d\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-80c9879e-b216-4a81-a1a7-aec9babc1b7d')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-80c9879e-b216-4a81-a1a7-aec9babc1b7d button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenization"
      ],
      "metadata": {
        "id": "-IiIHpr-X8wb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.tokenize import word_tokenize,sent_tokenize\n"
      ],
      "metadata": {
        "id": "_i-CgNRjXOBX"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sent1 = 'I am going to visit delhi!'\n",
        "word_tokenize(sent1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpyDyvrNYAdF",
        "outputId": "8b514f8e-3c68-4b31-a3cb-97214f5b0f41"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I', 'am', 'going', 'to', 'visit', 'delhi', '!']"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"text_tokenized\"] = df[\"Tweet_content\"].apply(lambda text:\n",
        "  word_tokenize(text))\n"
      ],
      "metadata": {
        "id": "pLHYidXcYAge"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"text_tokenized\"].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dyxv8GvyYAje",
        "outputId": "b49846f8-a3fe-4644-ca56-4c1f3b956357"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [im, get, on, borderland, and, i, will, murder...\n",
              "1    [i, am, come, to, the, border, and, i, will, k...\n",
              "2    [im, get, on, borderland, and, i, will, kill, ...\n",
              "3    [im, come, on, borderland, and, i, will, murde...\n",
              "4    [im, get, on, borderland, and, i, will, murder...\n",
              "Name: text_tokenized, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    }
  ]
}