Sentiment_Classification PreProcessing Applied.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "fA89oNsQUwQU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53e32660-d0c6-4c54-c46d-2255e4f72c51"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "short_pos = open(\"/neg.txt\",\"r\").read()\n",
        "short_neg = open(\"/pos.txt\",\"r\").read()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "isUOFyd60ezi",
        "outputId": "0825746e-ac21-44cf-bafa-7491e4f0ba82"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "documents = []\n",
        "\n",
        "for r in short_pos.split('\\n'):\n",
        "    documents.append( (r, \"pos\") )\n",
        "\n",
        "for r in short_neg.split('\\n'):\n",
        "    documents.append( (r, \"neg\") )\n",
        "\n",
        "\n",
        "print(documents[:5])\n",
        "len(documents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M2bJUijDZxtf",
        "outputId": "26b9c181-9317-4d10-970d-7037e9fd6d1a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('simplistic , silly and tedious . ', 'pos'), (\"it's so laddish and juvenile , only teenage boys could possibly find it funny . \", 'pos'), ('exploitative and largely devoid of the depth or sophistication that would make watching such a graphic treatment of the crimes bearable . ', 'pos'), ('[garbus] discards the potential for pathological study , exhuming instead , the skewed melodrama of the circumstantial situation . ', 'pos'), ('a visually flashy but narratively opaque and emotionally vapid exercise in style and mystification . ', 'pos')]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "22"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "# !pip install autocorrect\n",
        "from nltk import word_tokenize\n",
        "from autocorrect import Speller\n",
        "import spacy\n",
        "\n",
        "nltk.download('punkt')\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "spell = Speller()\n",
        "all_words = []\n",
        "\n",
        "def preprocessing(text):\n",
        "    text1=spell(text)\n",
        "    doc = nlp(text1)\n",
        "    tokens = [token.text.lower() for token in doc if not (token.is_stop or not token.is_alpha)]\n",
        "    lemmas=[token.lemma_ for token in nlp(' '.join(tokens))]\n",
        "    return lemmas\n",
        "\n",
        "short_pos_words = preprocessing(short_pos)\n",
        "short_neg_words = preprocessing(short_neg)\n",
        "print(len(short_pos_words))\n",
        "print(len(short_neg_words))\n",
        "\n",
        "for w in short_pos_words:\n",
        "    all_words.append(w)\n",
        "# print(len(all_words))\n",
        "for w in short_neg_words:\n",
        "    all_words.append(w)\n",
        "# print(len(all_words))\n",
        "all_words = nltk.FreqDist(all_words)\n",
        "len(all_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hSj-W8O0aGC3",
        "outputId": "42c9e451-3e81-4dc6-c1eb-9bc7dfa2ab6b"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "79\n",
            "92\n",
            "79\n",
            "171\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "162"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "word_features = list(all_words.keys())[:5000]\n",
        "word_features"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JVKi-bUZbIQd",
        "outputId": "708f9463-75f3-4dbd-b014-3e451ff8b7eb"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['simplistic',\n",
              " 'silly',\n",
              " 'tedious',\n",
              " 'reddish',\n",
              " 'juvenile',\n",
              " 'teenage',\n",
              " 'boy',\n",
              " 'possibly',\n",
              " 'find',\n",
              " 'funny',\n",
              " 'exploitation',\n",
              " 'largely',\n",
              " 'devoid',\n",
              " 'depth',\n",
              " 'sophistication',\n",
              " 'watch',\n",
              " 'graphic',\n",
              " 'treatment',\n",
              " 'crime',\n",
              " 'wearable',\n",
              " 'grab',\n",
              " 'discard',\n",
              " 'potential',\n",
              " 'pathological',\n",
              " 'study',\n",
              " 'exhuming',\n",
              " 'instead',\n",
              " 'skew',\n",
              " 'melodrama',\n",
              " 'circumstantial',\n",
              " 'situation',\n",
              " 'visually',\n",
              " 'flashy',\n",
              " 'narrative',\n",
              " 'opaque',\n",
              " 'emotionally',\n",
              " 'valid',\n",
              " 'exercise',\n",
              " 'style',\n",
              " 'justification',\n",
              " 'story',\n",
              " 'original',\n",
              " 'come',\n",
              " 'having',\n",
              " 'recycle',\n",
              " 'time',\n",
              " 'care',\n",
              " 'count',\n",
              " 'thing',\n",
              " 'movie',\n",
              " 'point',\n",
              " 'bravo',\n",
              " 'entirely',\n",
              " 'stale',\n",
              " 'concept',\n",
              " 'push',\n",
              " 'audience',\n",
              " 'meat',\n",
              " 'grinder',\n",
              " 'facial',\n",
              " 'sour',\n",
              " 'unfortunately',\n",
              " 'actor',\n",
              " 'serve',\n",
              " 'hack',\n",
              " 'script',\n",
              " 'disquiet',\n",
              " 'relatively',\n",
              " 'gore',\n",
              " 'free',\n",
              " 'allusion',\n",
              " 'serial',\n",
              " 'murder',\n",
              " 'fall',\n",
              " 'attempts',\n",
              " 'humanity',\n",
              " 'subject',\n",
              " 'rock',\n",
              " 'destine',\n",
              " 'century',\n",
              " 'new',\n",
              " 'conan',\n",
              " 'going',\n",
              " 'splash',\n",
              " 'great',\n",
              " 'arnold',\n",
              " 'schwarzenegger',\n",
              " 'jean',\n",
              " 'cloud',\n",
              " 'van',\n",
              " 'dame',\n",
              " 'steven',\n",
              " 'legal',\n",
              " 'gorgeous',\n",
              " 'elaborate',\n",
              " 'continuation',\n",
              " 'lord',\n",
              " 'ring',\n",
              " 'trilogy',\n",
              " 'huge',\n",
              " 'column',\n",
              " 'word',\n",
              " 'adequately',\n",
              " 'describe',\n",
              " 'co',\n",
              " 'writer',\n",
              " 'director',\n",
              " 'peter',\n",
              " 'jackson',\n",
              " 'expand',\n",
              " 'vision',\n",
              " 'j',\n",
              " 'r',\n",
              " 'token',\n",
              " 'middle',\n",
              " 'earth',\n",
              " 'effective',\n",
              " 'revid',\n",
              " 'biopic',\n",
              " 'like',\n",
              " 'fun',\n",
              " 'asahi',\n",
              " 'good',\n",
              " 'place',\n",
              " 'start',\n",
              " 'emerge',\n",
              " 'rare',\n",
              " 'issue',\n",
              " 'honest',\n",
              " 'weekly',\n",
              " 'observe',\n",
              " 'feel',\n",
              " 'film',\n",
              " 'provide',\n",
              " 'insight',\n",
              " 'erotic',\n",
              " 'mindset',\n",
              " 'comic_strip',\n",
              " 'reach',\n",
              " 'absolute',\n",
              " 'game',\n",
              " 'offer',\n",
              " 'combination',\n",
              " 'entertainment',\n",
              " 'education',\n",
              " 'picture',\n",
              " 'literally',\n",
              " 'show',\n",
              " 'road',\n",
              " 'hell',\n",
              " 'pave',\n",
              " 'intention',\n",
              " 'steer',\n",
              " 'turn',\n",
              " 'happy',\n",
              " 'screenplay',\n",
              " 'url',\n",
              " 'edge',\n",
              " 'clever',\n",
              " 'want',\n",
              " 'hate',\n",
              " 'pull']"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def find_features(document):\n",
        "    words = word_tokenize(document)\n",
        "    features = {}\n",
        "    for w in word_features:\n",
        "        features[w] = (w in words)\n",
        "\n",
        "    return features"
      ],
      "metadata": {
        "id": "WSfP-bIlblTN"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "featuresets = [(find_features(rev), category) for (rev, category) in documents]"
      ],
      "metadata": {
        "id": "2skJqUeLbmFv"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "random.shuffle(featuresets)\n",
        "\n",
        "featuresets[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ys3EhUPJb3g9",
        "outputId": "843d1242-a35f-4653-d4f3-be1cc7ec3d25"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'simplistic': False,\n",
              "  'silly': False,\n",
              "  'tedious': False,\n",
              "  'reddish': False,\n",
              "  'juvenile': False,\n",
              "  'teenage': False,\n",
              "  'boy': False,\n",
              "  'possibly': False,\n",
              "  'find': False,\n",
              "  'funny': False,\n",
              "  'exploitation': False,\n",
              "  'largely': False,\n",
              "  'devoid': False,\n",
              "  'depth': False,\n",
              "  'sophistication': False,\n",
              "  'watch': False,\n",
              "  'graphic': False,\n",
              "  'treatment': False,\n",
              "  'crime': False,\n",
              "  'wearable': False,\n",
              "  'grab': False,\n",
              "  'discard': False,\n",
              "  'potential': False,\n",
              "  'pathological': False,\n",
              "  'study': False,\n",
              "  'exhuming': False,\n",
              "  'instead': False,\n",
              "  'skew': False,\n",
              "  'melodrama': False,\n",
              "  'circumstantial': False,\n",
              "  'situation': False,\n",
              "  'visually': False,\n",
              "  'flashy': False,\n",
              "  'narrative': False,\n",
              "  'opaque': False,\n",
              "  'emotionally': False,\n",
              "  'valid': False,\n",
              "  'exercise': False,\n",
              "  'style': False,\n",
              "  'justification': False,\n",
              "  'story': False,\n",
              "  'original': False,\n",
              "  'come': False,\n",
              "  'having': False,\n",
              "  'recycle': False,\n",
              "  'time': False,\n",
              "  'care': False,\n",
              "  'count': False,\n",
              "  'thing': False,\n",
              "  'movie': False,\n",
              "  'point': False,\n",
              "  'bravo': False,\n",
              "  'entirely': False,\n",
              "  'stale': False,\n",
              "  'concept': False,\n",
              "  'push': False,\n",
              "  'audience': False,\n",
              "  'meat': False,\n",
              "  'grinder': False,\n",
              "  'facial': False,\n",
              "  'sour': False,\n",
              "  'unfortunately': False,\n",
              "  'actor': False,\n",
              "  'serve': False,\n",
              "  'hack': False,\n",
              "  'script': False,\n",
              "  'disquiet': False,\n",
              "  'relatively': True,\n",
              "  'gore': False,\n",
              "  'free': False,\n",
              "  'allusion': False,\n",
              "  'serial': True,\n",
              "  'murder': False,\n",
              "  'fall': False,\n",
              "  'attempts': True,\n",
              "  'humanity': False,\n",
              "  'subject': True,\n",
              "  'rock': False,\n",
              "  'destine': False,\n",
              "  'century': False,\n",
              "  'new': False,\n",
              "  'conan': False,\n",
              "  'going': False,\n",
              "  'splash': False,\n",
              "  'great': False,\n",
              "  'arnold': False,\n",
              "  'schwarzenegger': False,\n",
              "  'jean': False,\n",
              "  'cloud': False,\n",
              "  'van': False,\n",
              "  'dame': False,\n",
              "  'steven': False,\n",
              "  'legal': False,\n",
              "  'gorgeous': False,\n",
              "  'elaborate': False,\n",
              "  'continuation': False,\n",
              "  'lord': False,\n",
              "  'ring': False,\n",
              "  'trilogy': False,\n",
              "  'huge': False,\n",
              "  'column': False,\n",
              "  'word': False,\n",
              "  'adequately': False,\n",
              "  'describe': False,\n",
              "  'co': False,\n",
              "  'writer': False,\n",
              "  'director': False,\n",
              "  'peter': False,\n",
              "  'jackson': False,\n",
              "  'expand': False,\n",
              "  'vision': False,\n",
              "  'j': False,\n",
              "  'r': False,\n",
              "  'token': False,\n",
              "  'middle': False,\n",
              "  'earth': False,\n",
              "  'effective': False,\n",
              "  'revid': False,\n",
              "  'biopic': False,\n",
              "  'like': False,\n",
              "  'fun': False,\n",
              "  'asahi': False,\n",
              "  'good': False,\n",
              "  'place': False,\n",
              "  'start': False,\n",
              "  'emerge': False,\n",
              "  'rare': False,\n",
              "  'issue': False,\n",
              "  'honest': False,\n",
              "  'weekly': False,\n",
              "  'observe': False,\n",
              "  'feel': False,\n",
              "  'film': False,\n",
              "  'provide': False,\n",
              "  'insight': False,\n",
              "  'erotic': False,\n",
              "  'mindset': False,\n",
              "  'comic_strip': False,\n",
              "  'reach': False,\n",
              "  'absolute': False,\n",
              "  'game': False,\n",
              "  'offer': False,\n",
              "  'combination': False,\n",
              "  'entertainment': False,\n",
              "  'education': False,\n",
              "  'picture': False,\n",
              "  'literally': False,\n",
              "  'show': False,\n",
              "  'road': False,\n",
              "  'hell': False,\n",
              "  'pave': False,\n",
              "  'intention': False,\n",
              "  'steer': False,\n",
              "  'turn': False,\n",
              "  'happy': False,\n",
              "  'screenplay': False,\n",
              "  'url': False,\n",
              "  'edge': False,\n",
              "  'clever': False,\n",
              "  'want': False,\n",
              "  'hate': False,\n",
              "  'pull': False},\n",
              " 'pos')"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    }
  ]
}