Preprocessing_Dataset_Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "vocational-norwegian",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "grave-small",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import twitter_samples  # Samples twitter dataset for doing pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dress-monaco",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package twitter_samples to\n",
      "[nltk_data]     C:\\Users\\gopal\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package twitter_samples is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('twitter_samples') # Download the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "increased-gathering",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set of all positive and negative tweets\n",
    "all_positive_tweets = twitter_samples.strings('positive_tweets.json')\n",
    "all_negative_tweets = twitter_samples.strings('negative_tweets.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "steady-annotation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of positive tweets : 5000\n",
      "Total number of negative tweets : 5000\n"
     ]
    }
   ],
   "source": [
    "# Total number of tweets\n",
    "print(f'Total number of positive tweets : {len(all_positive_tweets)}')\n",
    "print(f'Total number of negative tweets : {len(all_negative_tweets)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "republican-fellowship",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#FollowFriday @France_Inte @PKuchly57 @Milipol_Paris for being top engaged members in my community this week :)\n",
      "hopeless for tmr :(\n"
     ]
    }
   ],
   "source": [
    "# Print one sample\n",
    "print(all_positive_tweets[0]) # Positive sample\n",
    "print(all_negative_tweets[0]) # Negative sample"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aerial-preliminary",
   "metadata": {},
   "source": [
    "Four important steps to follow:\n",
    "* Tokenizing the string\n",
    "* Lowercasing\n",
    "* Removing stop words and punctuation\n",
    "* Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "based-local",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My beautiful sunflowers on a sunny Friday morning off :) #sunflowers #favourites #happy #Friday off… https://t.co/3tfYom0N1i\n"
     ]
    }
   ],
   "source": [
    "# One complex tweet to check all steps\n",
    "tweet = all_positive_tweets[2277]\n",
    "print(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "institutional-reservation",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\gopal\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download stopwords\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "resistant-meeting",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original tweet :  My beautiful sunflowers on a sunny Friday morning off :) #sunflowers #favourites #happy #Friday off… https://t.co/3tfYom0N1i\n",
      "Without hyperlinks :  My beautiful sunflowers on a sunny Friday morning off :) #sunflowers #favourites #happy #Friday off… \n",
      "Without hashtags :  My beautiful sunflowers on a sunny Friday morning off :) sunflowers favourites happy Friday off… \n",
      "Without retweets tag :  My beautiful sunflowers on a sunny Friday morning off :) sunflowers favourites happy Friday off… \n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "print(\"Original tweet : \",tweet)\n",
    "# Remove hyperlinks\n",
    "tweet2 = re.sub(r'https?:\\/\\/.*[a-zA-Z0-9\\r\\n]*','',tweet)\n",
    "print(\"Without hyperlinks : \",tweet2)\n",
    "# Remove hastags\n",
    "tweet2 = re.sub(r'#','',tweet2)\n",
    "print(\"Without hashtags : \",tweet2)\n",
    "# Remove retweets : Structure like RT\n",
    "tweet2 = re.sub(r'^RT[\\s]+','',tweet2)\n",
    "print(\"Without retweets tag : \",tweet2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "alpha-vacation",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import TweetTokenizer # Tokenizer for tweets\n",
    "from nltk.stem import PorterStemmer # Used for stemming"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "residential-roots",
   "metadata": {},
   "source": [
    "# Tokenize the string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "adverse-harris",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = TweetTokenizer(preserve_case = False,reduce_len = True,strip_handles = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "touched-power",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My beautiful sunflowers on a sunny Friday morning off :) sunflowers favourites happy Friday off… \n",
      "['my', 'beautiful', 'sunflowers', 'on', 'a', 'sunny', 'friday', 'morning', 'off', ':)', 'sunflowers', 'favourites', 'happy', 'friday', 'off', '…']\n"
     ]
    }
   ],
   "source": [
    "# Tokenize\n",
    "print(tweet2)\n",
    "tweet_tokens = tokenizer.tokenize(tweet2)\n",
    "print(tweet_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "conscious-secondary",
   "metadata": {},
   "source": [
    "# Remove stopwords and punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "funky-momentum",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n",
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stopwords_english = stopwords.words('english')\n",
    "print(string.punctuation)\n",
    "print(stopwords_english)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "boxed-madonna",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_tokens = []\n",
    "for word in tweet_tokens:\n",
    "    if (word not in stopwords_english and word not in string.punctuation):\n",
    "        cleaned_tokens.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "alpine-dictionary",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['beautiful', 'sunflowers', 'sunny', 'friday', 'morning', ':)', 'sunflowers', 'favourites', 'happy', 'friday', '…']\n"
     ]
    }
   ],
   "source": [
    "print(cleaned_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "express-province",
   "metadata": {},
   "source": [
    "# Stem the tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "impossible-opposition",
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = PorterStemmer()\n",
    "stemmed_tokens = []\n",
    "for token in cleaned_tokens:\n",
    "    stemmed_word = stemmer.stem(token)\n",
    "    stemmed_tokens.append(stemmed_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "reported-leadership",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['beauti', 'sunflow', 'sunni', 'friday', 'morn', ':)', 'sunflow', 'favourit', 'happi', 'friday', '…']\n"
     ]
    }
   ],
   "source": [
    "print(stemmed_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "herbal-wells",
   "metadata": {},
   "source": [
    "That's it for the pre-processing"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gputest",
   "language": "python",
   "name": "gputest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}