Fake and Real News Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7afe0480",
   "metadata": {},
   "source": [
    "# Project 1\n",
    "\n",
    "# Fake-News Cleaning+Word2Vec+LSTM (99% Accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81506cc5",
   "metadata": {},
   "source": [
    "dataset in kaggle in:\n",
    "\n",
    "https://www.kaggle.com/clmentbisaillon/fake-and-real-news-dataset\n",
    "\n",
    "https://www.kaggle.com/atishadhikari/fake-news-cleaning-word2vec-Istm-99-accuracy\n",
    "\n",
    "divided to 2 files:\n",
    "-Fake.csv\n",
    "-True.csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "119faeec",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings \n",
    "warnings.filterwarnings ('ignore')\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns \n",
    "import nltk \n",
    "import re\n",
    "from wordcloud \n",
    "import WordCloud   #pip install wordcloud \n",
    "from tensorflow.keras.preprocessing.text import Tokenizer \n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences \n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Embedding, LSTM, Conv1D, MaxPool1D \n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "fake = pd.read_csv (\"Fake.csv\")\n",
    "print (fake.head ( ))\n",
    "\n",
    "# Counting by Subjects\n",
    "for key, count in fake.subject.value_counts ().iteritems ():\n",
    "    print(f\"{key}:\\t{count}\")\n",
    "# Getting Total Rows\n",
    "print (f\"Total Records: \\t{fake.shape [0]}\")\n",
    "pit.figure(figsize=(8,5))\n",
    "sns.countplot (\"subject\", data=fake)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d672904a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Word Cloud\n",
    "# will show the most words freq in text\n",
    "text = \"\"\n",
    "for news in fake.text.values:\n",
    "    text += f\" {news}\"\n",
    "wordcloud = WordCloud(\n",
    "    width = 3000,\n",
    "    height = 2000,\n",
    "    background_color = 'black',\n",
    "    stopwords = set(nltk.corpus.stopwords.words(\"english\"))).generate(text)\n",
    "fig = plt.figure(\n",
    "    figsize = (40, 30),\n",
    "    facecolor = 'k',\n",
    "    edgecolor = 'k')\n",
    "plt.imshow (wordcloud, interpolation = 'bilinear')\n",
    "plt.axis('off')\n",
    "plt.tight_layout (pad=0)\n",
    "plt.show()\n",
    "del text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0167cb95",
   "metadata": {},
   "outputs": [],
   "source": [
    "real = pd.read_csv (\"True.csv\")\n",
    "print (real.head ())\n",
    "#First Creating list of index that do not have publication part\n",
    "unknown_publishers = []\n",
    "for index, row in enumerate (real.text.values):\n",
    "    try:\n",
    "        record = row.split(\" -\", maxsplit=1)\n",
    "        #if no text part is present, following will give error \n",
    "        record [1]\n",
    "        #if len of piblication part is greater than 260 \n",
    "        #following will give error, ensuring no text having \"_\" in between is counted\n",
    "        assert(len(record [0]) ‹ 260)\n",
    "    except:\n",
    "        unknown_publishers.append(index)\n",
    "#Thus we have list of indices where publisher is not mentioned \n",
    "#lets check\n",
    "print (real. iloc [unknown_publishers].text)\n",
    "#true, they do not have text like \"WASHINGTON (Reuters)\"\n",
    "\n",
    "\n",
    "print (real. iloc [8970])\n",
    "#уер empty\n",
    "#will remove this soon\n",
    "# Seperating Publication info, from actual text\n",
    "publisher = []\n",
    "tmp_text = []\n",
    "for index, row in enumerate (real. text. values) :\n",
    "    if index in unknown_publishers:\n",
    "        # Add unknown of publisher not mentioned\n",
    "        tmp_text.append(row)\n",
    "        publisher .append(\"Unknown\")\n",
    "        continue\n",
    "    record = row.split(\" -\", maxsplit=1)\n",
    "    publisher.append(record[0])\n",
    "    tmp text. append (record[1])\n",
    "# Replace existing text column with new text\n",
    "# add seperate column for publication info\n",
    "real[\"publisher\"] = publisher\n",
    "real[\"text\"] = tmp text\n",
    "\n",
    "\n",
    "del publisher, tmp_text, record, unknown_publishers \n",
    "print ( real. head())\n",
    "#checking for rows with empty text like row: 8970\n",
    "[print(\"empty text:\",index) for index, text in enumerate (real.text.values) if str(text).strip() == ''] \n",
    "#seems only one :)\n",
    "#empty text: 8970\n",
    "#dropping this record\n",
    "real = real. drop (8970, axis=0)\n",
    "# checking for the same in fake news\n",
    "empty_fake_index = [index for index, text in enumerate (fake.text.values) if str(text).strip() == '']\n",
    "print (f\"No of empty rows: {len (empty_fake_index)}\")\n",
    "fake. iloc[empty_fake_index].tail()\n",
    "#No of empty rows: 630\n",
    "#checking for rows with empty text like row: 8970\n",
    "[print(\"empty text:\", index) for index, text in enumerate (real.text.values) if str(text) .strip() == \"']\n",
    "#seems only one :)\n",
    " \n",
    "sns.countplot(x=\"subject\", data=real) \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29c2df5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#WordCloud For Real News\n",
    "text = ''\n",
    "for news in real.text.values:\n",
    "    text += f\" {news}\"\n",
    "wordcloud = WordCloud(\n",
    "    width = 3000,\n",
    "    height = 2000,\n",
    "    background_color = 'black',\n",
    "    stopwords = set(nltk.corpus.stopwords.words(\"english\"))).generate(str (text))\n",
    "fig = pit.figure(\n",
    "    figsize = (40, 30),\n",
    "    facecolor = 'k',\n",
    "    edgecolor = 'k')\n",
    "plt.imshow(wordcloud, interpolation = 'bilinear')\n",
    "plt.axis ('off')\n",
    "plt.tight_ layout (pad=0)\n",
    "plt.show()\n",
    "del text\n",
    "\n",
    "\n",
    "# Adding class Information\n",
    "real[\"class\"] = 1\n",
    "fake [\"class\"] = 0\n",
    "#Combining Title and Text\n",
    "real [\"text\"] = real[\"title\" ]+ \" \" + real [\"text\"]\n",
    "fake[\"text\"] = fake[\"title\"] + \" \" + fake[\"text\"]\n",
    "# Subject is diffrent for real and fake thus dropping it\n",
    "# Also dropping Date, title and Publication Info of real\n",
    "real = real.drop ([\"subject\", \"date\", \"title\", \"publisher\"], axis=1)\n",
    "fake = fake.drop([\"subject\", \"date\", \"title\"], axis=1)\n",
    "#Combining both into new dataframe\n",
    "data = real.append(fake, ignore_index=True)\n",
    "del real, fake\n",
    "\n",
    "\n",
    "# Download following if not downloaded in local machine\n",
    "# nltk.download('stopwords\")\n",
    "# nLtk. download ( 'punkt\")\n",
    "\n",
    "#Removing StopWords, Punctuations and single-character words\n",
    "y = data[\"class\"].values\n",
    "#Converting X to format acceptable by gensim, removing and punctuation stopwords in the process\n",
    "X = []\n",
    "stop_words = set(nltk.corpus.stopwords.words(\"english\"))\n",
    "tokenizer = nltk.tokenize. RegexpTokenizer (r'\\w+')\n",
    "for par in data[\"text\"].values:\n",
    "    tmp = []\n",
    "    sentences = nltk.sent tokenize(par)\n",
    "    for sent in sentences:\n",
    "        sent = sent. lower ()\n",
    "        tokens = tokenizer.tokenize (sent)\n",
    "        filtered _words = [w.strip() for w in tokens if w not in stop_words and len(w) > 1]\n",
    "        tmp.extend(filtered_words)\n",
    "    X. append (tmp)\n",
    "\n",
    "    \n",
    "import gensim #pip install gensim\n",
    "#Dimension of vectors we are generating\n",
    "EMBEDDING_DIM = 100\n",
    "#Creating Word Vectors by Word2Vec Method (takes time...)\n",
    "w2v_model = gensim.models.Word2Vec(sentences=X, size=EMBEDDING_DIM, window=5, min_count=1)\n",
    "#vocab size \n",
    "len (w2v_model.wv.vocab)\n",
    "#We have now represented each of 122248 words by a 100dim vector.\n",
    "#see a sample vector for random word, lets say Corona \n",
    "print (w2v_model [\"corona\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f12550f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "print ( w2v_model.wv.most_similar(\"iran\"))\n",
    "print ( w2v_model.wv.most_similar(\"fbi\"))\n",
    "print ( w2v_model.wv.most_similar(\"facebook\"))\n",
    "print ( w2v_model.wv.most_similar(\"computer\"))\n",
    "#Feeding US Presidents\n",
    "print ( w2v_model.wv.most_similar(positive=[\"trump\", \"obama\", \"clinton\"]))\n",
    "#First was Bush\n",
    "\n",
    "\n",
    "# Tokenizing Text -> Repsesenting each word by a number\n",
    "# Mapping of orginal word to number is preserved in word_index property of tokenizer\n",
    "#Tokenized applies basic processing Like changing it yo Lower case, explicitely setting that as False\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts (X)\n",
    "X = tokenizer.texts_to_sequences (X)\n",
    "# lets check the first 10 words of first news\n",
    "#every word has been represented with a number \n",
    "print ( X[0][:10])\n",
    "#Lets check few word to numerical replesentation\n",
    "#Mapping is preserved in dictionary -> word_ index property of instance\n",
    "word_index = tokenizer.word_index\n",
    "for word, num in word_index.items ():\n",
    "    print (f\"{word} -> {num}\")\n",
    "    if num == 10:\n",
    "        break\n",
    "\n",
    "# For determining size of input.\n",
    "# Making histogram for no of words in news shows that most news\n",
    "# Lets keep each news small and truncate all news to 700 while t\n",
    "pit.hist([len(x) for × in X], bins=500)\n",
    "plt.show()\n",
    "# Its heavily skewed. There are news with 5000 words? Lets trunc\n",
    "nos = np.array([len(x) for × in X])\n",
    "len (nos [nos ‹ 7001])\n",
    "# Out of 48k news, 44k have less than 700 words\n",
    "#Lets keep all news to 700, add padding to news with less than + +aa\n",
    "maxlen = 700\n",
    "\n",
    "#Making all news of size maxlen defined above\n",
    "X = pad_sequences (X, maxlen=maxlen)\n",
    "#all news has 700 words (in numerical form now). If they had less words, they have been padded with o\n",
    "# 0 is not associated to any word, as mapping of words started from 1\n",
    "# o will also be used later, if unknows word is encountered in test set\n",
    "len (X[0])\n",
    "# Adding 1 because of reserved 0 index\n",
    "# Embedding Layer creates one more vector for \"UNKNOWN\" words, or padded words (Os). This Vector is filled with zeros\n",
    "# Thus our vocab size inceeases by 1\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "# Function to create weight matrix from word2vec gensim model\n",
    "def get_weight_matrix(model, vocab):\n",
    "    # total vocabulary size plus 0 for unknown words\n",
    "    vocab_size = len (vocab) + 1\n",
    "    # define weight matrix dimensions with all o\n",
    "    weight_matrix = np.zeros ((vocab_size, EMBEDDING_DIM))\n",
    "    # step vocab, store vectors using the Tokenizer's integer mapping\n",
    "    for word, i in vocab.items ():\n",
    "        weight_matrix[i] = model [word]\n",
    "    return weight_matrix\n",
    "\n",
    "#Getting embedding vectors from wordzvec and usings it as weights of non-trainable keras embedding layer\n",
    "embedding_vectors = get_weight_matrix(w2v_model, word_index)\n",
    "#Defining Neural Network\n",
    "model = Sequential ()\n",
    "#Non-trainable embedding Layer\n",
    "model.add(Embedding(vocab_size, output_dim=EMBEDDING_DIM, weights=[embedding_vectors], input_length=maxlen, trainable=False))\n",
    "#LSTM \n",
    "model.add(LSTM(units=128))\n",
    "model. add(Dense (1, activation='sigmoid'))\n",
    "model. compile (optimizer='adam', loss='binary_crossentropy', metrics=['acc'])\n",
    "del embedding_vectors \n",
    "print ( model.summary ())\n",
    "#Train test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "model.fit (X_train, y_train, validation_split=0.3, epochs=6)\n",
    "\n",
    "#Prediction is in probability of news being real, so converting into classes\n",
    "# Class 0 (Fake) if predicted prob ‹ 0.5, else class 1 (Real) \n",
    "y_pred = (model.predict (X_test) ›= 0.5).astype(\"int\") \n",
    "accuracy_score (y_test, y_pred)\n",
    "print (classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09a0c739",
   "metadata": {},
   "source": [
    "# Project 2\n",
    "\n",
    "# NLP for fake news detection\n",
    "\n",
    "dataset in kaggle in:\n",
    "\n",
    "https://www.kaggle.com/vibinmarish/nlp-for-fake-news-detection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96b5aced",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # Linear algebra\n",
    "import pandas as pd # data processing, CSV file I/0 (e.g. pd.read_csv)\n",
    "tn = pd.read_csv(\"True.csv\", nrows=4000)\n",
    "fn = pd.read_cs®(\"Fake.csv\", nrows=4000) \n",
    "print (fn.head ())\n",
    "tn[ 'class'] = 1\n",
    "fn['class'] = 0\n",
    "print (tn.head())\n",
    "df = pd.concat ([tn. head (4000), fn. head (4000)], ignore_index = True)\n",
    "print (df)\n",
    "f = df.sample(frac=1).reset_index(drop=True)\n",
    "print (df)\n",
    "df['News'] = df['title'] + \" \" + df['text'] + \" \" + df['subject' ]\n",
    "print (df)\n",
    "\n",
    "\n",
    "import re \n",
    "import nltk\n",
    "from nltk.corpus import stopwords \n",
    "from nitk.stem import WordNetLemmatizer\n",
    "TL = WordNetLemmatizer ()\n",
    "corpus = []\n",
    "for i in range (0,8000):\n",
    "    line = re.sub('[^a-zA-Z]', \" \", df[ 'News '][i])\n",
    "    line = line.lower ()\n",
    "    line = line.split()\n",
    "    line = [TL.lemmatize(word) for word in line if not word in set(stopwords.words('english'))]\n",
    "    line = \" \".join(line)\n",
    "    corpus.append (line)\n",
    "from sklearn. feature_extraction.text import CountVectorizer\n",
    "cv = CountVectorizer()\n",
    "X = cv.fit_transform(corpus).toarray ()\n",
    "y = df['class'].values\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.2)\n",
    "from sklearn.naive_bayes import GaussianNB \n",
    "model=GaussianNB()\n",
    "final=model.fit (X_train,y_train)\n",
    "print(\"Testing score\", final.score(X_test,y_test))\n",
    "\n",
    "import seaborn as sb \n",
    "from sklearn.metrics import confusion matrix \n",
    "from matplotlib import pyplot as plt \n",
    "svm_predicted=final.predict (X test)\n",
    "svm_confuse=confusion_matrix(y_test, svm_predicted) \n",
    "df_cm=pd.DataFrame (svm_ confuse)\n",
    "\n",
    "plt.figure(figsize=(5.5,4))\n",
    "sb. heatmap (df_cm, annot=True, fmt= 'g') \n",
    "plt.title(\"Confusion Matrix Heatmap\") \n",
    "plt.xlabel(\"True Label\") \n",
    "plt.ylabel(\"Predicted Label\") \n",
    "plt.show()\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"classification Report\")\n",
    "print(classification_report (y_test,sm_predicted))\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "model=KNeighborsClassifier (n_neighbors=2)\n",
    "final=model.fit (X_train,y_train)\n",
    "print(\"Testing score\", final.score(X_test,y_test))\n",
    "import seaborn as sb \n",
    "from sklearn.metrics import confusion_ matrix \n",
    "from matplotlib import pyplot as plt\n",
    "knn_predicted=final.predict(X_test)\n",
    "knn_confuse=confusion_matrix(y_test, kn_predicted) \n",
    "df_cm=pd.DataFrame (knn_confuse)\n",
    "\n",
    "pit.figure(figsize= (5.5,4))\n",
    "sb.heatmap (df_cm, annot=True, fmt= 'g' )  \n",
    "plt.title(\"Confusion Matrix Heatmap\") \n",
    "plt.xlabel(\"True Label\") \n",
    "plt.ylabel(\"Predicted Label\") \n",
    "plt.show()\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"Classification Report\")\n",
    "print(classification_report (y_test,knn_predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36a36126",
   "metadata": {},
   "source": [
    "# Project 3\n",
    "\n",
    "# NB and RF models (99% accuracy)\n",
    "\n",
    "dataset in kaggle in:\n",
    "\n",
    "https://www.kaggle.com/rahulvv/nb-and-rf-models-99-accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cab820d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # Linear algebra\n",
    "import pandas as pd # data processing, CSV file I/0 (e.g. pd. read_csv)\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns \n",
    "import re \n",
    "import nltk\n",
    "from nitk.corpus import stopwords \n",
    "from nltk.stem.porter import PorterStemmer \n",
    "from nltk.tokenize import word_tokenize, sent_tokenize \n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator \n",
    "import string\n",
    "from sklearn.model selection import train_test_split \n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, f1_score \n",
    "from sklearn.naive bayes import MultinomiaINB \n",
    "from sklearn. feature_extraction.text import CountVectorizer, TfidfTransformer \n",
    "from sklearn.pipeline import Pipeline \n",
    "import xgboost as xgb \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "seed = 4353\n",
    "true = pd.read _csv('True.csv\")\n",
    "fake = pd.read_csv('Fake.csv')\n",
    "# Introducing new column in both dataframes\n",
    "true ['impression']=1\n",
    "fake[ 'impression'1=0\n",
    "# Concatenating them using pandas concatenate to form a single dataframe\n",
    "data_raw = pd. concat ([true, fake], axis=0)\n",
    "# Combining title and text to obtain a single string\n",
    "# dropping title and\n",
    "data_raw['fulltext'] = data_raw.title + ' ' + data_raw.text\n",
    "data_raw.drop(['title', 'text'], axis=1, inplace=True)\n",
    "# Extracting a new dataframe using features fulltext and impression\n",
    "data = data_raw[[ 'fulltext', 'impression']]\n",
    "data = data.reset_index ()\n",
    "data.drop(['index'], axis=1, inplace=True)\n",
    "# Check for missing values\n",
    "\n",
    "print ( data.isnull ().sum())\n",
    "print('The dataset contans {} rows and {} columns'.format (data.shape [0], data.shape [1]))\n",
    "# Word extraction from true and fake texts\n",
    "true_text = data [data.impression == 1]['fulltext']\n",
    "fake_text = data[data.impression == 0]['fulltext']\n",
    "fake_text = fake_text.reset_index().drop(['index'], axis=1)\n",
    "\n",
    "# Function to extract major words from true and fake news\n",
    "def wordcloud_words(X_data_full):\n",
    "# function for removing punctuations\n",
    "    def remove_punct (X_data_func):\n",
    "        string1 = X_data_func. lower ()\n",
    "        translation_table = dict.fromkeys (map(ord, string.punctuation), \" \")\n",
    "        string2 = striag1.translate (translation_table)\n",
    "        return string2\n",
    "     \n",
    "    X_data_full_clear_punct = []\n",
    "    for i in range(len(X_data_full)):\n",
    "        test_data = remove_punct (X_data_full[i])\n",
    "        X_data_full_clear_punct.append(test_data)\n",
    "     \n",
    "    # function to remove stopwords\n",
    "    def remove_stopwords (X_data_func) :\n",
    "        pattern = re.compile(r'\\b(' + r'|'.join(stopwords.words(\"english\")) + r') \\b|s*')\n",
    "        string2 = pattern.sub(' ', X_data_func)\n",
    "        return string2\n",
    "     \n",
    "    X_data_full_clear_stopwords = []\n",
    "    for i in range(len (X_data_full)):\n",
    "        test_data = remove_stopwords (X_data_full[i])\n",
    "        X_data_full_clear_stopwords.append(test_data)\n",
    "\n",
    "    # function for tokenizing\n",
    "    def tokenize_words (X_data_func):\n",
    "        words = nitk.word_tokenize(X_data_func)\n",
    "        return words\n",
    "    X_data_full_tokenized_words = []\n",
    "    for i in range (len (X_data_full)):\n",
    "        test_data = tokenize_words (X_data_full[i])\n",
    "        X_data_full_tokenized_words.append(test_data)\n",
    "        Output\n",
    "    # function for lemmatizing\n",
    "    lemmatizer = WordNetLemmatizer ()\n",
    "    def lemmatize_words(X_data_func):\n",
    "        words = lemmatizer.lemmatize(X_data_func)\n",
    "        return words\n",
    "     X_data_full_lemmatized_words = []\n",
    "     for i in range(len (X_data_full)):\n",
    "        test_data = lemmatize_words(X_data_full[i])\n",
    "        × data full lemmatized_words.append(test_data)\n",
    "    return X_data_full_lemmatized_words\n",
    "\n",
    "true_words = wordcloud_words (true_text)\n",
    "fake_words = wordcloud_words (fake_text.fulltext)\n",
    "def plot wordcloud (text):\n",
    "    wordcloud = WordCloud (background_color='black', max_words=3000, width=1600, height=800).generate(text)\n",
    "\n",
    "    plt.clf()\n",
    "    plt.imshow(wordcloud, interpolation='bilinear') \n",
    "     plt.axis('off') \n",
    "     plt.show()\n",
    "plt.figure(figsize=(20, 18)) \n",
    "plot_wordcloud(' '.join(true_words)) \n",
    "plt.show() \n",
    "plt.figure(figsize=(20, 18)) \n",
    "plot_wordcloud(' '.join(fake_words) ) \n",
    "plt.show()\n",
    "\n",
    "# Data preparation\n",
    "X _data = data[ 'fulltext ']\n",
    "y_data = data. impression\n",
    "X_data = X_data. astype(str)\n",
    "# Function to retrieve processed words\n",
    "def final(X_data_full):\n",
    "# function for removing punctuations\n",
    "    def remove_punct (X_data_func):\n",
    "        string1 = X_data_func.lower ()\n",
    "        translation_table = dict.fromkeys (map(ord, string-punctuation), \" \")\n",
    "        string2 = string1.translate (translation_table)\n",
    "        return string2\n",
    "     X_data_full_clear_punct = []\n",
    "    for i in range(len(X_data_full)):\n",
    "        test_data = remove_punct(X_data_full[i])\n",
    "        X_data_full_clear_punct.append(test_data)\n",
    "        # function to remove stopwords\n",
    "    def remove_stopwords (X_data_func):\n",
    "        pattern = re.compile(r'\\b(' + r'|'.join(stopwords.words('english')) + r') \\b\\s*')\n",
    "        string2 = pattern. sub(' ', X_data_func)\n",
    "        return string2\n",
    "\n",
    "    X_data_full_clear_stopwords = []\n",
    "    for i in range(len(X_data_ full)):\n",
    "        test_data = remove_stopwords (X_data_ful1[i])\n",
    "        X_data_full_clear_stopwords.append(test_data)\n",
    "        # function for tokenizing\n",
    "    def tokenize_words(X_data_func):\n",
    "        words = nltk.word_tokenize(X_data_func)\n",
    "        return words\n",
    "    X_data_full_tokenized_words = []\n",
    "    for i in range(len (X_data_full)):\n",
    "        test_data = tokenize_words (X_data_full[i])\n",
    "        X_data_full_tokenized_words.append(test_data)\n",
    "        # function for lemmatizing\n",
    "     lemmatizer = WordNetLemmatizer ()\n",
    "     def lemmatize_words (X_data_func):\n",
    "        words = lemmatizer.lemmatize(X_data_func)\n",
    "        return words\n",
    "     X_data_full_lemmatized _words = []\n",
    "     for i in range(len(X_data_full)):\n",
    "        test_data = lemmatize_words(X_data_full[i])\n",
    "        X_data_full_lemmatized_words.append(test_data)\n",
    "\n",
    "        # creating the bag of words model\n",
    "        cV = CountVectorizer (max_features=1000)\n",
    "        X_data_full_vector = cv.fit_transform(X_data_full_lemmatized_words).toarray ()\n",
    "        tfidf = TfidfTransformer )\n",
    "        X_data_full_tfidf = tfidf.fit_transform(X_data_full_vector).toarray()\n",
    "        return X_data_full_tfidf\n",
    "# Setting the function with parameters\n",
    "data X = final (X data)\n",
    "# Preparing training and testing data using train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data_X, y_data, test_size=0.25, random_state=seed)\n",
    "\n",
    "# Instatiation, fitting and prediction\n",
    "MNB = MultinomialNB ()\n",
    "MNB. fit (X_train, y_train)\n",
    "predictions = MNB.predict(X test)\n",
    "# Model evaluation\n",
    "print(classification_report (_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions))\n",
    "# Instatiation, fitting and prediction\n",
    "rfc=RandomForestClassifier(n_estimators= 10, random_state= seed) \n",
    "rfc.fit(X_train, y_train)\n",
    "predictions = rfc.predict(X_test)\n",
    "# Model evaluation\n",
    "print (classification_report (y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}