ComparingTextDocuments_TfidfCosim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9fe5bfd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\anaconda3\\lib\\site-packages\\scipy\\__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.26.0\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import multiprocessing as mp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "import nltk\n",
    "nltk.download('stopwords', quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6e5e78b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    # lowercasing\n",
    "    lowercased_text = text.lower()\n",
    "\n",
    "    # cleaning \n",
    "    import re \n",
    "    remove_punctuation = re.sub(r'[^\\w\\s]', '', lowercased_text)\n",
    "    remove_white_space = remove_punctuation.strip()\n",
    "\n",
    "    # Tokenization = Breaking down each sentence into an array\n",
    "    from nltk.tokenize import word_tokenize\n",
    "    tokenized_text = word_tokenize(remove_white_space)\n",
    "\n",
    "    # Stop Words/filtering = Removing irrelevant words\n",
    "    from nltk.corpus import stopwords\n",
    "    stopwords = set(stopwords.words('english'))\n",
    "    stopwords_removed = [word for word in tokenized_text if word not in stopwords]\n",
    "\n",
    "    # Stemming = Transforming words into their base form\n",
    "    from nltk.stem import PorterStemmer\n",
    "    ps = PorterStemmer()\n",
    "    stemmed_text = [ps.stem(word) for word in stopwords_removed]\n",
    "    \n",
    "    # Putting all the results into a dataframe.\n",
    "    df = pd.DataFrame({\n",
    "        'DOCUMENT': [text],\n",
    "        'LOWERCASE' : [lowercased_text],\n",
    "        'CLEANING': [remove_white_space],\n",
    "        'TOKENIZATION': [tokenized_text],\n",
    "        'STOP-WORDS': [stopwords_removed],\n",
    "        'STEMMING': [stemmed_text]\n",
    "    })\n",
    "\n",
    "    return df\n",
    "        \n",
    "def calculate_tfidf(corpus):\n",
    "    # Call the preprocessing result\n",
    "    df = preprocessing(corpus)\n",
    "        \n",
    "    # Make each array row from stopwords_removed to be a sentence\n",
    "    stemming = corpus['STEMMING'].apply(' '.join)\n",
    "    \n",
    "    # Count TF-IDF\n",
    "    from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "    vectorizer = TfidfVectorizer()\n",
    "    tfidf_matrix = vectorizer.fit_transform(stemming)\n",
    "    \n",
    "    # Get words from stopwords array to use as headers\n",
    "    feature_names = vectorizer.get_feature_names_out()\n",
    "\n",
    "    # Combine header titles and weights\n",
    "    df_tfidf = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names)\n",
    "    df_tfidf = pd.concat([df, df_tfidf], axis=1)\n",
    "\n",
    "    return df_tfidf\n",
    "\n",
    "def cosineSimilarity(corpus):\n",
    "    # Call the TF-IDF result\n",
    "    df_tfidf = calculate_tfidf(corpus)\n",
    "    \n",
    "    # Get the TF-IDF vector for the first item (index 0)\n",
    "    vector1 = df_tfidf.iloc[0, 6:].values.reshape(1, -1)\n",
    "\n",
    "    # Get the TF-IDF vector for all items except the first item\n",
    "    vectors = df_tfidf.iloc[:, 6:].values\n",
    "    \n",
    "    # Calculate cosine similarity between the first item and all other items\n",
    "    from sklearn.metrics.pairwise import cosine_similarity\n",
    "    cosim = cosine_similarity(vector1, vectors)\n",
    "    cosim = pd.DataFrame(cosim)\n",
    "    \n",
    "    # Convert the DataFrame into a one-dimensional array\n",
    "    cosim = cosim.values.flatten()\n",
    "\n",
    "    # Convert the cosine similarity result into a DataFrame\n",
    "    df_cosim = pd.DataFrame(cosim, columns=['COSIM'])\n",
    "\n",
    "    # Combine the TF-IDF array with the cosine similarity result\n",
    "    df_cosim = pd.concat([df_tfidf, df_cosim], axis=1)\n",
    "\n",
    "    return df_cosim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a6e4d40f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOCUMENT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                             DOCUMENT\n",
       "0                            Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.\n",
       "1                              Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.\n",
       "2                                             I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.\n",
       "3  Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# file semua judul proposal pada prodi sistem informasi UINSU Medan\n",
    "data = pd.read_csv('data.csv', delimiter=';', encoding='latin')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2a2c586e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOCUMENT</th>\n",
       "      <th>LOWERCASE</th>\n",
       "      <th>CLEANING</th>\n",
       "      <th>TOKENIZATION</th>\n",
       "      <th>STOP-WORDS</th>\n",
       "      <th>STEMMING</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day</td>\n",
       "      <td>[being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]</td>\n",
       "      <td>[sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]</td>\n",
       "      <td>[sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart</td>\n",
       "      <td>[playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]</td>\n",
       "      <td>[playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]</td>\n",
       "      <td>[play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring</td>\n",
       "      <td>[i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]</td>\n",
       "      <td>[really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]</td>\n",
       "      <td>[realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle</td>\n",
       "      <td>[whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]</td>\n",
       "      <td>[whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]</td>\n",
       "      <td>[whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                             DOCUMENT  \\\n",
       "0                            Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.   \n",
       "2                                             I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.   \n",
       "3  Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                            LOWERCASE  \\\n",
       "0                            being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.   \n",
       "2                                             i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.   \n",
       "3  whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                          CLEANING  \\\n",
       "0                              being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day   \n",
       "1                              playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart   \n",
       "2                                              i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring   \n",
       "3  whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle   \n",
       "\n",
       "                                                                                                                                                                                  TOKENIZATION  \\\n",
       "0                                  [being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]   \n",
       "1                                  [playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]   \n",
       "2                                                    [i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]   \n",
       "3  [whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                                              STOP-WORDS  \\\n",
       "0                                             [sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]   \n",
       "1                                   [playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]   \n",
       "2                              [really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]   \n",
       "3  [whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                          STEMMING  \n",
       "0                                    [sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]  \n",
       "1                             [play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]  \n",
       "2                       [realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]  \n",
       "3  [whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def preprocessing(corpus):\n",
    "    # Create an empty DataFrame\n",
    "    df = pd.DataFrame(columns=['DOCUMENT'])\n",
    "\n",
    "    # Running preprocessing one by one\n",
    "    for doc in corpus['DOCUMENT']:\n",
    "        # Call the preprocess_text function\n",
    "        result_df = preprocess_text(doc)\n",
    "        \n",
    "        # Concatenate the result of preprocessing to the main DataFrame\n",
    "        df = pd.concat([df, result_df], ignore_index=True)\n",
    "        \n",
    "    return df\n",
    "\n",
    "result_preprocessing = preprocessing(data)\n",
    "result_preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e489ffef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOCUMENT</th>\n",
       "      <th>LOWERCASE</th>\n",
       "      <th>CLEANING</th>\n",
       "      <th>TOKENIZATION</th>\n",
       "      <th>STOP-WORDS</th>\n",
       "      <th>STEMMING</th>\n",
       "      <th>achiev</th>\n",
       "      <th>activ</th>\n",
       "      <th>bring</th>\n",
       "      <th>buzz</th>\n",
       "      <th>...</th>\n",
       "      <th>realli</th>\n",
       "      <th>sens</th>\n",
       "      <th>sing</th>\n",
       "      <th>speak</th>\n",
       "      <th>sport</th>\n",
       "      <th>stay</th>\n",
       "      <th>stori</th>\n",
       "      <th>touch</th>\n",
       "      <th>watch</th>\n",
       "      <th>whether</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day</td>\n",
       "      <td>[being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]</td>\n",
       "      <td>[sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]</td>\n",
       "      <td>[sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart</td>\n",
       "      <td>[playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]</td>\n",
       "      <td>[playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]</td>\n",
       "      <td>[play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.301115</td>\n",
       "      <td>0.301115</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring</td>\n",
       "      <td>[i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]</td>\n",
       "      <td>[really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]</td>\n",
       "      <td>[realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.230982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle</td>\n",
       "      <td>[whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]</td>\n",
       "      <td>[whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]</td>\n",
       "      <td>[whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.511775</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255888</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.201745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                             DOCUMENT  \\\n",
       "0                            Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.   \n",
       "2                                             I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.   \n",
       "3  Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                            LOWERCASE  \\\n",
       "0                            being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.   \n",
       "2                                             i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.   \n",
       "3  whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                          CLEANING  \\\n",
       "0                              being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day   \n",
       "1                              playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart   \n",
       "2                                              i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring   \n",
       "3  whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle   \n",
       "\n",
       "                                                                                                                                                                                  TOKENIZATION  \\\n",
       "0                                  [being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]   \n",
       "1                                  [playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]   \n",
       "2                                                    [i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]   \n",
       "3  [whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                                              STOP-WORDS  \\\n",
       "0                                             [sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]   \n",
       "1                                   [playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]   \n",
       "2                              [really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]   \n",
       "3  [whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                          STEMMING  \\\n",
       "0                                    [sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]   \n",
       "1                             [play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]   \n",
       "2                       [realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]   \n",
       "3  [whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]   \n",
       "\n",
       "     achiev     activ     bring      buzz  ...    realli      sens      sing  \\\n",
       "0  0.328919  0.000000  0.000000  0.000000  ...  0.000000  0.328919  0.000000   \n",
       "1  0.000000  0.000000  0.000000  0.000000  ...  0.000000  0.000000  0.301115   \n",
       "2  0.000000  0.000000  0.292972  0.000000  ...  0.292972  0.000000  0.000000   \n",
       "3  0.000000  0.511775  0.000000  0.255888  ...  0.000000  0.000000  0.000000   \n",
       "\n",
       "      speak     sport      stay     stori     touch     watch   whether  \n",
       "0  0.000000  0.328919  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "1  0.301115  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "2  0.000000  0.000000  0.000000  0.292972  0.292972  0.292972  0.230982  \n",
       "3  0.000000  0.000000  0.255888  0.000000  0.000000  0.000000  0.201745  \n",
       "\n",
       "[4 rows x 49 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_tfidf = calculate_tfidf(result_preprocessing)\n",
    "result_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c7cb9400",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOCUMENT</th>\n",
       "      <th>LOWERCASE</th>\n",
       "      <th>CLEANING</th>\n",
       "      <th>TOKENIZATION</th>\n",
       "      <th>STOP-WORDS</th>\n",
       "      <th>STEMMING</th>\n",
       "      <th>achiev</th>\n",
       "      <th>activ</th>\n",
       "      <th>bring</th>\n",
       "      <th>buzz</th>\n",
       "      <th>...</th>\n",
       "      <th>sens</th>\n",
       "      <th>sing</th>\n",
       "      <th>speak</th>\n",
       "      <th>sport</th>\n",
       "      <th>stay</th>\n",
       "      <th>stori</th>\n",
       "      <th>touch</th>\n",
       "      <th>watch</th>\n",
       "      <th>whether</th>\n",
       "      <th>COSIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.</td>\n",
       "      <td>being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day</td>\n",
       "      <td>[being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]</td>\n",
       "      <td>[sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]</td>\n",
       "      <td>[sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.328919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.</td>\n",
       "      <td>playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart</td>\n",
       "      <td>[playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]</td>\n",
       "      <td>[playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]</td>\n",
       "      <td>[play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.301115</td>\n",
       "      <td>0.301115</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.</td>\n",
       "      <td>i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring</td>\n",
       "      <td>[i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]</td>\n",
       "      <td>[really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]</td>\n",
       "      <td>[realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.292972</td>\n",
       "      <td>0.230982</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.</td>\n",
       "      <td>whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle</td>\n",
       "      <td>[whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]</td>\n",
       "      <td>[whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]</td>\n",
       "      <td>[whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.511775</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255888</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255888</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.201745</td>\n",
       "      <td>0.104634</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                             DOCUMENT  \\\n",
       "0                            Being into sports is more than just exercising. It's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              Playing the guitar and singing helps me express my feelings. It's like creating my own language through music that speaks to my heart.   \n",
       "2                                             I really like watching movies. Whether it's a funny one or a touching story, I enjoy the different feelings they bring.   \n",
       "3  Whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                            LOWERCASE  \\\n",
       "0                            being into sports is more than just exercising. it's a lifestyle that gives me energy, discipline, and a sense of achievement every day.   \n",
       "1                              playing the guitar and singing helps me express my feelings. it's like creating my own language through music that speaks to my heart.   \n",
       "2                                             i really like watching movies. whether it's a funny one or a touching story, i enjoy the different feelings they bring.   \n",
       "3  whether it's hitting the gym or doing outdoor activities, staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle.   \n",
       "\n",
       "                                                                                                                                                          CLEANING  \\\n",
       "0                              being into sports is more than just exercising its a lifestyle that gives me energy discipline and a sense of achievement every day   \n",
       "1                              playing the guitar and singing helps me express my feelings its like creating my own language through music that speaks to my heart   \n",
       "2                                              i really like watching movies whether its a funny one or a touching story i enjoy the different feelings they bring   \n",
       "3  whether its hitting the gym or doing outdoor activities staying active and feeling that exercise buzz is a crucial part of my commitment to a healthy lifestyle   \n",
       "\n",
       "                                                                                                                                                                                  TOKENIZATION  \\\n",
       "0                                  [being, into, sports, is, more, than, just, exercising, its, a, lifestyle, that, gives, me, energy, discipline, and, a, sense, of, achievement, every, day]   \n",
       "1                                  [playing, the, guitar, and, singing, helps, me, express, my, feelings, its, like, creating, my, own, language, through, music, that, speaks, to, my, heart]   \n",
       "2                                                    [i, really, like, watching, movies, whether, its, a, funny, one, or, a, touching, story, i, enjoy, the, different, feelings, they, bring]   \n",
       "3  [whether, its, hitting, the, gym, or, doing, outdoor, activities, staying, active, and, feeling, that, exercise, buzz, is, a, crucial, part, of, my, commitment, to, a, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                                              STOP-WORDS  \\\n",
       "0                                             [sports, exercising, lifestyle, gives, energy, discipline, sense, achievement, every, day]   \n",
       "1                                   [playing, guitar, singing, helps, express, feelings, like, creating, language, music, speaks, heart]   \n",
       "2                              [really, like, watching, movies, whether, funny, one, touching, story, enjoy, different, feelings, bring]   \n",
       "3  [whether, hitting, gym, outdoor, activities, staying, active, feeling, exercise, buzz, crucial, part, commitment, healthy, lifestyle]   \n",
       "\n",
       "                                                                                                          STEMMING  \\\n",
       "0                                    [sport, exercis, lifestyl, give, energi, disciplin, sens, achiev, everi, day]   \n",
       "1                             [play, guitar, sing, help, express, feel, like, creat, languag, music, speak, heart]   \n",
       "2                       [realli, like, watch, movi, whether, funni, one, touch, stori, enjoy, differ, feel, bring]   \n",
       "3  [whether, hit, gym, outdoor, activ, stay, activ, feel, exercis, buzz, crucial, part, commit, healthi, lifestyl]   \n",
       "\n",
       "     achiev     activ     bring      buzz  ...      sens      sing     speak  \\\n",
       "0  0.328919  0.000000  0.000000  0.000000  ...  0.328919  0.000000  0.000000   \n",
       "1  0.000000  0.000000  0.000000  0.000000  ...  0.000000  0.301115  0.301115   \n",
       "2  0.000000  0.000000  0.292972  0.000000  ...  0.000000  0.000000  0.000000   \n",
       "3  0.000000  0.511775  0.000000  0.255888  ...  0.000000  0.000000  0.000000   \n",
       "\n",
       "      sport      stay     stori     touch     watch   whether     COSIM  \n",
       "0  0.328919  0.000000  0.000000  0.000000  0.000000  0.000000  1.000000  \n",
       "1  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000  \n",
       "2  0.000000  0.000000  0.292972  0.292972  0.292972  0.230982  0.000000  \n",
       "3  0.000000  0.255888  0.000000  0.000000  0.000000  0.201745  0.104634  \n",
       "\n",
       "[4 rows x 50 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosim_result = cosineSimilarity(result_tfidf)\n",
    "cosim_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7cafa5f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c10aa1e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "857b61a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}