vector space model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "23e4ebaf",
   "metadata": {},
   "source": [
    "### TF-IDF\n",
    "- TF (tf_ij) = f_ij / max{f_1j, f_2j, ..., f_|v|j}\n",
    "- IDF (idf_i) = log(N/df_i) # N = total number of documents ; df_i = number of documents where t_i appears\n",
    "- TF-IDF(f_ij): weight of term t_i in document d_j; the number of times t_i appears in d_j\n",
    "\n",
    "### Queries\n",
    "\n",
    "- w_iq = (0.5 + (0.5 * f_iq / max{f_1q, f_2q, ..., f_|V|q}) * log(N/df_i)\n",
    "\n",
    "### Cosine similarity\n",
    "cosine (d_j, q) = <d_j @ q> / ||d_j|| * ||q||\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b099dd56",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "32b12ca7",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_freq_matrix = pd.read_csv('vsm.csv', index_col='Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "80fe29ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>antony and cleopatra</th>\n",
       "      <th>julius caesar</th>\n",
       "      <th>the tempest</th>\n",
       "      <th>hamlet</th>\n",
       "      <th>othello</th>\n",
       "      <th>macbeth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>antony</th>\n",
       "      <td>157</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brutus</th>\n",
       "      <td>4</td>\n",
       "      <td>157</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>caeser</th>\n",
       "      <td>232</td>\n",
       "      <td>227</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calpurnia</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cleopatra</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mercy</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worser</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           antony and cleopatra  julius caesar  the tempest  hamlet  othello  \\\n",
       "antony                      157             73            0       0        0   \n",
       "brutus                        4            157            0       2        0   \n",
       "caeser                      232            227            0       2        1   \n",
       "calpurnia                     0             10            0       0        0   \n",
       "cleopatra                    57              0            0       0        0   \n",
       "mercy                         2              0            3       8        5   \n",
       "worser                        2              0            1       1        1   \n",
       "\n",
       "           macbeth  \n",
       "antony           1  \n",
       "brutus           0  \n",
       "caeser           8  \n",
       "calpurnia        0  \n",
       "cleopatra        0  \n",
       "mercy            5  \n",
       "worser           0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_freq_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f76754f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf_11 = 0.6767241379310345\n",
      "idf_11 = 1.0\n",
      "w_11 = 0.6767241379310345\n"
     ]
    }
   ],
   "source": [
    "# w4 slide 26\n",
    "# weight of 'Antony' in 'Antony and Cleopatra'\n",
    "\n",
    "f_11 = word_freq_matrix['antony and cleopatra']['antony'] \n",
    "f_max = max(word_freq_matrix['antony and cleopatra'])\n",
    "tf_11 = f_11 / f_max\n",
    "print(f'tf_11 = {tf_11}')\n",
    "# math.log(x, base)\n",
    "idf_11 = math.log(6/word_freq_matrix.loc['antony'].astype(bool).sum(), 2)\n",
    "print(f'idf_11 = {idf_11}')\n",
    "\n",
    "w_11 = tf_11 * idf_11\n",
    "print(f'w_11 = {w_11}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bae992e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# slide 27\n",
    "\n",
    "weight_matrix = dict()\n",
    "\n",
    "def calc_tfidf(term, doc):\n",
    "    freq = word_freq_matrix[doc][term]\n",
    "    freq_max = max(word_freq_matrix[doc])\n",
    "    tf = freq / freq_max\n",
    "    \n",
    "    idf = math.log(6/word_freq_matrix.loc[term].astype(bool).sum(), 2)\n",
    "    \n",
    "    return format(tf*idf, '.3f')\n",
    "\n",
    "for doc in word_freq_matrix.columns:\n",
    "    weight_matrix[doc] = list()\n",
    "    for query in word_freq_matrix.index:\n",
    "        weight_matrix[doc].append(calc_tfidf(query, doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69675b3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>antony and cleopatra</th>\n",
       "      <th>julius caesar</th>\n",
       "      <th>the tempest</th>\n",
       "      <th>hamlet</th>\n",
       "      <th>othello</th>\n",
       "      <th>macbeth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>antony</th>\n",
       "      <td>0.677</td>\n",
       "      <td>0.322</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brutus</th>\n",
       "      <td>0.017</td>\n",
       "      <td>0.692</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>caeser</th>\n",
       "      <td>0.263</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.066</td>\n",
       "      <td>0.053</td>\n",
       "      <td>0.263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calpurnia</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.114</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cleopatra</th>\n",
       "      <td>0.635</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mercy</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worser</th>\n",
       "      <td>0.005</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.195</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.117</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           antony and cleopatra  julius caesar  the tempest  hamlet  othello  \\\n",
       "antony                    0.677          0.322        0.000   0.000    0.000   \n",
       "brutus                    0.017          0.692        0.000   0.250    0.000   \n",
       "caeser                    0.263          0.263        0.000   0.066    0.053   \n",
       "calpurnia                 0.000          0.114        0.000   0.000    0.000   \n",
       "cleopatra                 0.635          0.000        0.000   0.000    0.000   \n",
       "mercy                     0.002          0.000        0.263   0.263    0.263   \n",
       "worser                    0.005          0.000        0.195   0.073    0.117   \n",
       "\n",
       "           macbeth  \n",
       "antony       0.125  \n",
       "brutus       0.000  \n",
       "caeser       0.263  \n",
       "calpurnia    0.000  \n",
       "cleopatra    0.000  \n",
       "mercy        0.164  \n",
       "worser       0.000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_weights = pd.DataFrame(data=weight_matrix, index=word_freq_matrix.index, dtype=float)\n",
    "tfidf_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f4b6f5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'brutus caeser': antony       0.50\n",
       " brutus       1.00\n",
       " caeser       0.26\n",
       " calpurnia    1.29\n",
       " cleopatra    1.29\n",
       " mercy        0.13\n",
       " worser       0.29\n",
       " dtype: float64}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# slide 28\n",
    "# calculate the weight of \"Brutus\" in the query\n",
    "'''\n",
    "query weights of 'brutus caeser' \n",
    "q = {0.5, 1, 0.26, 1.29, 1.29, 0.13, 0.29}\n",
    "  = {w_1q, w_2q, w_3q, w_4q, w_5q, w_6q}\n",
    "    * w_1q : weight of term_1 in query q\n",
    "'''\n",
    "def calc_query_weight(query): \n",
    "    f_Vq = list()\n",
    "    for term in list(word_freq_matrix.index): \n",
    "        f_Vq.append(query.count(term))    \n",
    "\n",
    "    def calc_term_weight(term):\n",
    "        f_iq = query.count(term)\n",
    "        idf = math.log(6/word_freq_matrix.loc[term].astype(bool).sum(), 2)\n",
    "        weight = (0.5 + (0.5 * f_iq) / max(f_Vq)) * idf\n",
    "        return format(weight ,'.2f')\n",
    "\n",
    "    query_weight = list()\n",
    "    for term in list(word_freq_matrix.index):\n",
    "        query_weight.append(calc_term_weight(term))\n",
    "    \n",
    "    return pd.Series(data=query_weight, dtype=float, index=word_freq_matrix.index)\n",
    "\n",
    "query_weights = dict()\n",
    "\n",
    "QUERY = 'brutus caeser'\n",
    "query_weights[QUERY] = calc_query_weight(QUERY)\n",
    "query_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ff4382ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "antony and cleopatra    0.59\n",
       "julius caesar           0.60\n",
       "the tempest             0.13\n",
       "hamlet                  0.39\n",
       "othello                 0.13\n",
       "macbeth                 0.21\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# slide 28 'Similarity scores'\n",
    "# calculate similarity scores based on cosine similarity between \n",
    "# the query 'Brutus Caeser'(q) and the document 'Antony and Cleopatra'(d_j)\n",
    "\n",
    "cosine_similarities = list()\n",
    "for document in word_freq_matrix.columns:\n",
    "    w_ij = tfidf_weights[document]\n",
    "    w_iq = query_weights[QUERY] \n",
    "\n",
    "    mul_dj_q = w_iq.mul(w_ij).sum()\n",
    "    mod_dj = math.sqrt(np.square(w_ij).sum())\n",
    "    mod_q = math.sqrt(np.square(w_iq).sum())\n",
    "    cos_sim = mul_dj_q / (mod_dj * mod_q)\n",
    "    cosine_similarities.append(format(cos_sim, '.2f'))\n",
    "    \n",
    "similarity_scores = pd.Series(cosine_similarities, index=word_freq_matrix.columns)\n",
    "similarity_scores"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ids2020] *",
   "language": "python",
   "name": "conda-env-ids2020-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}