SongRecommender.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Item-Item based song recommender"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "-------------\n",
    "Dataset used:\n",
    "-------------\n",
    "Million Songs Dataset\n",
    "Source: http://labrosa.ee.columbia.edu/millionsong/\n",
    "Paper: http://ismir2011.ismir.net/papers/OS6-1.pdf\n",
    "\n",
    "The current notebook uses a subset of the above data containing 10,000 songs obtained from:\n",
    "https://github.com/turi-code/tutorials/blob/master/notebooks/recsys_rank_10K_song.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aryan/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import time\n",
    "from sklearn.externals import joblib\n",
    "import Recommenders as Recommenders\n",
    "import Evaluation as Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load music data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read userid-songid-listen_count triplets\n",
    "#This step might take time to download data from external sources\n",
    "triplets_file = 'https://static.turi.com/datasets/millionsong/10000.txt'\n",
    "songs_metadata_file = 'https://static.turi.com/datasets/millionsong/song_data.csv'\n",
    "\n",
    "song_df_1 = pandas.read_table(triplets_file,header=None)\n",
    "song_df_1.columns = ['user_id', 'song_id', 'listen_count']\n",
    "\n",
    "#Read song  metadata\n",
    "song_df_2 =  pandas.read_csv(songs_metadata_file)\n",
    "\n",
    "#Merge the two dataframes above to create input dataframe for recommender systems\n",
    "song_df = pandas.merge(song_df_1, song_df_2.drop_duplicates(['song_id']), on=\"song_id\", how=\"left\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explore data\n",
    "\n",
    "Music data shows how many times a user listened to a song, as well as the details of the song."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song_id</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOAKIMP12A8C130995</td>\n",
       "      <td>1</td>\n",
       "      <td>The Cove</td>\n",
       "      <td>Thicker Than Water</td>\n",
       "      <td>Jack Johnson</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBBMDR12A8C13253B</td>\n",
       "      <td>2</td>\n",
       "      <td>Entre Dos Aguas</td>\n",
       "      <td>Flamenco Para Niños</td>\n",
       "      <td>Paco De Lucia</td>\n",
       "      <td>1976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBXHDL12A81C204C0</td>\n",
       "      <td>1</td>\n",
       "      <td>Stronger</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Kanye West</td>\n",
       "      <td>2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBYHAJ12A6701BF1D</td>\n",
       "      <td>1</td>\n",
       "      <td>Constellations</td>\n",
       "      <td>In Between Dreams</td>\n",
       "      <td>Jack Johnson</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SODACBL12A8C13C273</td>\n",
       "      <td>1</td>\n",
       "      <td>Learn To Fly</td>\n",
       "      <td>There Is Nothing Left To Lose</td>\n",
       "      <td>Foo Fighters</td>\n",
       "      <td>1999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    user_id             song_id  listen_count  \\\n",
       "0  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOAKIMP12A8C130995             1   \n",
       "1  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBBMDR12A8C13253B             2   \n",
       "2  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBXHDL12A81C204C0             1   \n",
       "3  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBYHAJ12A6701BF1D             1   \n",
       "4  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SODACBL12A8C13C273             1   \n",
       "\n",
       "             title                        release    artist_name  year  \n",
       "0         The Cove             Thicker Than Water   Jack Johnson     0  \n",
       "1  Entre Dos Aguas            Flamenco Para Niños  Paco De Lucia  1976  \n",
       "2         Stronger                     Graduation     Kanye West  2007  \n",
       "3   Constellations              In Between Dreams   Jack Johnson  2005  \n",
       "4     Learn To Fly  There Is Nothing Left To Lose   Foo Fighters  1999  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "song_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Length of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2000000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(song_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a subset of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song_id</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>title</th>\n",
       "      <th>release</th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year</th>\n",
       "      <th>song</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOAKIMP12A8C130995</td>\n",
       "      <td>1</td>\n",
       "      <td>The Cove</td>\n",
       "      <td>Thicker Than Water</td>\n",
       "      <td>Jack Johnson</td>\n",
       "      <td>0</td>\n",
       "      <td>The Cove - Jack Johnson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBBMDR12A8C13253B</td>\n",
       "      <td>2</td>\n",
       "      <td>Entre Dos Aguas</td>\n",
       "      <td>Flamenco Para Niños</td>\n",
       "      <td>Paco De Lucia</td>\n",
       "      <td>1976</td>\n",
       "      <td>Entre Dos Aguas - Paco De Lucia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBXHDL12A81C204C0</td>\n",
       "      <td>1</td>\n",
       "      <td>Stronger</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Kanye West</td>\n",
       "      <td>2007</td>\n",
       "      <td>Stronger - Kanye West</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SOBYHAJ12A6701BF1D</td>\n",
       "      <td>1</td>\n",
       "      <td>Constellations</td>\n",
       "      <td>In Between Dreams</td>\n",
       "      <td>Jack Johnson</td>\n",
       "      <td>2005</td>\n",
       "      <td>Constellations - Jack Johnson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b80344d063b5ccb3212f76538f3d9e43d87dca9e</td>\n",
       "      <td>SODACBL12A8C13C273</td>\n",
       "      <td>1</td>\n",
       "      <td>Learn To Fly</td>\n",
       "      <td>There Is Nothing Left To Lose</td>\n",
       "      <td>Foo Fighters</td>\n",
       "      <td>1999</td>\n",
       "      <td>Learn To Fly - Foo Fighters</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    user_id             song_id  listen_count  \\\n",
       "0  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOAKIMP12A8C130995             1   \n",
       "1  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBBMDR12A8C13253B             2   \n",
       "2  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBXHDL12A81C204C0             1   \n",
       "3  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SOBYHAJ12A6701BF1D             1   \n",
       "4  b80344d063b5ccb3212f76538f3d9e43d87dca9e  SODACBL12A8C13C273             1   \n",
       "\n",
       "             title                        release    artist_name  year  \\\n",
       "0         The Cove             Thicker Than Water   Jack Johnson     0   \n",
       "1  Entre Dos Aguas            Flamenco Para Niños  Paco De Lucia  1976   \n",
       "2         Stronger                     Graduation     Kanye West  2007   \n",
       "3   Constellations              In Between Dreams   Jack Johnson  2005   \n",
       "4     Learn To Fly  There Is Nothing Left To Lose   Foo Fighters  1999   \n",
       "\n",
       "                              song  \n",
       "0          The Cove - Jack Johnson  \n",
       "1  Entre Dos Aguas - Paco De Lucia  \n",
       "2            Stronger - Kanye West  \n",
       "3    Constellations - Jack Johnson  \n",
       "4      Learn To Fly - Foo Fighters  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "song_df = song_df.head(10000)\n",
    "\n",
    "#Merge song title and artist_name columns to make a new merged column\n",
    "song_df['song'] = song_df['title'].map(str) + \" - \" + song_df['artist_name']\n",
    "song_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Showing the most popular songs in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>song</th>\n",
       "      <th>listen_count</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3660</th>\n",
       "      <td>Sehr kosmisch - Harmonia</td>\n",
       "      <td>45</td>\n",
       "      <td>0.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4678</th>\n",
       "      <td>Undo - Björk</td>\n",
       "      <td>32</td>\n",
       "      <td>0.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5105</th>\n",
       "      <td>You're The One - Dwight Yoakam</td>\n",
       "      <td>32</td>\n",
       "      <td>0.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1071</th>\n",
       "      <td>Dog Days Are Over (Radio Edit) - Florence + Th...</td>\n",
       "      <td>28</td>\n",
       "      <td>0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3655</th>\n",
       "      <td>Secrets - OneRepublic</td>\n",
       "      <td>28</td>\n",
       "      <td>0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5139</th>\n",
       "      <td>high fives - Four Tet</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5140</th>\n",
       "      <td>in white rooms - Booka Shade</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5143</th>\n",
       "      <td>paranoid android - Christopher O'Riley</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5149</th>\n",
       "      <td>¿Lo Ves? [Piano Y Voz] - Alejandro Sanz</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5150</th>\n",
       "      <td>Época - Gotan Project</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5151 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   song  listen_count  \\\n",
       "3660                           Sehr kosmisch - Harmonia            45   \n",
       "4678                                       Undo - Björk            32   \n",
       "5105                     You're The One - Dwight Yoakam            32   \n",
       "1071  Dog Days Are Over (Radio Edit) - Florence + Th...            28   \n",
       "3655                              Secrets - OneRepublic            28   \n",
       "...                                                 ...           ...   \n",
       "5139                              high fives - Four Tet             1   \n",
       "5140                       in white rooms - Booka Shade             1   \n",
       "5143             paranoid android - Christopher O'Riley             1   \n",
       "5149            ¿Lo Ves? [Piano Y Voz] - Alejandro Sanz             1   \n",
       "5150                              Época - Gotan Project             1   \n",
       "\n",
       "      percentage  \n",
       "3660        0.45  \n",
       "4678        0.32  \n",
       "5105        0.32  \n",
       "1071        0.28  \n",
       "3655        0.28  \n",
       "...          ...  \n",
       "5139        0.01  \n",
       "5140        0.01  \n",
       "5143        0.01  \n",
       "5149        0.01  \n",
       "5150        0.01  \n",
       "\n",
       "[5151 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#group by song and mention their listen counts\n",
    "song_grouped = song_df.groupby(['song']).agg({'listen_count': 'count'}).reset_index()\n",
    "# calculate total listen count of all songs summed together\n",
    "grouped_sum = song_grouped['listen_count'].sum()\n",
    "#calculate percentage\n",
    "song_grouped['percentage']  = song_grouped['listen_count'].div(grouped_sum)*100\n",
    "#sort songs by listen count\n",
    "song_grouped.sort_values(['listen_count', 'song'], ascending = [0,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count number of unique users in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = song_df['user_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "365"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Quiz 1. Count the number of unique songs in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5151"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###Fill in the code here\n",
    "songs = song_df['song'].unique()\n",
    "len(songs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a song recommender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                       user_id             song_id  \\\n",
      "7389  94d5bdc37683950e90c56c9b32721edb5d347600  SOXNZOW12AB017F756   \n",
      "\n",
      "      listen_count             title         release artist_name  year  \\\n",
      "7389             2  Half Of My Heart  Battle Studies  John Mayer     0   \n",
      "\n",
      "                               song  \n",
      "7389  Half Of My Heart - John Mayer  \n"
     ]
    }
   ],
   "source": [
    "train_data, test_data = train_test_split(song_df, test_size = 0.20, random_state=0)\n",
    "print(train_data.head(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple popularity-based recommender class (Can be used as a black box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Recommenders.popularity_recommender_py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create an instance of popularity based recommender class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pm = Recommenders.popularity_recommender_py()\n",
    "pm.create(train_data, 'user_id', 'song')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the popularity model to make some predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>Rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3194</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Sehr kosmisch - Harmonia</td>\n",
       "      <td>37</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4083</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Undo - Björk</td>\n",
       "      <td>27</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>931</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Dog Days Are Over (Radio Edit) - Florence + Th...</td>\n",
       "      <td>24</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4443</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>You're The One - Dwight Yoakam</td>\n",
       "      <td>24</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3034</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Revelry - Kings Of Leon</td>\n",
       "      <td>21</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3189</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Secrets - OneRepublic</td>\n",
       "      <td>21</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4112</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Use Somebody - Kings Of Leon</td>\n",
       "      <td>21</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1207</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Fireflies - Charttraxx Karaoke</td>\n",
       "      <td>20</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1577</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Hey_ Soul Sister - Train</td>\n",
       "      <td>19</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1626</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Horn Concerto No. 4 in E flat K495: II. Romanc...</td>\n",
       "      <td>19</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user_id  \\\n",
       "3194  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "4083  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "931   4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "4443  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "3034  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "3189  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "4112  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "1207  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "1577  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "1626  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "\n",
       "                                                   song  score  Rank  \n",
       "3194                           Sehr kosmisch - Harmonia     37   1.0  \n",
       "4083                                       Undo - Björk     27   2.0  \n",
       "931   Dog Days Are Over (Radio Edit) - Florence + Th...     24   3.0  \n",
       "4443                     You're The One - Dwight Yoakam     24   4.0  \n",
       "3034                            Revelry - Kings Of Leon     21   5.0  \n",
       "3189                              Secrets - OneRepublic     21   6.0  \n",
       "4112                       Use Somebody - Kings Of Leon     21   7.0  \n",
       "1207                     Fireflies - Charttraxx Karaoke     20   8.0  \n",
       "1577                           Hey_ Soul Sister - Train     19   9.0  \n",
       "1626  Horn Concerto No. 4 in E flat K495: II. Romanc...     19  10.0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# mention user number\n",
    "user_id = users[5]\n",
    "pm.recommend(user_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quiz 2: Use the popularity based model to make predictions for the following user id (Note the difference in recommendations from the first user id)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>Rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3194</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Sehr kosmisch - Harmonia</td>\n",
       "      <td>37</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4083</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Undo - Björk</td>\n",
       "      <td>27</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>931</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Dog Days Are Over (Radio Edit) - Florence + Th...</td>\n",
       "      <td>24</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4443</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>You're The One - Dwight Yoakam</td>\n",
       "      <td>24</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3034</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Revelry - Kings Of Leon</td>\n",
       "      <td>21</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3189</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Secrets - OneRepublic</td>\n",
       "      <td>21</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4112</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Use Somebody - Kings Of Leon</td>\n",
       "      <td>21</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1207</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Fireflies - Charttraxx Karaoke</td>\n",
       "      <td>20</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1577</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Hey_ Soul Sister - Train</td>\n",
       "      <td>19</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1626</th>\n",
       "      <td>9bb911319fbc04f01755814cb5edb21df3d1a336</td>\n",
       "      <td>Horn Concerto No. 4 in E flat K495: II. Romanc...</td>\n",
       "      <td>19</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user_id  \\\n",
       "3194  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "4083  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "931   9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "4443  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "3034  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "3189  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "4112  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "1207  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "1577  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "1626  9bb911319fbc04f01755814cb5edb21df3d1a336   \n",
       "\n",
       "                                                   song  score  Rank  \n",
       "3194                           Sehr kosmisch - Harmonia     37   1.0  \n",
       "4083                                       Undo - Björk     27   2.0  \n",
       "931   Dog Days Are Over (Radio Edit) - Florence + Th...     24   3.0  \n",
       "4443                     You're The One - Dwight Yoakam     24   4.0  \n",
       "3034                            Revelry - Kings Of Leon     21   5.0  \n",
       "3189                              Secrets - OneRepublic     21   6.0  \n",
       "4112                       Use Somebody - Kings Of Leon     21   7.0  \n",
       "1207                     Fireflies - Charttraxx Karaoke     20   8.0  \n",
       "1577                           Hey_ Soul Sister - Train     19   9.0  \n",
       "1626  Horn Concerto No. 4 in E flat K495: II. Romanc...     19  10.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###Fill in the code here\n",
    "user_id = users[8]\n",
    "pm.recommend(user_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a song recommender with personalization\n",
    "\n",
    "We now create an item similarity based collaborative filtering model that allows us to make personalized recommendations to each user. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class for an item similarity based personalized recommender system "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Recommenders.item_similarity_recommender_py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create an instance of item similarity based recommender class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_model = Recommenders.item_similarity_recommender_py()\n",
    "is_model.create(train_data, 'user_id', 'song')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the personalized model to make some song recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------\n",
      "Training data songs for the user userid: 4bd88bfb25263a75bbdd467e74018f4ae570e5df:\n",
      "------------------------------------------------------------------------------------\n",
      "Just Lose It - Eminem\n",
      "Without Me - Eminem\n",
      "16 Candles - The Crests\n",
      "Speechless - Lady GaGa\n",
      "Push It - Salt-N-Pepa\n",
      "Ghosts 'n' Stuff (Original Instrumental Mix) - Deadmau5\n",
      "Say My Name - Destiny's Child\n",
      "My Dad's Gone Crazy - Eminem / Hailie Jade\n",
      "The Real Slim Shady - Eminem\n",
      "Somebody To Love - Justin Bieber\n",
      "Forgive Me - Leona Lewis\n",
      "Missing You - John Waite\n",
      "Ya Nada Queda - Kudai\n",
      "----------------------------------------------------------------------\n",
      "Recommendation process going on:\n",
      "----------------------------------------------------------------------\n",
      "No. of unique songs for the user: 13\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :2097\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Superman - Eminem / Dina Rae</td>\n",
       "      <td>0.088692</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Mockingbird - Eminem</td>\n",
       "      <td>0.067663</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>I'm Back - Eminem</td>\n",
       "      <td>0.065385</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>U Smile - Justin Bieber</td>\n",
       "      <td>0.064525</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Here Without You - 3 Doors Down</td>\n",
       "      <td>0.062293</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Hellbound - J-Black &amp; Masta Ace</td>\n",
       "      <td>0.055769</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>The Seed (2.0) - The Roots / Cody Chestnutt</td>\n",
       "      <td>0.052564</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>I'm The One Who Understands (Edit Version) - War</td>\n",
       "      <td>0.052564</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Falling - Iration</td>\n",
       "      <td>0.052564</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4bd88bfb25263a75bbdd467e74018f4ae570e5df</td>\n",
       "      <td>Armed And Ready (2009 Digital Remaster) - The ...</td>\n",
       "      <td>0.052564</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    user_id  \\\n",
       "0  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "1  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "2  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "3  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "4  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "5  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "6  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "7  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "8  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "9  4bd88bfb25263a75bbdd467e74018f4ae570e5df   \n",
       "\n",
       "                                                song     score rank  \n",
       "0                       Superman - Eminem / Dina Rae  0.088692    1  \n",
       "1                               Mockingbird - Eminem  0.067663    2  \n",
       "2                                  I'm Back - Eminem  0.065385    3  \n",
       "3                            U Smile - Justin Bieber  0.064525    4  \n",
       "4                    Here Without You - 3 Doors Down  0.062293    5  \n",
       "5                    Hellbound - J-Black & Masta Ace  0.055769    6  \n",
       "6        The Seed (2.0) - The Roots / Cody Chestnutt  0.052564    7  \n",
       "7   I'm The One Who Understands (Edit Version) - War  0.052564    8  \n",
       "8                                  Falling - Iration  0.052564    9  \n",
       "9  Armed And Ready (2009 Digital Remaster) - The ...  0.052564   10  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Print the songs for the user in training data\n",
    "user_id = users[5]\n",
    "user_items = is_model.get_user_items(user_id)\n",
    "#\n",
    "print(\"------------------------------------------------------------------------------------\")\n",
    "print(\"Training data songs for the user userid: %s:\" % user_id)\n",
    "print(\"------------------------------------------------------------------------------------\")\n",
    "\n",
    "for user_item in user_items:\n",
    "    print(user_item)\n",
    "\n",
    "print(\"----------------------------------------------------------------------\")\n",
    "print(\"Recommendation process going on:\")\n",
    "print(\"----------------------------------------------------------------------\")\n",
    "\n",
    "#Recommend songs for the user using personalized model\n",
    "is_model.recommend(user_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quiz 3. Use the personalized model to make recommendations for the following user id. (Note the difference in recommendations from the first user id.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------\n",
      "Training data songs for the user userid: 9d6f0ead607ac2a6c2460e4d14fb439a146b7dec:\n",
      "------------------------------------------------------------------------------------\n",
      "Swallowed In The Sea - Coldplay\n",
      "Life In Technicolor ii - Coldplay\n",
      "Life In Technicolor - Coldplay\n",
      "The Scientist - Coldplay\n",
      "Trouble - Coldplay\n",
      "Strawberry Swing - Coldplay\n",
      "Lost! - Coldplay\n",
      "Clocks - Coldplay\n",
      "----------------------------------------------------------------------\n",
      "Recommendation process going on:\n",
      "----------------------------------------------------------------------\n",
      "No. of unique songs for the user: 8\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :3429\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>She Just Likes To Fight - Four Tet</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>Warning Sign - Coldplay</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>We Never Change - Coldplay</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>Puppetmad - Puppetmastaz</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>God Put A Smile Upon Your Face - Coldplay</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>Susie Q - Creedence Clearwater Revival</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>The Joker - Fatboy Slim</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>Korg Rhythm Afro - Holy Fuck</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>This Unfolds - Four Tet</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9d6f0ead607ac2a6c2460e4d14fb439a146b7dec</td>\n",
       "      <td>high fives - Four Tet</td>\n",
       "      <td>0.281579</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    user_id  \\\n",
       "0  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "1  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "2  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "3  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "4  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "5  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "6  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "7  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "8  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "9  9d6f0ead607ac2a6c2460e4d14fb439a146b7dec   \n",
       "\n",
       "                                        song     score rank  \n",
       "0         She Just Likes To Fight - Four Tet  0.281579    1  \n",
       "1                    Warning Sign - Coldplay  0.281579    2  \n",
       "2                 We Never Change - Coldplay  0.281579    3  \n",
       "3                   Puppetmad - Puppetmastaz  0.281579    4  \n",
       "4  God Put A Smile Upon Your Face - Coldplay  0.281579    5  \n",
       "5     Susie Q - Creedence Clearwater Revival  0.281579    6  \n",
       "6                    The Joker - Fatboy Slim  0.281579    7  \n",
       "7               Korg Rhythm Afro - Holy Fuck  0.281579    8  \n",
       "8                    This Unfolds - Four Tet  0.281579    9  \n",
       "9                      high fives - Four Tet  0.281579   10  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_id = users[7]\n",
    "#Fill in the code here\n",
    "user_items = is_model.get_user_items(user_id)\n",
    "#\n",
    "print(\"------------------------------------------------------------------------------------\")\n",
    "print(\"Training data songs for the user userid: %s:\" % user_id)\n",
    "print(\"------------------------------------------------------------------------------------\")\n",
    "\n",
    "for user_item in user_items:\n",
    "    print(user_item)\n",
    "\n",
    "print(\"----------------------------------------------------------------------\")\n",
    "print(\"Recommendation process going on:\")\n",
    "print(\"----------------------------------------------------------------------\")\n",
    "\n",
    "#Recommend songs for the user using personalized model\n",
    "is_model.recommend(user_id)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We can also apply the model to find similar songs to any song in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :271\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>Somebody To Love - Justin Bieber</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>Bad Company - Five Finger Death Punch</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>Love Me - Justin Bieber</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>One Time - Justin Bieber</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>Here Without You - 3 Doors Down</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>Stuck In The Moment - Justin Bieber</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>Teach Me How To Dougie - California Swag District</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>Paper Planes - M.I.A.</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>Already Gone - Kelly Clarkson</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>The Funeral (Album Version) - Band Of Horses</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id                                               song     score rank\n",
       "0                           Somebody To Love - Justin Bieber  0.428571    1\n",
       "1                      Bad Company - Five Finger Death Punch  0.375000    2\n",
       "2                                    Love Me - Justin Bieber  0.333333    3\n",
       "3                                   One Time - Justin Bieber  0.333333    4\n",
       "4                            Here Without You - 3 Doors Down  0.333333    5\n",
       "5                        Stuck In The Moment - Justin Bieber  0.333333    6\n",
       "6          Teach Me How To Dougie - California Swag District  0.333333    7\n",
       "7                                      Paper Planes - M.I.A.  0.333333    8\n",
       "8                              Already Gone - Kelly Clarkson  0.333333    9\n",
       "9               The Funeral (Album Version) - Band Of Horses  0.300000   10"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_model.get_similar_items(['U Smile - Justin Bieber'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quiz 4. Use the personalized recommender model to get similar songs for the following song."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :969\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>song</th>\n",
       "      <th>score</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>Fix You - Coldplay</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>Creep (Explicit) - Radiohead</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>Clocks - Coldplay</td>\n",
       "      <td>0.280000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>Seven Nation Army - The White Stripes</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>Paper Planes - M.I.A.</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>Halo - Beyoncé</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>The Funeral (Album Version) - Band Of Horses</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>In My Place - Coldplay</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>Kryptonite - 3 Doors Down</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>When You Were Young - The Killers</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id                                          song     score rank\n",
       "0                                    Fix You - Coldplay  0.375000    1\n",
       "1                          Creep (Explicit) - Radiohead  0.291667    2\n",
       "2                                     Clocks - Coldplay  0.280000    3\n",
       "3                 Seven Nation Army - The White Stripes  0.250000    4\n",
       "4                                 Paper Planes - M.I.A.  0.208333    5\n",
       "5                                        Halo - Beyoncé  0.200000    6\n",
       "6          The Funeral (Album Version) - Band Of Horses  0.181818    7\n",
       "7                                In My Place - Coldplay  0.181818    8\n",
       "8                             Kryptonite - 3 Doors Down  0.166667    9\n",
       "9                     When You Were Young - The Killers  0.166667   10"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "song = 'Yellow - Coldplay'\n",
    "###Fill in the code here\n",
    "is_model.get_similar_items([song])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quantitative comparison between the models\n",
    "\n",
    "We now formally compare the popularity and the personalized models using precision-recall curves. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class to calculate precision and recall (This can be used as a black box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Evaluation.precision_recall_calculator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use the above precision recall calculator class to calculate the evaluation measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of user_test_and_training:319\n",
      "Length of user sample:15\n",
      "Getting recommendations for user:1645b689f873529ab85e3b72742be44813e82bd3\n",
      "No. of unique songs for the user: 13\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :5100\n",
      "Getting recommendations for user:ee03697dfdf668a9cd19a44a6f257b9d6a1dc60a\n",
      "No. of unique songs for the user: 27\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :4995\n",
      "Getting recommendations for user:95942345306393998eb3a051ae5fa3c4d5afbaa4\n",
      "No. of unique songs for the user: 80\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :10810\n",
      "Getting recommendations for user:a4a1daac7ba5534ade06e32c5b9d220eb5ce145d\n",
      "No. of unique songs for the user: 6\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :377\n",
      "Getting recommendations for user:332de6dcfc5b597500a661ab51a84653705b492b\n",
      "No. of unique songs for the user: 26\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :3608\n",
      "Getting recommendations for user:fb41d1c374d093ab643ef3bcd70eeb258d479076\n",
      "No. of unique songs for the user: 15\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :2561\n",
      "Getting recommendations for user:6beb4699102775dab57aa406c5ea1217c4ff4869\n",
      "No. of unique songs for the user: 42\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :6181\n",
      "Getting recommendations for user:fbc8b4327aa8ae38d17b567f486468ca10cacde3\n",
      "No. of unique songs for the user: 7\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :1413\n",
      "Getting recommendations for user:dd88ea94f605a63d9fc37a214127e3f00e85e42d\n",
      "No. of unique songs for the user: 6\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :228\n",
      "Getting recommendations for user:6b7a5895d266599bc414b9eb8fbad59c95b9a99d\n",
      "No. of unique songs for the user: 15\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :1412\n",
      "Getting recommendations for user:930d2be6c85315d72cab9823ec0f7bfe7e477794\n",
      "No. of unique songs for the user: 47\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :11886\n",
      "Getting recommendations for user:956044d724390e40c8511b49e5bf6bc28071de3a\n",
      "No. of unique songs for the user: 21\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :1444\n",
      "Getting recommendations for user:ff4322e94814d3c7895d07e6f94139b092862611\n",
      "No. of unique songs for the user: 8\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :424\n",
      "Getting recommendations for user:d305e8d98cf77ce3b90c69580dd3fc38e86fee93\n",
      "No. of unique songs for the user: 7\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :315\n",
      "Getting recommendations for user:b12c786deef0e618b5f277bc337f67128f425efe\n",
      "No. of unique songs for the user: 98\n",
      "no. of unique songs in the training set: 4483\n",
      "Non zero values in cooccurence_matrix :25122\n",
      "80.49175262451172\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "#Define what percentage of users to use for precision recall calculation\n",
    "user_sample = 0.05\n",
    "\n",
    "#Instantiate the precision_recall_calculator class\n",
    "pr = Evaluation.precision_recall_calculator(test_data, train_data, pm, is_model)\n",
    "\n",
    "#Call method to calculate precision and recall values\n",
    "(pm_avg_precision_list, pm_avg_recall_list, ism_avg_precision_list, ism_avg_recall_list) = pr.calculate_measures(user_sample)\n",
    "\n",
    "end = time.time()\n",
    "print(end - start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code to plot precision recall curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pylab as pl\n",
    "\n",
    "#Method to generate precision and recall curve\n",
    "def plot_precision_recall(m1_precision_list, m1_recall_list, m1_label, m2_precision_list, m2_recall_list, m2_label):\n",
    "    pl.clf()    \n",
    "    pl.plot(m1_recall_list, m1_precision_list, label=m1_label)\n",
    "    pl.plot(m2_recall_list, m2_precision_list, label=m2_label)\n",
    "    pl.xlabel('Recall')\n",
    "    pl.ylabel('Precision')\n",
    "    pl.ylim([0.0, 0.20])\n",
    "    pl.xlim([0.0, 0.20])\n",
    "    pl.title('Precision-Recall curve')\n",
    "    #pl.legend(loc=\"upper right\")\n",
    "    pl.legend(loc=9, bbox_to_anchor=(0.5, -0.2))\n",
    "    pl.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plotting precision recall curves.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'plot_precision_recall' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-de42221c1c58>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Plotting precision recall curves.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m plot_precision_recall(pm_avg_precision_list, pm_avg_recall_list, \"popularity_model\",\n\u001b[0m\u001b[1;32m      4\u001b[0m                       ism_avg_precision_list, ism_avg_recall_list, \"item_similarity_model\")\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plot_precision_recall' is not defined"
     ]
    }
   ],
   "source": [
    "print(\"Plotting precision recall curves.\")\n",
    "\n",
    "plot_precision_recall(pm_avg_precision_list, pm_avg_recall_list, \"popularity_model\",\n",
    "                      ism_avg_precision_list, ism_avg_recall_list, \"item_similarity_model\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Precision Recall curve using pickled results on a larger data subset(Python 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Plotting precision recall curves for a larger subset of data (100,000 rows) (user sample = 0.005).\")\n",
    "\n",
    "#Read the persisted files \n",
    "pm_avg_precision_list = joblib.load('pm_avg_precision_list_3.pkl')\n",
    "pm_avg_recall_list = joblib.load('pm_avg_recall_list_3.pkl')\n",
    "ism_avg_precision_list = joblib.load('ism_avg_precision_list_3.pkl')\n",
    "ism_avg_recall_list = joblib.load('ism_avg_recall_list_3.pkl')\n",
    "\n",
    "print(\"Plotting precision recall curves.\")\n",
    "plot_precision_recall(pm_avg_precision_list, pm_avg_recall_list, \"popularity_model\",\n",
    "                      ism_avg_precision_list, ism_avg_recall_list, \"item_similarity_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate Precision Recall curve using pickled results on a larger data subset(Python 2.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Plotting precision recall curves for a larger subset of data (100,000 rows) (user sample = 0.005).\")\n",
    "\n",
    "pm_avg_precision_list = joblib.load('pm_avg_precision_list_2.pkl')\n",
    "pm_avg_recall_list = joblib.load('pm_avg_recall_list_2.pkl')\n",
    "ism_avg_precision_list = joblib.load('ism_avg_precision_list_2.pkl')\n",
    "ism_avg_recall_list = joblib.load('ism_avg_recall_list_2.pkl')\n",
    "\n",
    "print(\"Plotting precision recall curves.\")\n",
    "plot_precision_recall(pm_avg_precision_list, pm_avg_recall_list, \"popularity_model\",\n",
    "                      ism_avg_precision_list, ism_avg_recall_list, \"item_similarity_model\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}