ml_prepro.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random\n",
    "import numpy as np\n",
    "import pickle\n",
    "import csv\n",
    "import tensorflow as tf\n",
    "\n",
    "dataset_dir = '/data/private/Ad/amazon/np_prepro/'\n",
    "with open(dataset_dir+'dataset.pkl', 'rb') as f:\n",
    "    train_set = pickle.load(f, encoding='latin1') # uid, [hist], vid, label\n",
    "    test_set = pickle.load(f, encoding='latin1')  # uid, [hist], pid, nid\n",
    "    cate_list = pickle.load(f, encoding='latin1') # id2cate list\n",
    "    cate_list = tf.convert_to_tensor(cate_list, dtype=tf.int64)\n",
    "    user_count, item_count, cate_count = pickle.load(f) # (user_count, item_count, cate_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = '/data/private/Ad/ml-20m/'\n",
    "reviews_df  = pd.read_csv(root+'ratings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112486027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>112</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1094785740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>151</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1094785734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>223</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112485573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>253</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>260</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>293</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>296</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>318</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>337</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1094785709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>367</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112485980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>541</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112484603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>589</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112485557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>593</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>653</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1094785691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>919</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1094785621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>924</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1094785598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>1009</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112486013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>1036</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112485480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>1079</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1094785665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>1080</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112485375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>1089</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>1090</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112485453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>1097</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1112485701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>1136</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000233</th>\n",
       "      <td>138493</td>\n",
       "      <td>50872</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1256750388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000234</th>\n",
       "      <td>138493</td>\n",
       "      <td>51086</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1255810566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000235</th>\n",
       "      <td>138493</td>\n",
       "      <td>51662</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1255856908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000236</th>\n",
       "      <td>138493</td>\n",
       "      <td>51884</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1256294768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000237</th>\n",
       "      <td>138493</td>\n",
       "      <td>52579</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1255856957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000238</th>\n",
       "      <td>138493</td>\n",
       "      <td>52975</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1256680293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000239</th>\n",
       "      <td>138493</td>\n",
       "      <td>53123</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1255816320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000240</th>\n",
       "      <td>138493</td>\n",
       "      <td>53125</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1255810649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000241</th>\n",
       "      <td>138493</td>\n",
       "      <td>53322</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1255812146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000242</th>\n",
       "      <td>138493</td>\n",
       "      <td>53464</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1260209920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000243</th>\n",
       "      <td>138493</td>\n",
       "      <td>53996</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1259865104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000244</th>\n",
       "      <td>138493</td>\n",
       "      <td>55269</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1255816088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000245</th>\n",
       "      <td>138493</td>\n",
       "      <td>55814</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1255811181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000246</th>\n",
       "      <td>138493</td>\n",
       "      <td>56757</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1255810698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000247</th>\n",
       "      <td>138493</td>\n",
       "      <td>56801</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1255809988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000248</th>\n",
       "      <td>138493</td>\n",
       "      <td>58879</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1255816798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000249</th>\n",
       "      <td>138493</td>\n",
       "      <td>59315</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1255818138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000250</th>\n",
       "      <td>138493</td>\n",
       "      <td>59725</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1255818078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000251</th>\n",
       "      <td>138493</td>\n",
       "      <td>59784</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1255816901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000252</th>\n",
       "      <td>138493</td>\n",
       "      <td>60069</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1258134687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000253</th>\n",
       "      <td>138493</td>\n",
       "      <td>60816</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1259865163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000254</th>\n",
       "      <td>138493</td>\n",
       "      <td>61160</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1258390537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000255</th>\n",
       "      <td>138493</td>\n",
       "      <td>65682</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1255816373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000256</th>\n",
       "      <td>138493</td>\n",
       "      <td>66762</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1255805408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000257</th>\n",
       "      <td>138493</td>\n",
       "      <td>68319</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1260209720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000258</th>\n",
       "      <td>138493</td>\n",
       "      <td>68954</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1258126920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000259</th>\n",
       "      <td>138493</td>\n",
       "      <td>69526</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1259865108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000260</th>\n",
       "      <td>138493</td>\n",
       "      <td>69644</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1260209457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000261</th>\n",
       "      <td>138493</td>\n",
       "      <td>70286</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1258126944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20000262</th>\n",
       "      <td>138493</td>\n",
       "      <td>71619</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1255811136</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000263 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          userId  movieId  rating   timestamp\n",
       "0              1        2     3.5  1112486027\n",
       "1              1       29     3.5  1112484676\n",
       "2              1       32     3.5  1112484819\n",
       "3              1       47     3.5  1112484727\n",
       "4              1       50     3.5  1112484580\n",
       "5              1      112     3.5  1094785740\n",
       "6              1      151     4.0  1094785734\n",
       "7              1      223     4.0  1112485573\n",
       "8              1      253     4.0  1112484940\n",
       "9              1      260     4.0  1112484826\n",
       "10             1      293     4.0  1112484703\n",
       "11             1      296     4.0  1112484767\n",
       "12             1      318     4.0  1112484798\n",
       "13             1      337     3.5  1094785709\n",
       "14             1      367     3.5  1112485980\n",
       "15             1      541     4.0  1112484603\n",
       "16             1      589     3.5  1112485557\n",
       "17             1      593     3.5  1112484661\n",
       "18             1      653     3.0  1094785691\n",
       "19             1      919     3.5  1094785621\n",
       "20             1      924     3.5  1094785598\n",
       "21             1     1009     3.5  1112486013\n",
       "22             1     1036     4.0  1112485480\n",
       "23             1     1079     4.0  1094785665\n",
       "24             1     1080     3.5  1112485375\n",
       "25             1     1089     3.5  1112484669\n",
       "26             1     1090     4.0  1112485453\n",
       "27             1     1097     4.0  1112485701\n",
       "28             1     1136     3.5  1112484609\n",
       "29             1     1193     3.5  1112484690\n",
       "...          ...      ...     ...         ...\n",
       "20000233  138493    50872     3.5  1256750388\n",
       "20000234  138493    51086     3.5  1255810566\n",
       "20000235  138493    51662     4.5  1255856908\n",
       "20000236  138493    51884     4.5  1256294768\n",
       "20000237  138493    52579     4.0  1255856957\n",
       "20000238  138493    52975     4.0  1256680293\n",
       "20000239  138493    53123     4.0  1255816320\n",
       "20000240  138493    53125     3.0  1255810649\n",
       "20000241  138493    53322     4.0  1255812146\n",
       "20000242  138493    53464     4.0  1260209920\n",
       "20000243  138493    53996     4.5  1259865104\n",
       "20000244  138493    55269     5.0  1255816088\n",
       "20000245  138493    55814     5.0  1255811181\n",
       "20000246  138493    56757     3.0  1255810698\n",
       "20000247  138493    56801     3.0  1255809988\n",
       "20000248  138493    58879     4.5  1255816798\n",
       "20000249  138493    59315     4.0  1255818138\n",
       "20000250  138493    59725     3.0  1255818078\n",
       "20000251  138493    59784     5.0  1255816901\n",
       "20000252  138493    60069     4.0  1258134687\n",
       "20000253  138493    60816     4.5  1259865163\n",
       "20000254  138493    61160     4.0  1258390537\n",
       "20000255  138493    65682     4.5  1255816373\n",
       "20000256  138493    66762     4.5  1255805408\n",
       "20000257  138493    68319     4.5  1260209720\n",
       "20000258  138493    68954     4.5  1258126920\n",
       "20000259  138493    69526     4.5  1259865108\n",
       "20000260  138493    69644     3.0  1260209457\n",
       "20000261  138493    70286     5.0  1258126944\n",
       "20000262  138493    71619     2.5  1255811136\n",
       "\n",
       "[20000263 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(root+'np_prepro/reviews.pkl', 'wb') as f:\n",
    "    pickle.dump(reviews_df, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_df = pd.read_csv(root+'movies.csv')\n",
    "\n",
    "meta_df[meta_df['movieId'].isin(reviews_df['movieId'].unique())]\n",
    "meta_df = meta_df.reset_index(drop=True)\n",
    "with open(root+'np_prepro/meta.pkl', 'wb') as f:\n",
    "    pickle.dump(meta_df, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>Heat (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>Sabrina (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>Tom and Huck (1995)</td>\n",
       "      <td>Adventure|Children</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Sudden Death (1995)</td>\n",
       "      <td>Action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>Action|Adventure|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>American President, The (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>Dracula: Dead and Loving It (1995)</td>\n",
       "      <td>Comedy|Horror</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>Balto (1995)</td>\n",
       "      <td>Adventure|Animation|Children</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>Nixon (1995)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>Cutthroat Island (1995)</td>\n",
       "      <td>Action|Adventure|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>Casino (1995)</td>\n",
       "      <td>Crime|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>Sense and Sensibility (1995)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>Ace Ventura: When Nature Calls (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>Money Train (1995)</td>\n",
       "      <td>Action|Comedy|Crime|Drama|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>Comedy|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>Crime|Drama|Horror|Mystery|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23</td>\n",
       "      <td>Assassins (1995)</td>\n",
       "      <td>Action|Crime|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>Powder (1995)</td>\n",
       "      <td>Drama|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25</td>\n",
       "      <td>Leaving Las Vegas (1995)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>26</td>\n",
       "      <td>Othello (1995)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>27</td>\n",
       "      <td>Now and Then (1995)</td>\n",
       "      <td>Children|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>28</td>\n",
       "      <td>Persuasion (1995)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>29</td>\n",
       "      <td>City of Lost Children, The (Cité des enfants p...</td>\n",
       "      <td>Adventure|Drama|Fantasy|Mystery|Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>30</td>\n",
       "      <td>Shanghai Triad (Yao a yao yao dao waipo qiao) ...</td>\n",
       "      <td>Crime|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27248</th>\n",
       "      <td>131146</td>\n",
       "      <td>Werner - Volles Rooäää (1999)</td>\n",
       "      <td>Animation|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27249</th>\n",
       "      <td>131148</td>\n",
       "      <td>What A Man (2011)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27250</th>\n",
       "      <td>131150</td>\n",
       "      <td>7 Dwarves: The Forest Is Not Enough (2006)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27251</th>\n",
       "      <td>131152</td>\n",
       "      <td>The Fat Spy (1966)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27252</th>\n",
       "      <td>131154</td>\n",
       "      <td>Die Bademeister – Weiber, saufen, Leben retten...</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27253</th>\n",
       "      <td>131156</td>\n",
       "      <td>Ants in the Pants 2 (2002)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27254</th>\n",
       "      <td>131158</td>\n",
       "      <td>Manta, Manta (1991)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27255</th>\n",
       "      <td>131160</td>\n",
       "      <td>Oscar and the Lady in Pink (2009)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27256</th>\n",
       "      <td>131162</td>\n",
       "      <td>Por un puñado de besos (2014)</td>\n",
       "      <td>Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27257</th>\n",
       "      <td>131164</td>\n",
       "      <td>Vietnam in HD (2011)</td>\n",
       "      <td>War</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27258</th>\n",
       "      <td>131166</td>\n",
       "      <td>WWII IN HD (2009)</td>\n",
       "      <td>(no genres listed)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27259</th>\n",
       "      <td>131168</td>\n",
       "      <td>Phoenix (2014)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27260</th>\n",
       "      <td>131170</td>\n",
       "      <td>Parallels (2015)</td>\n",
       "      <td>Sci-Fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27261</th>\n",
       "      <td>131172</td>\n",
       "      <td>Closed Curtain (2013)</td>\n",
       "      <td>(no genres listed)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27262</th>\n",
       "      <td>131174</td>\n",
       "      <td>Gentlemen (2014)</td>\n",
       "      <td>Drama|Romance|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27263</th>\n",
       "      <td>131176</td>\n",
       "      <td>A Second Chance (2014)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27264</th>\n",
       "      <td>131180</td>\n",
       "      <td>Dead Rising: Watchtower (2015)</td>\n",
       "      <td>Action|Horror|Thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27265</th>\n",
       "      <td>131231</td>\n",
       "      <td>Standby (2014)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27266</th>\n",
       "      <td>131237</td>\n",
       "      <td>What Men Talk About (2010)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27267</th>\n",
       "      <td>131239</td>\n",
       "      <td>Three Quarter Moon (2011)</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27268</th>\n",
       "      <td>131241</td>\n",
       "      <td>Ants in the Pants (2000)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27269</th>\n",
       "      <td>131243</td>\n",
       "      <td>Werner - Gekotzt wird später (2003)</td>\n",
       "      <td>Animation|Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27270</th>\n",
       "      <td>131248</td>\n",
       "      <td>Brother Bear 2 (2006)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27271</th>\n",
       "      <td>131250</td>\n",
       "      <td>No More School (2000)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27272</th>\n",
       "      <td>131252</td>\n",
       "      <td>Forklift Driver Klaus: The First Day on the Jo...</td>\n",
       "      <td>Comedy|Horror</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27273</th>\n",
       "      <td>131254</td>\n",
       "      <td>Kein Bund für's Leben (2007)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27274</th>\n",
       "      <td>131256</td>\n",
       "      <td>Feuer, Eis &amp; Dosenbier (2002)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27275</th>\n",
       "      <td>131258</td>\n",
       "      <td>The Pirates (2014)</td>\n",
       "      <td>Adventure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27276</th>\n",
       "      <td>131260</td>\n",
       "      <td>Rentun Ruusu (2001)</td>\n",
       "      <td>(no genres listed)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27277</th>\n",
       "      <td>131262</td>\n",
       "      <td>Innocence (2014)</td>\n",
       "      <td>Adventure|Fantasy|Horror</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27278 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       movieId                                              title  \\\n",
       "0            1                                   Toy Story (1995)   \n",
       "1            2                                     Jumanji (1995)   \n",
       "2            3                            Grumpier Old Men (1995)   \n",
       "3            4                           Waiting to Exhale (1995)   \n",
       "4            5                 Father of the Bride Part II (1995)   \n",
       "5            6                                        Heat (1995)   \n",
       "6            7                                     Sabrina (1995)   \n",
       "7            8                                Tom and Huck (1995)   \n",
       "8            9                                Sudden Death (1995)   \n",
       "9           10                                   GoldenEye (1995)   \n",
       "10          11                     American President, The (1995)   \n",
       "11          12                 Dracula: Dead and Loving It (1995)   \n",
       "12          13                                       Balto (1995)   \n",
       "13          14                                       Nixon (1995)   \n",
       "14          15                            Cutthroat Island (1995)   \n",
       "15          16                                      Casino (1995)   \n",
       "16          17                       Sense and Sensibility (1995)   \n",
       "17          18                                  Four Rooms (1995)   \n",
       "18          19              Ace Ventura: When Nature Calls (1995)   \n",
       "19          20                                 Money Train (1995)   \n",
       "20          21                                  Get Shorty (1995)   \n",
       "21          22                                     Copycat (1995)   \n",
       "22          23                                   Assassins (1995)   \n",
       "23          24                                      Powder (1995)   \n",
       "24          25                           Leaving Las Vegas (1995)   \n",
       "25          26                                     Othello (1995)   \n",
       "26          27                                Now and Then (1995)   \n",
       "27          28                                  Persuasion (1995)   \n",
       "28          29  City of Lost Children, The (Cité des enfants p...   \n",
       "29          30  Shanghai Triad (Yao a yao yao dao waipo qiao) ...   \n",
       "...        ...                                                ...   \n",
       "27248   131146                      Werner - Volles Rooäää (1999)   \n",
       "27249   131148                                  What A Man (2011)   \n",
       "27250   131150         7 Dwarves: The Forest Is Not Enough (2006)   \n",
       "27251   131152                                 The Fat Spy (1966)   \n",
       "27252   131154  Die Bademeister – Weiber, saufen, Leben retten...   \n",
       "27253   131156                         Ants in the Pants 2 (2002)   \n",
       "27254   131158                                Manta, Manta (1991)   \n",
       "27255   131160                  Oscar and the Lady in Pink (2009)   \n",
       "27256   131162                      Por un puñado de besos (2014)   \n",
       "27257   131164                               Vietnam in HD (2011)   \n",
       "27258   131166                                  WWII IN HD (2009)   \n",
       "27259   131168                                     Phoenix (2014)   \n",
       "27260   131170                                   Parallels (2015)   \n",
       "27261   131172                              Closed Curtain (2013)   \n",
       "27262   131174                                   Gentlemen (2014)   \n",
       "27263   131176                             A Second Chance (2014)   \n",
       "27264   131180                     Dead Rising: Watchtower (2015)   \n",
       "27265   131231                                     Standby (2014)   \n",
       "27266   131237                         What Men Talk About (2010)   \n",
       "27267   131239                          Three Quarter Moon (2011)   \n",
       "27268   131241                           Ants in the Pants (2000)   \n",
       "27269   131243                Werner - Gekotzt wird später (2003)   \n",
       "27270   131248                              Brother Bear 2 (2006)   \n",
       "27271   131250                              No More School (2000)   \n",
       "27272   131252  Forklift Driver Klaus: The First Day on the Jo...   \n",
       "27273   131254                       Kein Bund für's Leben (2007)   \n",
       "27274   131256                      Feuer, Eis & Dosenbier (2002)   \n",
       "27275   131258                                 The Pirates (2014)   \n",
       "27276   131260                                Rentun Ruusu (2001)   \n",
       "27277   131262                                   Innocence (2014)   \n",
       "\n",
       "                                            genres  \n",
       "0      Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                       Adventure|Children|Fantasy  \n",
       "2                                   Comedy|Romance  \n",
       "3                             Comedy|Drama|Romance  \n",
       "4                                           Comedy  \n",
       "5                            Action|Crime|Thriller  \n",
       "6                                   Comedy|Romance  \n",
       "7                               Adventure|Children  \n",
       "8                                           Action  \n",
       "9                        Action|Adventure|Thriller  \n",
       "10                            Comedy|Drama|Romance  \n",
       "11                                   Comedy|Horror  \n",
       "12                    Adventure|Animation|Children  \n",
       "13                                           Drama  \n",
       "14                        Action|Adventure|Romance  \n",
       "15                                     Crime|Drama  \n",
       "16                                   Drama|Romance  \n",
       "17                                          Comedy  \n",
       "18                                          Comedy  \n",
       "19              Action|Comedy|Crime|Drama|Thriller  \n",
       "20                           Comedy|Crime|Thriller  \n",
       "21             Crime|Drama|Horror|Mystery|Thriller  \n",
       "22                           Action|Crime|Thriller  \n",
       "23                                    Drama|Sci-Fi  \n",
       "24                                   Drama|Romance  \n",
       "25                                           Drama  \n",
       "26                                  Children|Drama  \n",
       "27                                   Drama|Romance  \n",
       "28          Adventure|Drama|Fantasy|Mystery|Sci-Fi  \n",
       "29                                     Crime|Drama  \n",
       "...                                            ...  \n",
       "27248                             Animation|Comedy  \n",
       "27249                               Comedy|Romance  \n",
       "27250                                       Comedy  \n",
       "27251                                       Comedy  \n",
       "27252                                       Comedy  \n",
       "27253                                       Comedy  \n",
       "27254                                       Comedy  \n",
       "27255                                        Drama  \n",
       "27256                                Drama|Romance  \n",
       "27257                                          War  \n",
       "27258                           (no genres listed)  \n",
       "27259                                        Drama  \n",
       "27260                                       Sci-Fi  \n",
       "27261                           (no genres listed)  \n",
       "27262                       Drama|Romance|Thriller  \n",
       "27263                                        Drama  \n",
       "27264                       Action|Horror|Thriller  \n",
       "27265                               Comedy|Romance  \n",
       "27266                                       Comedy  \n",
       "27267                                 Comedy|Drama  \n",
       "27268                               Comedy|Romance  \n",
       "27269                             Animation|Comedy  \n",
       "27270  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "27271                                       Comedy  \n",
       "27272                                Comedy|Horror  \n",
       "27273                                       Comedy  \n",
       "27274                                       Comedy  \n",
       "27275                                    Adventure  \n",
       "27276                           (no genres listed)  \n",
       "27277                     Adventure|Fantasy|Horror  \n",
       "\n",
       "[27278 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(root+'np_prepro/reviews.pkl', 'rb') as f:\n",
    "#     reviews_df = pickle.load(f)\n",
    "# with open(root+'np_prepro/meta.pkl', 'rb') as f:\n",
    "#     meta_df = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_df = reviews_df[['userId','movieId','rating','timestamp']]\n",
    "reviews_df.loc[:,'rating'] = reviews_df['rating'].map(lambda x: 1 if x > 3 else 0)\n",
    "meta_df = meta_df[['movieId', 'genres']]\n",
    "meta_df.loc[:,'genres'] = meta_df['genres'].map(lambda x: x.split('|')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_map(df, col_name):\n",
    "    key = sorted(df[col_name].unique().tolist())\n",
    "    m = dict(zip(key, range(len(key))))\n",
    "    df.loc[:,col_name] = df[col_name].map(lambda x: m[x])\n",
    "    return m, key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "vid_map, vid_key = build_map(meta_df, 'movieId')\n",
    "cat_map, cat_key = build_map(meta_df, 'genres')\n",
    "uid_map, uid_key = build_map(reviews_df, 'userId')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_count: 138493\titem_count: 27278\tcate_count: 20\texample_count: 20000263\n"
     ]
    }
   ],
   "source": [
    "user_count, item_count, cate_count, example_count =\\\n",
    "    len(uid_map), len(vid_map), len(cat_map), reviews_df.shape[0]\n",
    "print('user_count: %d\\titem_count: %d\\tcate_count: %d\\texample_count: %d' %\n",
    "      (user_count, item_count, cate_count, example_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'userID'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-c0cb3c1bc324>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mreviews_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'movieId'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreviews_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'movieId'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvid_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mreviews_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreviews_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'userID'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'timestamp'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mreviews_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreviews_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36msort_values\u001b[0;34m(self, by, axis, ascending, inplace, kind, na_position)\u001b[0m\n\u001b[1;32m   4709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4710\u001b[0m             keys = [self._get_label_or_level_values(x, axis=axis)\n\u001b[0;32m-> 4711\u001b[0;31m                     for x in by]\n\u001b[0m\u001b[1;32m   4712\u001b[0m             indexer = lexsort_indexer(keys, orders=ascending,\n\u001b[1;32m   4713\u001b[0m                                       na_position=na_position)\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   4709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4710\u001b[0m             keys = [self._get_label_or_level_values(x, axis=axis)\n\u001b[0;32m-> 4711\u001b[0;31m                     for x in by]\n\u001b[0m\u001b[1;32m   4712\u001b[0m             indexer = lexsort_indexer(keys, orders=ascending,\n\u001b[1;32m   4713\u001b[0m                                       na_position=na_position)\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_label_or_level_values\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1704\u001b[0m             \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1705\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1706\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1708\u001b[0m         \u001b[0;31m# Check for duplicates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'userID'"
     ]
    }
   ],
   "source": [
    "meta_df = meta_df.sort_values('movieId')\n",
    "meta_df = meta_df.reset_index(drop=True)\n",
    "\n",
    "reviews_df['movieId'] = reviews_df['movieId'].map(lambda x: vid_map[x])\n",
    "reviews_df = reviews_df.sort_values(['userId', 'timestamp'])\n",
    "reviews_df = reviews_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "cate_list = [meta_df['genres'][i] for i in range(len(vid_map))]\n",
    "cate_list = np.array(cate_list, dtype=np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(root+'np_prepro/remap.pkl', 'wb') as f:\n",
    "    pickle.dump(reviews_df, f, pickle.HIGHEST_PROTOCOL) # uid, iid\n",
    "    pickle.dump(cate_list, f, pickle.HIGHEST_PROTOCOL) # cid of iid line\n",
    "    pickle.dump((user_count, item_count, cate_count, example_count),\n",
    "              f, pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump((vid_key, cat_key, uid_key), f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_cnt, neg_cnt = 0, 0\n",
    "for userId, hist in reviews_df.groupby('userId'):\n",
    "    movie_list = hist['movieId'].tolist()\n",
    "    label_list = hist['rating'].tolist()\n",
    "\n",
    "    pos_cnt += sum(label_list)\n",
    "    neg_cnt += len(label_list) - sum(label_list)\n",
    "    \n",
    "print(pos_cnt, neg_cnt, pos_cnt/(pos_cnt+neg_cnt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234)\n",
    "\n",
    "train_set = []\n",
    "test_set = []\n",
    "train_count = 100000\n",
    "train_user = np.random.choice(user_count, train_count, replace=False)\n",
    "for userId, hist in reviews_df.groupby('userId'):\n",
    "    movie_list = hist['movieId'].tolist()\n",
    "    label_list = hist['rating'].tolist()\n",
    "    pos_list, neg_list = [], []\n",
    "    for i, (v,r) in enumerate(zip(movie_list, label_list)):\n",
    "        if r == 1: pos_list.append(v);\n",
    "        else: neg_list.append(v)\n",
    "    \n",
    "    if len(pos_list) > len(neg_list):\n",
    "        for _ in range(len(pos_list)-len(neg_list)):\n",
    "            neg = pos_list[0]\n",
    "            while neg in pos_list + neg_list :\n",
    "                neg = random.randint(0, item_count-1)\n",
    "            neg_list.append(neg)\n",
    "\n",
    "    if userId in train_user:\n",
    "        for i in range(1, len(pos_list)):\n",
    "            hist = pos_list[:i]\n",
    "            train_set.append((userId, hist, pos_list[i], 1))\n",
    "            train_set.append((userId, hist, neg_list[i], 0))\n",
    "    else:\n",
    "        for i in range(1, len(pos_list)):\n",
    "            hist = movie_list[:i]\n",
    "            label = (pos_list[i], neg_list[i])\n",
    "            test_set.append((userId, hist, label))\n",
    "\n",
    "random.shuffle(train_set)\n",
    "random.shuffle(test_set)\n",
    "\n",
    "with open(root+'np_prepro/dataset.pkl', 'wb') as f:\n",
    "    pickle.dump(train_set, f, pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump(test_set, f, pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump(cate_list, f, pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump((user_count, item_count, cate_count), f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(root+'dataset.pkl', 'wb') as f:\n",
    "    pickle.dump(train_set, f, protocol=2)\n",
    "    pickle.dump(test_set, f, protocol=2)\n",
    "    pickle.dump(cate_list, f, protocol=2)\n",
    "    pickle.dump((user_count, item_count, cate_count), f, protocol=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}