Data_reader_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from data_reader import amazon_dataset_iters\n",
    "from amazon_train_test_split import split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading datasets...\n",
      "datasets loaded\n",
      "item vocab built\n",
      "user vocab built\n",
      "text vocab built\n",
      "tips vocab built\n"
     ]
    }
   ],
   "source": [
    "#movie dataset: 23.8 Gb in RAM\n",
    "text_vocab, tips_vocab, train_iter, val_iter, test_iter = (\n",
    "    amazon_dataset_iters('./data/sample_dataset/',\n",
    "                         device=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      "    2\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    6\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "   20\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    0\n",
      "    3\n",
      "[torch.cuda.LongTensor of size 27x1 (GPU 1)]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/akiselev/data/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: generator 'Iterator.__iter__' raised StopIteration\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "for batch in train_iter:\n",
    "    print(batch.text)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mentally'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_vocab.itos[14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'$start spiritually and mentally inspiring ! a book that allows you to question your morals and will help you discover who you really are ! $end'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([text_vocab.itos[i] for i in next(iter(train_iter)).text.data.cpu().numpy().reshape(-1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2],\n",
       "       [18],\n",
       "       [ 6],\n",
       "       [14],\n",
       "       [13],\n",
       "       [ 4],\n",
       "       [ 7],\n",
       "       [10],\n",
       "       [19],\n",
       "       [ 8],\n",
       "       [ 5],\n",
       "       [20],\n",
       "       [16],\n",
       "       [24],\n",
       "       [15],\n",
       "       [ 6],\n",
       "       [22],\n",
       "       [12],\n",
       "       [ 5],\n",
       "       [11],\n",
       "       [21],\n",
       "       [ 5],\n",
       "       [17],\n",
       "       [ 9],\n",
       "       [ 4],\n",
       "       [ 3]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(train_iter)).text.data.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}