quora-question-pairs-data-prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Quora question pairs: data preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "import csv, json\n",
    "from zipfile import ZipFile\n",
    "from os.path import expanduser, exists\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.data_utils import get_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Initialize global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "KERAS_DATASETS_DIR = expanduser('~/.keras/datasets/')\n",
    "QUESTION_PAIRS_FILE_URL = 'http://qim.ec.quoracdn.net/quora_duplicate_questions.tsv'\n",
    "QUESTION_PAIRS_FILE = 'quora_duplicate_questions.tsv'\n",
    "GLOVE_ZIP_FILE_URL = 'http://nlp.stanford.edu/data/glove.840B.300d.zip'\n",
    "GLOVE_ZIP_FILE = 'glove.840B.300d.zip'\n",
    "GLOVE_FILE = 'glove.840B.300d.txt'\n",
    "Q1_TRAINING_DATA_FILE = 'q1_train.npy'\n",
    "Q2_TRAINING_DATA_FILE = 'q2_train.npy'\n",
    "LABEL_TRAINING_DATA_FILE = 'label_train.npy'\n",
    "WORD_EMBEDDING_MATRIX_FILE = 'word_embedding_matrix.npy'\n",
    "NB_WORDS_DATA_FILE = 'nb_words.json'\n",
    "MAX_NB_WORDS = 200000\n",
    "MAX_SEQUENCE_LENGTH = 25\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Download and extract questions pairs data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from http://qim.ec.quoracdn.net/quora_duplicate_questions.tsv\n",
      "Processing quora_duplicate_questions.tsv\n",
      "Question pairs: 404290\n"
     ]
    }
   ],
   "source": [
    "if not exists(KERAS_DATASETS_DIR + QUESTION_PAIRS_FILE):\n",
    "    get_file(QUESTION_PAIRS_FILE, QUESTION_PAIRS_FILE_URL)\n",
    "\n",
    "print(\"Processing\", QUESTION_PAIRS_FILE)\n",
    "\n",
    "question1 = []\n",
    "question2 = []\n",
    "is_duplicate = []\n",
    "with open(KERAS_DATASETS_DIR + QUESTION_PAIRS_FILE, encoding='utf-8') as csvfile:\n",
    "    reader = csv.DictReader(csvfile, delimiter='\\t')\n",
    "    for row in reader:\n",
    "        question1.append(row['question1'])\n",
    "        question2.append(row['question2'])\n",
    "        is_duplicate.append(row['is_duplicate'])\n",
    "\n",
    "print('Question pairs: %d' % len(question1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Build tokenized word index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Words in index: 95596\n"
     ]
    }
   ],
   "source": [
    "questions = question1 + question2\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(questions)\n",
    "question1_word_sequences = tokenizer.texts_to_sequences(question1)\n",
    "question2_word_sequences = tokenizer.texts_to_sequences(question2)\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "print(\"Words in index: %d\" % len(word_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Download and process GloVe embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from http://nlp.stanford.edu/data/glove.840B.300d.zip\n",
      "Processing glove.840B.300d.txt\n",
      "Word embeddings: 2196016\n"
     ]
    }
   ],
   "source": [
    "if not exists(KERAS_DATASETS_DIR + GLOVE_ZIP_FILE):\n",
    "    zipfile = ZipFile(get_file(GLOVE_ZIP_FILE, GLOVE_ZIP_FILE_URL))\n",
    "    zipfile.extract(GLOVE_FILE, path=KERAS_DATASETS_DIR)\n",
    "    \n",
    "print(\"Processing\", GLOVE_FILE)\n",
    "\n",
    "embeddings_index = {}\n",
    "with open(KERAS_DATASETS_DIR + GLOVE_FILE, encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split(' ')\n",
    "        word = values[0]\n",
    "        embedding = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = embedding\n",
    "\n",
    "print('Word embeddings: %d' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Prepare word embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Null word embeddings: 29276\n"
     ]
    }
   ],
   "source": [
    "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
    "word_embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i > MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        word_embedding_matrix[i] = embedding_vector\n",
    "\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(word_embedding_matrix, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Prepare training data tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of question1 data tensor: (404290, 25)\n",
      "Shape of question2 data tensor: (404290, 25)\n",
      "Shape of label tensor: (404290,)\n"
     ]
    }
   ],
   "source": [
    "q1_data = pad_sequences(question1_word_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "q2_data = pad_sequences(question2_word_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "labels = np.array(is_duplicate, dtype=int)\n",
    "print('Shape of question1 data tensor:', q1_data.shape)\n",
    "print('Shape of question2 data tensor:', q2_data.shape)\n",
    "print('Shape of label tensor:', labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Persist training and configuration data to files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.save(open(Q1_TRAINING_DATA_FILE, 'wb'), q1_data)\n",
    "np.save(open(Q2_TRAINING_DATA_FILE, 'wb'), q2_data)\n",
    "np.save(open(LABEL_TRAINING_DATA_FILE, 'wb'), labels)\n",
    "np.save(open(WORD_EMBEDDING_MATRIX_FILE, 'wb'), word_embedding_matrix)\n",
    "with open(NB_WORDS_DATA_FILE, 'w') as f:\n",
    "    json.dump({'nb_words': nb_words}, f)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}