TF11 - Queues, Threads & Reading Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Main motivation of streamlining the process of reading data\n",
    "* TFRecords - efficient Tensorflow file format\n",
    "* We will learn how to read & write on TFRecords\n",
    "* Reading & Writing on queues\n",
    "* Multithreading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input Pipeline\n",
    "* For larger data, load chunks only when they are needed\n",
    "* Keep the rest in file system\n",
    "* Pre-processing of data is independent of training\n",
    "* TFRedocrds are suppose of handle such situation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.learn.python.learn.datasets import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_dir = \".\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting .\\train-images-idx3-ubyte.gz\n",
      "Extracting .\\train-labels-idx1-ubyte.gz\n",
      "Extracting .\\t10k-images-idx3-ubyte.gz\n",
      "Extracting .\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "data_sets = mnist.read_data_sets(save_dir, dtype= tf.uint8, reshape=False, validation_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 36],\n",
       "        [146],\n",
       "        [254],\n",
       "        [255],\n",
       "        [251],\n",
       "        [ 95],\n",
       "        [  6],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  3],\n",
       "        [ 97],\n",
       "        [234],\n",
       "        [254],\n",
       "        [254],\n",
       "        [232],\n",
       "        [254],\n",
       "        [254],\n",
       "        [ 35],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 89],\n",
       "        [140],\n",
       "        [254],\n",
       "        [254],\n",
       "        [174],\n",
       "        [ 67],\n",
       "        [ 33],\n",
       "        [200],\n",
       "        [254],\n",
       "        [190],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [108],\n",
       "        [253],\n",
       "        [254],\n",
       "        [235],\n",
       "        [ 51],\n",
       "        [  1],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 12],\n",
       "        [254],\n",
       "        [253],\n",
       "        [ 56],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 12],\n",
       "        [216],\n",
       "        [254],\n",
       "        [244],\n",
       "        [ 55],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  6],\n",
       "        [213],\n",
       "        [254],\n",
       "        [ 57],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 25],\n",
       "        [254],\n",
       "        [254],\n",
       "        [132],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [168],\n",
       "        [254],\n",
       "        [ 57],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 45],\n",
       "        [254],\n",
       "        [243],\n",
       "        [ 34],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [168],\n",
       "        [254],\n",
       "        [ 57],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [128],\n",
       "        [254],\n",
       "        [157],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [168],\n",
       "        [254],\n",
       "        [ 57],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 19],\n",
       "        [228],\n",
       "        [254],\n",
       "        [105],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  7],\n",
       "        [228],\n",
       "        [254],\n",
       "        [ 57],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 58],\n",
       "        [254],\n",
       "        [254],\n",
       "        [ 87],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 10],\n",
       "        [254],\n",
       "        [246],\n",
       "        [ 47],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 58],\n",
       "        [254],\n",
       "        [254],\n",
       "        [  9],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 10],\n",
       "        [254],\n",
       "        [210],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 58],\n",
       "        [254],\n",
       "        [254],\n",
       "        [  9],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [105],\n",
       "        [254],\n",
       "        [ 91],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  5],\n",
       "        [219],\n",
       "        [254],\n",
       "        [  9],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 24],\n",
       "        [230],\n",
       "        [254],\n",
       "        [ 24],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [216],\n",
       "        [254],\n",
       "        [  9],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 84],\n",
       "        [254],\n",
       "        [251],\n",
       "        [ 23],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [216],\n",
       "        [254],\n",
       "        [ 36],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 22],\n",
       "        [208],\n",
       "        [251],\n",
       "        [ 94],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [129],\n",
       "        [254],\n",
       "        [120],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  3],\n",
       "        [140],\n",
       "        [254],\n",
       "        [229],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 83],\n",
       "        [254],\n",
       "        [222],\n",
       "        [ 17],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 91],\n",
       "        [254],\n",
       "        [236],\n",
       "        [ 53],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 18],\n",
       "        [235],\n",
       "        [254],\n",
       "        [134],\n",
       "        [ 21],\n",
       "        [119],\n",
       "        [237],\n",
       "        [254],\n",
       "        [124],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [ 53],\n",
       "        [249],\n",
       "        [254],\n",
       "        [234],\n",
       "        [252],\n",
       "        [254],\n",
       "        [172],\n",
       "        [  3],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [116],\n",
       "        [237],\n",
       "        [254],\n",
       "        [254],\n",
       "        [133],\n",
       "        [ 20],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]],\n",
       "\n",
       "       [[  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0],\n",
       "        [  0]]], dtype=uint8)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sets[0].images[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_splits = [\"train\",\"test\",\"validation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving train\n",
      "saving test\n",
      "saving validation\n"
     ]
    }
   ],
   "source": [
    "for d in range(3):\n",
    "    print (\"saving \" + data_splits[d])\n",
    "    data_set = data_sets[d]\n",
    "    \n",
    "    filename = os.path.join(save_dir, data_splits[d] + '.tfrecords')\n",
    "    writer = tf.python_io.TFRecordWriter(filename)\n",
    "    for index in range(data_set.images.shape[0]):\n",
    "        image = data_set.images[index].tostring()\n",
    "        example = tf.train.Example(features = tf.train.Features(feature={\n",
    "            'height':tf.train.Feature(int64_list = tf.train.Int64List(value=[data_set.images.shape[1]])),\n",
    "            'width':tf.train.Feature(int64_list = tf.train.Int64List(value=[data_set.images.shape[2]])),\n",
    "            'depth':tf.train.Feature(int64_list = tf.train.Int64List(value=[data_set.images.shape[3]])),\n",
    "            'label':tf.train.Feature(int64_list = tf.train.Int64List(value=[data_set.labels[index]])),\n",
    "            'raw_image':tf.train.Feature(bytes_list = tf.train.BytesList(value=[image]))\n",
    "            \n",
    "        }))\n",
    "        \n",
    "        writer.write(example.SerializeToString())\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59000, 28, 28, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sets[0].images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reader = tf.TFRecordReader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = os.path.join(save_dir,\"train.tfrecords\")\n",
    "filename_queue = tf.train.string_input_producer([filename],num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, serialized_data = reader.read(filename_queue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ReaderReadV2_1:1' shape=() dtype=string>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "serialized_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tf.parse_single_example(serialized_data, features={\n",
    "    'image_raw': tf.FixedLenFeature([],tf.string),\n",
    "    'label':tf.FixedLenFeature([],tf.int64)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'image_raw': <tf.Tensor 'ParseSingleExample/Squeeze_image_raw:0' shape=() dtype=string>,\n",
       " 'label': <tf.Tensor 'ParseSingleExample/Squeeze_label:0' shape=() dtype=int64>}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59000,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sets[0].labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Queue Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "#shapes - telling the size of items in advance\n",
    "queue1 = tf.FIFOQueue(capacity=10, dtypes=[tf.string],shapes=[()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enque_op = queue1.enqueue([\"Z\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(queue1.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enque_op.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(queue1.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enque_op = queue1.enqueue([\"E\"])\n",
    "enque_op.run()\n",
    "enque_op = queue1.enqueue([\"K\"])\n",
    "enque_op.run()\n",
    "enque_op = queue1.enqueue([\"E\"])\n",
    "enque_op.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(queue1.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = queue1.dequeue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'Z'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'E'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = queue1.dequeue()\n",
    "x.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'K'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = queue1.dequeue()\n",
    "x.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enque_op = queue1.enqueue([\"E\"])\n",
    "enque_op.run()\n",
    "enque_op = queue1.enqueue([\"K\"])\n",
    "enque_op.run()\n",
    "enque_op = queue1.enqueue([\"E\"])\n",
    "enque_op.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputs = queue1.dequeue_many(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'E', b'K'], dtype=object)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Tensorflow session is multi threaded\n",
    "#Multiple threads can use the same session\n",
    "#Run ops in parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "genral_random_normal = tf.random_normal(shape=())\n",
    "\n",
    "queue = tf.FIFOQueue(capacity=100, dtypes=[tf.float32], shapes=())\n",
    "enque = queue.enqueue(genral_random_normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add():\n",
    "    for i in range(10):\n",
    "        sess.run(enque)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threads = [threading.Thread(target=add, args=()) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for t in threads:\n",
    "    t.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(queue.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.91700041,  1.06900203, -1.25437331, -0.54801881, -0.38989627,\n",
       "       -0.46762475,  0.88529086, -1.05124879, -0.81140858,  1.25654447,\n",
       "       -0.01337299, -1.50150537,  0.52652627,  0.16372143, -0.4275195 ,\n",
       "        0.19320062, -0.77131397,  0.45199049, -0.96952766, -0.58470839,\n",
       "       -1.97644472, -2.25858378, -0.33022475, -0.54432708,  0.88813865,\n",
       "        0.298397  , -0.48165751, -1.10197115, -0.84567058, -0.50833195,\n",
       "        0.07419065,  0.34973881, -0.06683222, -0.47225034, -1.27323937,\n",
       "       -0.88559961, -0.85171402,  1.86602545, -0.11018598,  0.29020962,\n",
       "        0.81564206, -1.16399121, -1.08930326, -1.00915325,  0.87850976,\n",
       "       -0.09321502, -0.49639085,  0.47721642,  0.18031943,  1.38161182], dtype=float32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = queue.dequeue_many(50)\n",
    "x.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gen_random_normal = tf.random_normal(shape=())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "queue = tf.RandomShuffleQueue(capacity=100, dtypes=[tf.float32], min_after_dequeue=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enqueue_op = queue.enqueue(gen_random_normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "qr = tf.train.QueueRunner(queue,[enqueue_op] * 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord = tf.train.Coordinator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "queue_threads = qr.create_threads(sess, coord=coord, start=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord.request_stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord.join(queue_threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reader = tf.TFRecordReader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename_q = tf.train.string_input_producer([\"train.tfrecords\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ser  = reader.read(filename_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ser = ser[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = tf.parse_single_example(ser, features={\n",
    "    'image_raw': tf.FixedLenFeature([], tf.string),\n",
    "    'label' : tf.FixedLenFeature([], tf.int64)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Coordinator & QueueRunner\n",
    "* Coordinator for coordinating the termination of a set of threads\n",
    "* QueueRunner for getting multiple threads enqueue data to same queue seamlessly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gen_random_normal = tf.random_normal(shape=())\n",
    "queue = tf.FIFOQueue(capacity=100, dtypes=[tf.float32],shapes=())\n",
    "enque = queue.enqueue(genral_random_normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add(coord,i):\n",
    "    while not coord.should_stop():\n",
    "        sess.run(enque)\n",
    "        if i == 11:\n",
    "            coord.request_stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord = tf.train.Coordinator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threads = [threading.Thread(target=add, args=(coord,i)) for i in range(10)]\n",
    "coord.join(threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for t in threads:\n",
    "    t.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(queue.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gen_random_normal = tf.random_normal(shape=())\n",
    "queue = tf.RandomShuffleQueue(capacity=100, dtypes=[tf.float32],min_after_dequeue=1)\n",
    "enque_op = queue.enqueue(genral_random_normal)\n",
    "enque_op2 = queue.enqueue(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rq = tf.train.QueueRunner(queue, [enque_op, enque_op2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coord = tf.train.Coordinator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "threads = rq.create_threads(sess,coord=coord, start=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rq."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Operation 'random_shuffle_queue_enqueue' type=QueueEnqueueV2>,\n",
       " <tf.Operation 'random_shuffle_queue_enqueue' type=QueueEnqueueV2>,\n",
       " <tf.Operation 'random_shuffle_queue_enqueue' type=QueueEnqueueV2>,\n",
       " <tf.Operation 'random_shuffle_queue_enqueue' type=QueueEnqueueV2>,\n",
       " <tf.Operation 'random_shuffle_queue_enqueue' type=QueueEnqueueV2>]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[enque_op] * 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Thread(Thread-38, started 19636)>,\n",
       " <Thread(Thread-39, started 6140)>,\n",
       " <Thread(Thread-40, started 7696)>]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}