project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Engineer Capstone"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Preprocessing\n",
    "\n",
    "The preprocessing routine searches the ./data/videos dir and creates usable datasets for subsequent model learning tasks. It breaks down videos into one second clips and from those clips generates frames, spectrograms, and InceptionV3 feature vectors. Currently clips and frames aren't used directly and spectrograms aren't used at all, but they are kept for labeling, debugging, and future model upgrades. The feature vector is the only output used for learning. Feature vectors are generated in preprocessing because it drastically reduced training time which allows for a faster model iteration cycle. This methodology was inspired from this repo and associated blog post: https://github.com/harvitronix/five-video-classification-methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51f1295a81ab485db361f75e3cd92499"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0531780cc1b4ee1a734637fd813c115"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a1554e9b7a74283907697a0303900f6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05bfb6868ec0495d9064bda9e7a392aa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "775789da421e4cb3a3085f6252e0bcb7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f205d0f450cc42948ca384e300d2f1f1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9eb1bb34449947fdb2c1a001d79943c8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eaa982c1038b4cf6979330ac62c43ecf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f4be4b05fd457e88a36ee4e4aff859"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68bdcb9d474444de8692a5577970cf95"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "592ac6ac0bc44f1fb0fb9edc5a43f1cb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cee864ae9764a4aba4036ca57d7e88c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0e66d685ecc477c9e61df060ad1e1a8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c4e6cd687a447b7ab4af9491b768541"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b89d10636d934dd2b4f2cfc29351bc9c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54212d6c31f6450c893c9d64c3329bbd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78523f7f2e414e2b8679b9bc9157ed32"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64090d8ca6d24705966956e04defae39"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53d1df6ad73849a0adc1be25fdb74df5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "917747f301824d86869985aa738f8d0e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1021587068d247a6b4e939046fec5213"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73bfca6f46284ebc860dc2c64f243ea0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de9b5c2c9a724b38a846364fe6535706"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1f422334adf4e078d81a92fa4252c04"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfe5c570dcaa42689c0673d2dbcc41df"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be8868f64fd44188e2c83684bb54395"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48c4a93b87114609a5b703d66c2ea14f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8eed28bac0b4dd5b14a64f52413317a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c97a4d6666e4cb0ab598c68d1ba963d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd4de055bfca4d43bfa631f0cff7eb0d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64b1daf987064ce8acbc986bd0b3c882"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5115cdd7b2f34871b4b3a18e5bdae3b6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8684f7f848c4f0a8adb4c16bdbcf385"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "727efaebb59244159870639211329b8b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d532675d3adf4582ba61b065a2857453"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00a929b42c38482abc6e6e965abb6f02"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "375b414b71004fefac4149b1a439a1e8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b026b33062d04ec8a0facf423d56f2c1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "121405f78ed6451092d24a07626e2ab0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2935c899680c4365855130fea007c014"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c68047ec77e249dea5a366b5149ca7ad"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "import subprocess\n",
    "import json\n",
    "import os\n",
    "import csv\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.inception_v3 import InceptionV3, preprocess_input\n",
    "from keras.models import Model, load_model\n",
    "from keras.layers import Input\n",
    "import numpy as np\n",
    "\n",
    "# Adapted from https://github.com/harvitronix/five-video-classification-methods\n",
    "class Extractor():\n",
    "    \"\"\"Extractor builds an inception model without the top classification \n",
    "    layers and extracts a feature array from an image.\"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        # Get model with pretrained weights.\n",
    "        base_model = InceptionV3(\n",
    "            weights='imagenet',\n",
    "            include_top=True\n",
    "        )\n",
    "\n",
    "        # We'll extract features at the final pool layer.\n",
    "        self.model = Model(\n",
    "            inputs=base_model.input,\n",
    "            outputs=base_model.get_layer('avg_pool').output\n",
    "        )\n",
    "\n",
    "    def extract(self, image_path):\n",
    "        img = image.load_img(image_path, target_size=(299, 299))\n",
    "        x = image.img_to_array(img)\n",
    "        x = np.expand_dims(x, axis=0)\n",
    "        x = preprocess_input(x)\n",
    "\n",
    "        # Get the prediction.\n",
    "        features = self.model.predict(x)\n",
    "        features = features[0]\n",
    "        return features\n",
    "\n",
    "def video_length(path):\n",
    "    \"\"\"returns the length of the video in secs\"\"\"\n",
    "    cmd = \"ffprobe -i \" + path + \" -show_entries format=duration -v quiet -of json\"\n",
    "    pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout\n",
    "    output = pipe.read()\n",
    "    d = json.loads(output)\n",
    "    s = d[\"format\"][\"duration\"]\n",
    "    return int(float(s))\n",
    "\n",
    "def video_id(path):\n",
    "    \"\"\"returns the id of a video from a path in this format: ./data/videos/:video_id\"\"\"\n",
    "    return path.split(\"/\")[3].split(\".\")[0]\n",
    "\n",
    "def clip_dir_path(path):\n",
    "    \"\"\"returns the path to dir containing all clips for a video ./data/clips/:video_id\"\"\"\n",
    "    vid_id = video_id(path)\n",
    "    return \"./data/clips/\" + vid_id\n",
    "\n",
    "def create_clips(path):\n",
    "    \"\"\"given a path to a video create_clips writes one sec video segments to disk \n",
    "    in the following format ./data/clips/:video_id/:clip_id.mp4\"\"\"\n",
    "    \n",
    "    # create clip dir\n",
    "    dir_path = clip_dir_path(path)\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "    \n",
    "    # create one sec clips from src\n",
    "    video_len = video_length(path)\n",
    "    for i in tqdm_notebook(xrange(video_len), desc=\"Clips for \" + video_id(path)):\n",
    "        clip_path = dir_path + \"/\" + '%05d' % i + \".mp4\"    \n",
    "        if not os.path.exists(clip_path):\n",
    "            cmd = \"ffmpeg -v error -y -i \" + path + \" -ss \" + str(i) + \" -t 1 \" + clip_path\n",
    "            os.system(cmd)\n",
    "\n",
    "def create_frames(path):\n",
    "    \"\"\"given a path to a video create_frames writes frames from previous generated \n",
    "    clips.  create_clips must be run before create_frames.  Frames are saved in the \n",
    "    following format ./data/frames/:video_id/:clip_id/:frame_id.jpg\"\"\"\n",
    "        \n",
    "    # create frame dir\n",
    "    vid_id = video_id(path)\n",
    "    dir_path = \"./data/frames/\" + vid_id\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "    \n",
    "    # create frames from clip\n",
    "    video_len = video_length(path)\n",
    "    for i in tqdm_notebook(xrange(video_len), desc=\"Frames for \" + vid_id):\n",
    "        clip_path = clip_dir_path(path) + \"/\" + '%05d' % i + \".mp4\"\n",
    "        frame_dir_path = dir_path + \"/\" + '%05d' % i\n",
    "        if not os.path.exists(frame_dir_path):\n",
    "            os.makedirs(frame_dir_path)\n",
    "            cmd = \"ffmpeg -v error -y -i \" + clip_path + \" -r 5.0 \" + frame_dir_path + \"/%5d.jpg\"\n",
    "            os.system(cmd)\n",
    "            \n",
    "            # resize frames to 299x299 for InceptionV3\n",
    "            frame_paths = glob.glob(frame_dir_path + \"/*.jpg\")\n",
    "            for fi in xrange(len(frame_paths)):\n",
    "                path = frame_paths[fi]\n",
    "                # resize first\n",
    "                cmd = \"convert \" + path + \" -resize 299x299 \" + path\n",
    "                os.system(cmd)\n",
    "                # add black background\n",
    "                cmd = \"convert \" + path + \" -gravity center -background black -extent 299x299 \" + path\n",
    "                os.system(cmd)\n",
    "\n",
    "def create_spectrograms(path):\n",
    "    \"\"\"given a path to a video create_spectrograms writes spectrograms from previous generated \n",
    "    clips.  create_clips must be run before create_spectrograms.  Spectrograms are saved in the \n",
    "    following format ./data/audio/:video_id/:clip_id.png\"\"\"\n",
    "    \n",
    "    # create audio dir\n",
    "    vid_id = video_id(path)\n",
    "    dir_path = \"./data/audio/\" + vid_id\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "    \n",
    "    # create spectrogram from clip\n",
    "    video_len = video_length(path)\n",
    "    for i in tqdm_notebook(xrange(video_len), desc=\"Spectrograms for \" + vid_id):\n",
    "        clip_path = clip_dir_path(path) + \"/\" + '%05d' % i + \".mp4\"\n",
    "        spec_path = dir_path + \"/\" + '%05d' % i + \".png\"\n",
    "        if not os.path.exists(spec_path):\n",
    "            cmd = \"ffmpeg -v error -y -i \" + clip_path + \" -lavfi showspectrumpic=s=32x32:legend=false \" + spec_path\n",
    "            os.system(cmd)\n",
    "\n",
    "\n",
    "extractor = Extractor()\n",
    "\n",
    "def create_features(path):\n",
    "    \"\"\"given a path to a video create_features writes inceptionV3 feature outputs from previous generated \n",
    "    clips.  create_clips and create_frames must be run before create_features.  Feature outputs are saved \n",
    "    in the following format ./data/features/:video_id/:clip_id/:frame_id.txt.gz\"\"\"\n",
    "        \n",
    "    # create feature dir\n",
    "    vid_id = video_id(path)\n",
    "    dir_path = \"./data/features/\" + vid_id\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)    \n",
    "    \n",
    "    # save feature array for every frame\n",
    "    video_len = video_length(path)    \n",
    "    with tqdm_notebook(total=video_len, desc=\"Features for \" + vid_id) as pbar:\n",
    "        for root, dirs, files in os.walk('./data/frames/'+ vid_id):\n",
    "            for f in files:\n",
    "                if f.endswith(\".jpg\"):\n",
    "                    frame_path = root + \"/\" + f\n",
    "                    feature_path = frame_path.replace(\"frames\", \"features\").replace(\"jpg\", \"txt.gz\")\n",
    "                    feature_dir = root.replace(\"frames\", \"features\")\n",
    "                    if not os.path.exists(feature_dir):\n",
    "                        os.makedirs(feature_dir)\n",
    "                    if not os.path.exists(feature_path):\n",
    "                        features = extractor.extract(frame_path)\n",
    "                        np.savetxt(feature_path, features)\n",
    "            pbar.update(1)\n",
    "\n",
    "# create assets from folder of videos.  This takes a LONG TIME.\n",
    "video_paths = glob.glob(\"./data/videos/*.mp4\")\n",
    "videos_len = len(video_paths)\n",
    "for i in tqdm_notebook(xrange(videos_len), desc=\"Preprocessing Videos\"):\n",
    "    path = video_paths[i]\n",
    "    create_clips(path)\n",
    "    create_frames(path)\n",
    "    create_spectrograms(path)\n",
    "    create_features(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Labels\n",
    "\n",
    "Labels are generated from the labelmaker's csv output of its internal sqlite database.  Labels are shuffled and divided into training, validation, and test sets at a ratio of roughly 3:1:1  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels Shape: (926, 3)\n",
      "Trainging Labels Shape: (555, 3)\n",
      "Validation Labels Shape: (185, 3)\n",
      "Test Labels Shape: (186, 3)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import numpy as np\n",
    "\n",
    "# read in and shuffle data\n",
    "labels = pd.read_csv(\"./labelmaker/labels.csv\").as_matrix()\n",
    "print \"Labels Shape: {}\".format(labels.shape)\n",
    "np.random.seed(0)\n",
    "np.random.shuffle(labels)\n",
    "\n",
    "# split labels into train, validation, and test sets\n",
    "div = len(labels) / 5\n",
    "train_labels = labels[0:div*3,:]\n",
    "val_labels = labels[div*3:div*4,:]\n",
    "test_labels = labels[div*4:,:]\n",
    "\n",
    "print \"Trainging Labels Shape: {}\".format(train_labels.shape)\n",
    "print \"Validation Labels Shape: {}\".format(val_labels.shape)\n",
    "print \"Test Labels Shape: {}\".format(test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model\n",
    "\n",
    "The Keras model is composed of a sequential model with two time sensitive LSTM layers followed by two Dense layers and an output layer.  The initial input of (7, 2048) represents seven frames per clip each with a 2048 sized vector generated by InceptionV3.  The final 4x1 output vector is the category prediction.     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Compiled\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, LSTM, Dropout, Flatten, GRU\n",
    "from keras import backend as K\n",
    "\n",
    "model = Sequential([\n",
    "    LSTM(512, return_sequences=True, input_shape=(7, 2048)),\n",
    "    LSTM(512, return_sequences=True, input_shape=(7, 512)),  \n",
    "    Flatten(),\n",
    "    Dense(512, activation='relu'),\n",
    "    Dropout(0.5),\n",
    "    Dense(512, activation='relu'),\n",
    "    Dropout(0.5),    \n",
    "    Dense(4, activation='softmax')\n",
    "])\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "print \"Model Compiled\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Match Labels\n",
    "\n",
    "This routine retrieves the features from disk and pairs them with their one hot encoded labels.  Currently all datasets are loaded into memory, but with enough videos, the code should be switched to using a Keras generator."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting features\n",
      "(555, 7, 2048)\n",
      "(555, 4)\n"
     ]
    }
   ],
   "source": [
    "def one_hot(i):\n",
    "    return np.array([int(i==0),int(i==1),int(i==2),int(i==3)])\n",
    "\n",
    "def get_features(labels):\n",
    "    x, y = [], []\n",
    "    for i in xrange(len(labels)):\n",
    "        video_id = labels[i][0]\n",
    "        clip_id = labels[i][1]\n",
    "        label = labels[i][2]\n",
    "\n",
    "        features = []\n",
    "        for i in range(7):\n",
    "            fname = \"./data/features/\" + video_id + \"/\" + '%05d' % clip_id + \"/\" + '%05d' % (i+1) + \".txt.gz\"\n",
    "            f = np.loadtxt(fname)\n",
    "            features.append(f)\n",
    "        \n",
    "        x.append(features)\n",
    "        y.append(one_hot(label))\n",
    "    x = np.array(x)\n",
    "    return x, np.array(y)\n",
    "\n",
    "print \"Getting features\"\n",
    "\n",
    "X_train, Y_train = get_features(train_labels)\n",
    "X_val, Y_val = get_features(val_labels)\n",
    "\n",
    "print X_train.shape\n",
    "print Y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "\n",
    "This routine trains the model and logs updates to the console and Tensorboard.  After training is complete the model is saved using the current timestamp to distinguish training runs.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 555 samples, validate on 185 samples\n",
      "Epoch 1/30\n",
      "9s - loss: 0.8594 - acc: 0.5694 - val_loss: 0.9663 - val_acc: 0.5784\n",
      "Epoch 2/30\n",
      "8s - loss: 0.8299 - acc: 0.5964 - val_loss: 1.0400 - val_acc: 0.5622\n",
      "Epoch 3/30\n",
      "10s - loss: 0.8315 - acc: 0.6216 - val_loss: 0.8450 - val_acc: 0.6054\n",
      "Epoch 4/30\n",
      "9s - loss: 0.7997 - acc: 0.6342 - val_loss: 0.9109 - val_acc: 0.5189\n",
      "Epoch 5/30\n",
      "8s - loss: 0.7696 - acc: 0.6126 - val_loss: 0.7708 - val_acc: 0.6865\n",
      "Epoch 6/30\n",
      "8s - loss: 0.7296 - acc: 0.6288 - val_loss: 0.7850 - val_acc: 0.6000\n",
      "Epoch 7/30\n",
      "9s - loss: 0.6696 - acc: 0.6595 - val_loss: 0.7394 - val_acc: 0.6486\n",
      "Epoch 8/30\n",
      "8s - loss: 0.6241 - acc: 0.6937 - val_loss: 0.7530 - val_acc: 0.6649\n",
      "Epoch 9/30\n",
      "8s - loss: 0.6476 - acc: 0.6865 - val_loss: 1.0272 - val_acc: 0.6324\n",
      "Epoch 10/30\n",
      "8s - loss: 0.6775 - acc: 0.6775 - val_loss: 0.7947 - val_acc: 0.6432\n",
      "Epoch 11/30\n",
      "8s - loss: 0.6557 - acc: 0.6739 - val_loss: 0.7841 - val_acc: 0.6541\n",
      "Epoch 12/30\n",
      "8s - loss: 0.5822 - acc: 0.7261 - val_loss: 0.8010 - val_acc: 0.6541\n",
      "Epoch 13/30\n",
      "9s - loss: 0.5048 - acc: 0.7622 - val_loss: 0.8615 - val_acc: 0.6865\n",
      "Epoch 14/30\n",
      "8s - loss: 0.5487 - acc: 0.7550 - val_loss: 0.9534 - val_acc: 0.6216\n",
      "Epoch 15/30\n",
      "8s - loss: 0.6183 - acc: 0.7495 - val_loss: 0.9482 - val_acc: 0.5622\n",
      "Epoch 16/30\n",
      "8s - loss: 0.5665 - acc: 0.7405 - val_loss: 0.7669 - val_acc: 0.7243\n",
      "Epoch 17/30\n",
      "9s - loss: 0.5163 - acc: 0.7676 - val_loss: 0.8521 - val_acc: 0.6919\n",
      "Epoch 18/30\n",
      "8s - loss: 0.4823 - acc: 0.7766 - val_loss: 1.2364 - val_acc: 0.6703\n",
      "Epoch 19/30\n",
      "8s - loss: 0.5358 - acc: 0.7676 - val_loss: 0.8454 - val_acc: 0.7081\n",
      "Epoch 20/30\n",
      "8s - loss: 0.4224 - acc: 0.8234 - val_loss: 0.7991 - val_acc: 0.7405\n",
      "Epoch 21/30\n",
      "9s - loss: 0.3806 - acc: 0.8324 - val_loss: 0.9672 - val_acc: 0.6486\n",
      "Epoch 22/30\n",
      "9s - loss: 0.4048 - acc: 0.8162 - val_loss: 0.8924 - val_acc: 0.6973\n",
      "Epoch 23/30\n",
      "9s - loss: 0.4221 - acc: 0.8198 - val_loss: 0.7659 - val_acc: 0.7297\n",
      "Epoch 24/30\n",
      "9s - loss: 0.3954 - acc: 0.8342 - val_loss: 0.8665 - val_acc: 0.7297\n",
      "Epoch 25/30\n",
      "8s - loss: 0.3279 - acc: 0.8631 - val_loss: 0.9752 - val_acc: 0.7081\n",
      "Epoch 26/30\n",
      "8s - loss: 0.3267 - acc: 0.8631 - val_loss: 1.2729 - val_acc: 0.6865\n",
      "Epoch 27/30\n",
      "9s - loss: 0.3667 - acc: 0.8486 - val_loss: 1.4754 - val_acc: 0.5676\n",
      "Epoch 28/30\n",
      "8s - loss: 0.4259 - acc: 0.8234 - val_loss: 0.9472 - val_acc: 0.6054\n",
      "Epoch 29/30\n",
      "8s - loss: 0.4125 - acc: 0.8072 - val_loss: 1.0666 - val_acc: 0.6973\n",
      "Epoch 30/30\n",
      "9s - loss: 0.5015 - acc: 0.7874 - val_loss: 0.8595 - val_acc: 0.7027\n",
      "Model Saved\n"
     ]
    }
   ],
   "source": [
    "from keras.callbacks import TensorBoard\n",
    "import time\n",
    "import numpy as np\n",
    "\n",
    "tensorboard = TensorBoard(log_dir='./logs', \n",
    "                          histogram_freq=0,\n",
    "                          write_graph=True, \n",
    "                          write_images=True)\n",
    "\n",
    "model.fit(X_train, \n",
    "          Y_train, \n",
    "          batch_size=100, \n",
    "          epochs=30, \n",
    "          verbose=2, \n",
    "          callbacks=[tensorboard], \n",
    "          validation_data=(X_val, Y_val))\n",
    "\n",
    "file_name = \"shot_classifier_\" + str(int(time.time())) + \".h5\"\n",
    "model.save(file_name)\n",
    "print \"Model Saved\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction\n",
    "\n",
    "This routine tests the saved model using the Keras predict method. Overall accuracy and a confusion matrix are displayed to validate that the model is accurate against unseen data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Accuracy: 0.672043010753\n",
      "predicted forehand when forehand 52\n",
      "predicted backhand when forehand 12\n",
      "predicted volley when forehand 1\n",
      "predicted serve when forehand 6\n",
      "predicted forehand when backhand 28\n",
      "predicted backhand when backhand 15\n",
      "predicted volley when backhand 0\n",
      "predicted serve when backhand 4\n",
      "predicted forehand when volley 5\n",
      "predicted backhand when volley 0\n",
      "predicted volley when volley 20\n",
      "predicted serve when volley 2\n",
      "predicted forehand when serve 3\n",
      "predicted backhand when serve 0\n",
      "predicted volley when serve 0\n",
      "predicted serve when serve 38\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def reverse_one_hot(val):\n",
    "    hi_idx = -1\n",
    "    hi = -1\n",
    "    for i in range(len(val)):\n",
    "        v = val[i]\n",
    "        if hi == -1 or v > hi:\n",
    "            hi = v\n",
    "            hi_idx = i\n",
    "    return hi_idx\n",
    "\n",
    "def normalize_labels(Y):\n",
    "    norm = []\n",
    "    for v in Y:\n",
    "        norm.append(reverse_one_hot(v))\n",
    "    return np.array(norm)\n",
    "\n",
    "X_test, Y_test = get_features(test_labels)\n",
    "model = load_model(\"shot_classifier_1501284149.h5\")\n",
    "Y_pred = model.predict(X_test, verbose=2)\n",
    "\n",
    "Y_test_norm = normalize_labels(Y_test)\n",
    "Y_pred_norm = normalize_labels(Y_pred)\n",
    "\n",
    "print \"Overall Accuracy: \" + str(accuracy_score(Y_test_norm, Y_pred_norm))\n",
    "\n",
    "con_m = confusion_matrix(Y_test_norm, Y_pred_norm)\n",
    "\n",
    "titles = [\"forehand\", \"backhand\", \"volley\", \"serve\"]\n",
    "\n",
    "for i in range(4):\n",
    "    for j in range(4):\n",
    "        actual = titles[i]\n",
    "        predicted = titles[j]\n",
    "        print \"predicted \" + predicted + \" when \" + actual + \" \" + str(con_m[i][j])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}