Google Speech Command Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wav2Letter Example using Google Speech Command Dataset\n",
    "\n",
    "Google Speech Command Dataset can be found [here](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge/data). This dataset was chosen as a quick and convenient way to test Wav2Letter performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Wav2Letter.data import GoogleSpeechCommand\n",
    "\n",
    "# using google's speech command dataset\n",
    "gs = GoogleSpeechCommand()\n",
    "_inputs, _targets = gs.load_vectors(\"./speech_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mfcc_features = 13\n",
    "grapheme_count = gs.intencode.grapheme_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "inputs = torch.Tensor(_inputs)\n",
    "targets = torch.IntTensor(_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64721, 225, 13])\n",
      "torch.Size([64721, 6])\n"
     ]
    }
   ],
   "source": [
    "print(inputs.shape)\n",
    "print(targets.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Conv1d(13, 250, kernel_size=(48,), stride=(2,))\n",
      "  (1): ReLU()\n",
      "  (2): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (3): ReLU()\n",
      "  (4): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (5): ReLU()\n",
      "  (6): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (7): ReLU()\n",
      "  (8): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (9): ReLU()\n",
      "  (10): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (11): ReLU()\n",
      "  (12): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (13): ReLU()\n",
      "  (14): Conv1d(250, 250, kernel_size=(7,), stride=(1,))\n",
      "  (15): ReLU()\n",
      "  (16): Conv1d(250, 2000, kernel_size=(32,), stride=(1,))\n",
      "  (17): ReLU()\n",
      "  (18): Conv1d(2000, 2000, kernel_size=(1,), stride=(1,))\n",
      "  (19): ReLU()\n",
      "  (20): Conv1d(2000, 25, kernel_size=(1,), stride=(1,))\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from Wav2Letter.model import Wav2Letter\n",
    "\n",
    "model = Wav2Letter(mfcc_features, grapheme_count)\n",
    "print(model.layers)\n",
    "\n",
    "ctc_loss = nn.CTCLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64721, 13, 225])\n"
     ]
    }
   ],
   "source": [
    "# Each mfcc feature is a channel\n",
    "# https://pytorch.org/docs/stable/nn.html#torch.nn.Conv1d\n",
    "# transpose (sample_size, in_frame_len, mfcc_features)\n",
    "# to      (sample_size, mfcc_features, in_frame_len)\n",
    "inputs = inputs.transpose(1, 2)\n",
    "print(inputs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 : step 1 / 253 , loss  6.584851264953613\n",
      "epoch 1 : step 51 / 253 , loss  2.7819130420684814\n",
      "epoch 1 : step 101 / 253 , loss  2.7523272037506104\n",
      "epoch 1 : step 151 / 253 , loss  2.6992950439453125\n",
      "epoch 1 : step 201 / 253 , loss  2.7544894218444824\n",
      "epoch 1 : step 251 / 253 , loss  2.7575273513793945\n",
      "epoch 1 average epoch loss 2.921410348575577\n",
      "epoch 2 : step 1 / 253 , loss  2.7081358432769775\n",
      "epoch 2 : step 51 / 253 , loss  2.744292736053467\n",
      "epoch 2 : step 101 / 253 , loss  2.740218162536621\n",
      "epoch 2 : step 151 / 253 , loss  2.6845641136169434\n",
      "epoch 2 : step 201 / 253 , loss  2.742398738861084\n",
      "epoch 2 : step 251 / 253 , loss  2.7068052291870117\n",
      "epoch 2 average epoch loss 2.7363557551689297\n",
      "epoch 3 : step 1 / 253 , loss  2.6324448585510254\n",
      "epoch 3 : step 51 / 253 , loss  2.629676103591919\n",
      "epoch 3 : step 101 / 253 , loss  2.5913217067718506\n",
      "epoch 3 : step 151 / 253 , loss  2.526848793029785\n",
      "epoch 3 : step 201 / 253 , loss  2.5802133083343506\n",
      "epoch 3 : step 251 / 253 , loss  2.567997455596924\n",
      "epoch 3 average epoch loss 2.6048956058713286\n",
      "epoch 4 : step 1 / 253 , loss  2.5266926288604736\n",
      "epoch 4 : step 51 / 253 , loss  2.3278656005859375\n",
      "epoch 4 : step 101 / 253 , loss  2.1569912433624268\n",
      "epoch 4 : step 151 / 253 , loss  2.0453405380249023\n",
      "epoch 4 : step 201 / 253 , loss  1.9862103462219238\n",
      "epoch 4 : step 251 / 253 , loss  1.9878559112548828\n",
      "epoch 4 average epoch loss 2.18664552053444\n",
      "epoch 5 : step 1 / 253 , loss  1.9454066753387451\n",
      "epoch 5 : step 51 / 253 , loss  1.895155668258667\n",
      "epoch 5 : step 101 / 253 , loss  1.7017391920089722\n",
      "epoch 5 : step 151 / 253 , loss  1.7114245891571045\n",
      "epoch 5 : step 201 / 253 , loss  1.5051336288452148\n",
      "epoch 5 : step 251 / 253 , loss  1.4553732872009277\n",
      "epoch 5 average epoch loss 1.722472463200686\n",
      "epoch 6 : step 1 / 253 , loss  1.5583523511886597\n",
      "epoch 6 : step 51 / 253 , loss  1.4968676567077637\n",
      "epoch 6 : step 101 / 253 , loss  1.2722609043121338\n",
      "epoch 6 : step 151 / 253 , loss  1.2771130800247192\n",
      "epoch 6 : step 201 / 253 , loss  1.0980217456817627\n",
      "epoch 6 : step 251 / 253 , loss  1.1346909999847412\n",
      "epoch 6 average epoch loss 1.2742997864960681\n",
      "epoch 7 : step 1 / 253 , loss  1.1743314266204834\n",
      "epoch 7 : step 51 / 253 , loss  1.051263451576233\n",
      "epoch 7 : step 101 / 253 , loss  1.0299192667007446\n",
      "epoch 7 : step 151 / 253 , loss  1.024586796760559\n",
      "epoch 7 : step 201 / 253 , loss  0.8652447462081909\n",
      "epoch 7 : step 251 / 253 , loss  0.9505209922790527\n",
      "epoch 7 average epoch loss 0.9918098751264127\n",
      "epoch 8 : step 1 / 253 , loss  0.9982552528381348\n",
      "epoch 8 : step 51 / 253 , loss  0.9333376288414001\n",
      "epoch 8 : step 101 / 253 , loss  0.8149896860122681\n",
      "epoch 8 : step 151 / 253 , loss  0.8949161767959595\n",
      "epoch 8 : step 201 / 253 , loss  0.7376797199249268\n",
      "epoch 8 : step 251 / 253 , loss  0.8255767822265625\n",
      "epoch 8 average epoch loss 0.8385873165997592\n",
      "epoch 9 : step 1 / 253 , loss  0.9135379195213318\n",
      "epoch 9 : step 51 / 253 , loss  0.7583141326904297\n",
      "epoch 9 : step 101 / 253 , loss  0.6958908438682556\n",
      "epoch 9 : step 151 / 253 , loss  0.748501718044281\n",
      "epoch 9 : step 201 / 253 , loss  0.6429460048675537\n",
      "epoch 9 : step 251 / 253 , loss  0.7574820518493652\n",
      "epoch 9 average epoch loss 0.7251375805247914\n",
      "epoch 10 : step 1 / 253 , loss  0.805584192276001\n",
      "epoch 10 : step 51 / 253 , loss  0.6238442063331604\n",
      "epoch 10 : step 101 / 253 , loss  0.599227786064148\n",
      "epoch 10 : step 151 / 253 , loss  0.6445375084877014\n",
      "epoch 10 : step 201 / 253 , loss  0.5687616467475891\n",
      "epoch 10 : step 251 / 253 , loss  0.6524989008903503\n",
      "epoch 10 average epoch loss 0.6352501038034915\n"
     ]
    }
   ],
   "source": [
    "# do short training run\n",
    "batch_size = 256\n",
    "model.fit(inputs, targets, optimizer, ctc_loss, batch_size, epoch=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([13, 225])\n"
     ]
    }
   ],
   "source": [
    "from Wav2Letter.decoder import GreedyDecoder\n",
    "\n",
    "sample = inputs[0]\n",
    "sample_target = targets[0]\n",
    "\n",
    "print(sample.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 8, 9, 0, 6, 1, 0, 0, 0, 1, 0, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_prob model.eval(sample)\n",
    "output = GreedyDecoder(log_prob)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([8, 9, 6, 1, 1, 1], dtype=torch.int32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Blank labels are 0, Pads are 1**\n",
    "\n",
    "**As you can see,  If you remove the 0's and the 1's from the output the model predicted the correct labels!**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}