experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio\n",
    "from tensorflow.python.ops import io_ops\n",
    "import warnings\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from graphviz import Digraph\n",
    "from imageio import imwrite\n",
    "import scipy.spatial as sp\n",
    "\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restore model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model(checkpoint,first_layer_filters=64, mfcc=True):\n",
    "    \n",
    "    tf.reset_default_graph()\n",
    "    sess = tf.Session()\n",
    "    \n",
    "    v = tf.get_variable(\"Variable\", shape=[20,8,1,first_layer_filters])\n",
    "    v1 = tf.get_variable(\"Variable_1\", shape=[first_layer_filters])\n",
    "    v2 = tf.get_variable(\"Variable_2\", shape=[10,4,first_layer_filters,64])\n",
    "    v3 = tf.get_variable(\"Variable_3\", shape=[64])\n",
    "    if mfcc:\n",
    "        v4 = tf.get_variable(\"Variable_4\", shape=[62720,12])\n",
    "    else:\n",
    "        v4 = tf.get_variable(\"Variable_4\", shape=[404544,12])\n",
    "    v5 = tf.get_variable(\"Variable_5\", shape=[12])\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    \n",
    "    saver.restore(sess, checkpoint)\n",
    "    first_weights = np.asarray(v.eval(session=sess))\n",
    "    first_bias = np.asarray(v1.eval(session=sess))\n",
    "    second_weights = np.asarray(v2.eval(session=sess))\n",
    "    second_bias = np.asarray(v3.eval(session=sess))\n",
    "    final_fc_weights = np.asarray(v4.eval(session=sess))\n",
    "    final_fc_bias = np.asarray(v5.eval(session=sess))\n",
    "    \n",
    "    weights = {'first_weights':first_weights, 'first_bias':first_bias,\n",
    "                 'second_weights':second_weights, 'second_bias':second_bias,\n",
    "                 'final_fc_weights':final_fc_weights, 'final_fc_bias':final_fc_bias}\n",
    "    \n",
    "    return {'session':sess, 'weights':weights}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate spectogram and MFCC from wav file\n",
    "Wav: input wave file\n",
    "\n",
    "Mode:\n",
    "- Set to \"original\" to display image as used in the networks\n",
    "- Set to \"enhanced\" to display a brightened enhanced version\n",
    "\n",
    "Plot: if true, show spectogram and MFCC plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wav_to_spectogram(wav, mode=\"original\", plot=False):\n",
    "\n",
    "    if mode == \"original\":\n",
    "        model_settings = {'dct_coefficient_count': 40, \n",
    "                          'window_size_samples': 480, \n",
    "                          'label_count': 12, \n",
    "                          'desired_samples': 16000, \n",
    "                          'window_stride_samples': 160, \n",
    "                          'spectrogram_length': 98, \n",
    "                          'sample_rate': 16000, \n",
    "                          'fingerprint_size': 3920}\n",
    "    else:\n",
    "        # settings from wav_to_spectrogram script from tensorflow tutorial\n",
    "        model_settings = {'dct_coefficient_count': 40, \n",
    "                          'window_size_samples': 256,\n",
    "                          'label_count': 12, \n",
    "                          'desired_samples': 16000, \n",
    "                          'window_stride_samples': 128, \n",
    "                          'spectrogram_length': 98, \n",
    "                          'sample_rate': 16000, \n",
    "                          'fingerprint_size': 3920}\n",
    "\n",
    "    # load file\n",
    "    desired_samples = model_settings['desired_samples']\n",
    "    wav_filename_placeholder_ = tf.placeholder(tf.string, [])\n",
    "    wav_loader = io_ops.read_file(wav_filename_placeholder_)\n",
    "    wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1, desired_samples=desired_samples)\n",
    "\n",
    "    # required placeholders for things we don't really use, here no values are set yet,\n",
    "    # they are just placeholders\n",
    "    foreground_volume_placeholder_ = tf.placeholder(tf.float32, [])\n",
    "    time_shift_padding_placeholder_ = tf.placeholder(tf.int32, [2, 2])\n",
    "    time_shift_offset_placeholder_ = tf.placeholder(tf.int32, [2])\n",
    "    background_data_placeholder_ = tf.placeholder(tf.float32, [desired_samples, 1])\n",
    "    background_volume_placeholder_ = tf.placeholder(tf.float32, [])\n",
    "    time_shift_padding_placeholder_ = tf.placeholder(tf.int32, [2, 2])\n",
    "    time_shift_offset_placeholder_ = tf.placeholder(tf.int32, [2])\n",
    "    scaled_foreground = tf.multiply(wav_decoder.audio, foreground_volume_placeholder_)\n",
    "    padded_foreground = tf.pad(scaled_foreground, time_shift_padding_placeholder_, mode='CONSTANT')\n",
    "    sliced_foreground = tf.slice(padded_foreground, time_shift_offset_placeholder_, [desired_samples, -1])\n",
    "    background_mul = tf.multiply(background_data_placeholder_, background_volume_placeholder_)\n",
    "    background_add = tf.add(background_mul, sliced_foreground)\n",
    "    background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)\n",
    "\n",
    "    # Run the spectrogram and MFCC ops to get a 2D 'fingerprint' of the audio.\n",
    "    spectrogram = contrib_audio.audio_spectrogram(\n",
    "        background_clamp,\n",
    "        window_size=model_settings['window_size_samples'],\n",
    "        stride=model_settings['window_stride_samples'],\n",
    "        magnitude_squared=True)\n",
    "\n",
    "    mfcc = contrib_audio.mfcc(\n",
    "        spectrogram,\n",
    "        wav_decoder.sample_rate,\n",
    "        dct_coefficient_count=model_settings['dct_coefficient_count'])\n",
    "\n",
    "    # set some paramters / settings for the spectrogram / mfcc\n",
    "    input_dict = {\n",
    "        wav_filename_placeholder_: input_wav, # path to file we want to analyze\n",
    "        time_shift_padding_placeholder_: [[0, 0], [0, 0]],\n",
    "        time_shift_offset_placeholder_: [0, 0],\n",
    "        background_data_placeholder_ : np.zeros([desired_samples, 1]), # no background noise\n",
    "        background_volume_placeholder_ : 0.0, # no background noise\n",
    "        foreground_volume_placeholder_ : 1.0 # don't silence the wav file\n",
    "    }\n",
    "\n",
    "    # run spectrogram and mfcc analysis, output is a numpy array\n",
    "    spectrogram_data = sess.run( spectrogram, feed_dict= input_dict)\n",
    "    mfcc_data = sess.run( mfcc, feed_dict= input_dict)\n",
    "\n",
    "    spectrogram_data_plot = spectrogram_data[0]\n",
    "    mfcc_data_plot = mfcc_data[0]\n",
    "\n",
    "    # Do some extra preprocessing to make the spectrogram more easy to read_file\n",
    "    # if the enhanced mode was chosen\n",
    "    if mode == \"enhanced\":\n",
    "        # normalize the array to the 0-255 range\n",
    "        spectrogram_data_plot *= 255.0 / spectrogram_data_plot.max()\n",
    "\n",
    "        # brighten it a bit\n",
    "        brightness = 3 # brighten by 500%\n",
    "        spectrogram_data_plot = spectrogram_data_plot * brightness\n",
    "\n",
    "        # clip back to [0, 255] range\n",
    "        spectrogram_data_plot = np.clip(spectrogram_data_plot, 0.0, 255.0)\n",
    "\n",
    "    if plot:\n",
    "        # init plots\n",
    "        print (\"\\nSpectrogram data spectrogram: %s\" % str(np.shape(spectrogram_data[0])))\n",
    "        print (\"MFCC data shape: %s\" % str(np.shape(mfcc_data[0])))\n",
    "        print (\"MFCC has 40 coefficients\")\n",
    "\n",
    "        input_time_size = spectrogram_data.shape[1]\n",
    "        input_frequency_size = spectrogram_data.shape[2]\n",
    "        fig2=plt.figure(figsize=(8, 20))\n",
    "        fig2.suptitle(\"Spectrogram of wav file\")\n",
    "        plt.xlabel('Time')\n",
    "        plt.ylabel('Frequency')\n",
    "        \n",
    "        # normalize\n",
    "        row_sums = spectrogram_data_plot.sum(axis=1)\n",
    "        norm = spectrogram_data_plot / row_sums[:, np.newaxis]\n",
    "\n",
    "        plt.imshow(np.rot90(norm), cmap='binary')\n",
    "        plt.xticks([i*input_time_size/10 for i in range(10)], range(0,1000,100))\n",
    "        plt.yticks([i*input_frequency_size/39.8 for i in range(40)], range(8000,0,-200))\n",
    "        \n",
    "    return mfcc_data, spectrogram_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create 4D fingerprint from MFCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_fingerprint(mfcc):\n",
    "    input_frequency_size = mfcc.shape[2] # model_settings['dct_coefficient_count']\n",
    "    input_time_size = mfcc.shape[1] #model_settings['spectrogram_length']\n",
    "    fingerprint_input = mfcc\n",
    "    fingerprint_4d = tf.reshape(fingerprint_input, [-1, input_time_size, input_frequency_size, 1])  \n",
    "    \n",
    "    return fingerprint_4d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model predict\n",
    "Input:\n",
    "- 4D fingerprint of an MFCC\n",
    "- Restored weights (dictionary) of a model\n",
    "\n",
    "Returns:\n",
    "- The predicted label\n",
    "- Probability of the predicted label (accuracy)\n",
    "- The activations of the first convolutional layer, ReLu and max pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(fingerprint_4d, weights):\n",
    "    # initialize weights\n",
    "    first_weights = weights[\"first_weights\"]\n",
    "    first_bias = weights[\"first_bias\"]\n",
    "    second_weights = weights[\"second_weights\"]\n",
    "    second_bias = weights[\"second_bias\"]\n",
    "    final_fc_weights = weights[\"final_fc_weights\"]\n",
    "    final_fc_bias = weights[\"final_fc_bias\"]\n",
    "\n",
    "    # first pass\n",
    "    first_conv = tf.nn.conv2d(fingerprint_4d, first_weights, [1, 1, 1, 1], 'SAME') + first_bias\n",
    "    first_relu = tf.nn.relu(first_conv)\n",
    "    max_pool = tf.nn.max_pool(first_relu, [1, 2, 2, 1], [1, 2, 2, 1], 'SAME')\n",
    "\n",
    "    # second pass\n",
    "    second_conv = tf.nn.conv2d(max_pool, second_weights, [1, 1, 1, 1], 'SAME') + second_bias\n",
    "    second_relu = tf.nn.relu(second_conv)\n",
    "    second_conv_shape = second_relu.get_shape()\n",
    "    second_conv_output_width = second_conv_shape[2]\n",
    "    second_conv_output_height = second_conv_shape[1]\n",
    "    second_conv_element_count = int(second_conv_output_width * second_conv_output_height * 64)\n",
    "    flattened_second_conv = tf.reshape(second_relu, [-1, second_conv_element_count])\n",
    "    final_fc = tf.matmul(flattened_second_conv, final_fc_weights) + final_fc_bias\n",
    "\n",
    "    # prediction\n",
    "    pred = tf.nn.softmax(final_fc).eval(session=sess)\n",
    "    labels = [\"_silence_\", \"_unknown_\", \"yes\", \"no\", \"up\", \"down\", \"left\", \"right\", \"on\", \"off\", \"stop\", \"go\"]\n",
    "    pred_label = labels[np.argmax(pred)]\n",
    "    accuracy = np.max(pred)\n",
    "    \n",
    "    return [pred_label, accuracy, first_conv, first_relu, max_pool]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>\n",
    "## Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restore model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from C:/python/asr/model_data/spectrogram_64f_36000s/speech_commands_train/conv.ckpt-36000\n"
     ]
    }
   ],
   "source": [
    "# Use MFCC with 64 filters in first layer\n",
    "\n",
    "# C:/python/asr/model_data/mfcc_64f_18000s/speech_commands_train/conv.ckpt-18000\n",
    "#restore = load_model(\"C:/python/asr/model_data/mfcc_64f_18000s/speech_commands_train/conv.ckpt-18000\", first_layer_filters=64)\n",
    "\n",
    "# 2, 4, 8\n",
    "restore = load_model(\"C:/python/asr/model_data/spectrogram_64f_36000s/speech_commands_train/conv.ckpt-36000\", first_layer_filters=64, mfcc=False)\n",
    "\n",
    "sess = restore[\"session\"]\n",
    "weights = restore[\"weights\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = []\n",
    "input_dir = 'D:/tmp/speech_dataset/on/'\n",
    "wavs = [f for f in listdir(input_dir) if isfile(join(input_dir, f))]\n",
    "\n",
    "# Predict each wav in a dictionary\n",
    "for wav in wavs:\n",
    "    input_wav = input_dir + wav\n",
    "    \n",
    "    mfcc, spec = wav_to_spectogram(input_wav)\n",
    "    fingerprint = create_fingerprint(spec)\n",
    "    label, accuracy, _, _, _ = predict(fingerprint, weights)\n",
    "    \n",
    "    predictions.append((wav, label, accuracy))\n",
    "    print(wav, label, accuracy)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize first convolutional layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_weights = weights[\"first_weights\"]\n",
    "\n",
    "fig = plt.figure(figsize=(10, 10))\n",
    "\n",
    "for i in range(first_weights.shape[3]):\n",
    "    fig.add_subplot(8, 8, i+1)\n",
    "    plt.imshow(np.rot90( first_weights[:,:,0,i] ), cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize second convolutional layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "second_weights = weights[\"second_weights\"]\n",
    "\n",
    "fig=plt.figure(figsize=(10, 10))\n",
    "\n",
    "channel = 0\n",
    "\n",
    "for i in range(first_weights.shape[3]):\n",
    "    fig.add_subplot(8, 8, i+1)\n",
    "    plt.imshow(second_weights[:,:,channel,i], cmap='gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_wav = 'D:/tmp/speech_dataset/on/39dce8cc_nohash_0.wav'\n",
    "mfcc, spec = wav_to_spectogram(input_wav, mode=\"original\", plot=True)\n",
    "\n",
    "mfcc = spec\n",
    "filters = 64\n",
    "\n",
    "fingerprint = create_fingerprint(mfcc)\n",
    "_, _, first_conv, first_relu, max_pool = predict(fingerprint, weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convolutional layer activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_conv = np.asarray(first_conv.eval(session=sess))\n",
    "\n",
    "input_time_size = mfcc.shape[1]\n",
    "input_frequency_size = mfcc.shape[2] \n",
    "\n",
    "# All activations\n",
    "fig=plt.figure(figsize=(10, 10))\n",
    "for i in range(filters):\n",
    "    fig.add_subplot(8, 8, i+1)\n",
    "    im = act_conv[0,:,:,i].reshape((input_time_size,input_frequency_size))\n",
    "    plt.imshow(im, cmap='gray')\n",
    "plt.show()\n",
    "\n",
    "# Single activation\n",
    "fig=plt.figure(figsize=(8, 20))\n",
    "im = act_conv[0,:,:,0].reshape((input_time_size,input_frequency_size))\n",
    "plt.imshow(np.rot90(im), cmap='jet')\n",
    "plt.colorbar(fraction=0.02)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ReLU activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_relu = np.asarray(first_relu.eval(session=sess))\n",
    "\n",
    "input_time_size = mfcc.shape[1]\n",
    "input_frequency_size = mfcc.shape[2] \n",
    "\n",
    "# All activations\n",
    "fig=plt.figure(figsize=(10, 10))\n",
    "for i in range(filters):\n",
    "    fig.add_subplot(8, 8, i+1)\n",
    "    im = act_relu[0,:,:,i].reshape((input_time_size,input_frequency_size))\n",
    "    plt.imshow(im, cmap='jet')\n",
    "plt.show()\n",
    "\n",
    "# Single activation\n",
    "for i in range(filters):\n",
    "    fig=plt.figure(figsize=(8, 20))\n",
    "    im = act_relu[0,:,:,i].reshape((input_time_size,input_frequency_size))\n",
    "    plt.imshow(np.rot90(im), cmap='jet', interpolation='none')\n",
    "    \n",
    "    # Show a nice axis\n",
    "    plt.xticks([i*input_time_size/10 for i in range(10)], range(0,1000,100))\n",
    "    plt.yticks([i*input_frequency_size/80 for i in range(80)], range(8000,0,-100))\n",
    "    \n",
    "    plt.colorbar(fraction=0.02)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Maxpool activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_maxpool = np.asarray(max_pool.eval(session=sess))\n",
    "\n",
    "input_time_size = mfcc.shape[1]\n",
    "input_frequency_size = mfcc.shape[2] \n",
    "\n",
    "# All activations\n",
    "fig=plt.figure(figsize=(10, 10))\n",
    "for i in range(filters):\n",
    "    fig.add_subplot(8, 8, i+1)\n",
    "    im = act_maxpool[0,:,:,i].reshape((int(np.ceil(input_time_size/2)),\n",
    "                                       int(np.ceil(input_frequency_size/2))))\n",
    "    plt.imshow(im, cmap='gray')\n",
    "plt.show()\n",
    "\n",
    "# All activations (big)\n",
    "fig=plt.figure(figsize=(10, 64))\n",
    "for i in range(filters):\n",
    "    fig.add_subplot(1, 2, i+1)\n",
    "    im = act_maxpool[0,:,:,i].reshape((int(np.ceil(input_time_size/2)),\n",
    "                                       int(np.ceil(input_frequency_size/2))))\n",
    "    plt.imshow(np.rot90(im), cmap='jet')\n",
    "    #plt.colorbar(fraction=0.02)\n",
    "plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hierarchy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First save filters of all models to images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = {}\n",
    "\n",
    "for model in [2,4,8,16,32,64]:\n",
    "    # Load model and restore first layer weights\n",
    "    restore = load_model(\"C:/python/asr/model_data/spectrogram_{}f_36000s/speech_commands_train/conv.ckpt-36000\".format(model),\n",
    "                        first_layer_filters=model, mfcc=False)\n",
    "    first_weights = restore[\"weights\"][\"first_weights\"]\n",
    "    \n",
    "    for i in range(first_weights.shape[3]):\n",
    "        f = first_weights[:,:,0,i] \n",
    "        \n",
    "        # Save the filter as matrix to a dictionary\n",
    "        filters['{}_{}'.format(model,i+1)] = f\n",
    "        \n",
    "        # Save weight as image for visualization\n",
    "        im = (255.*(f-f.min())/(f.max()-f.min())).astype(np.uint8)\n",
    "        imwrite('C:/python/asr/filters/spectrogram/{}/{}.png'.format(model,i+1), np.rot90(im))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save ReLU activations of all models for input wav"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Functions for recoloring activations\n",
    "\n",
    "def jetR(x):\n",
    "    x /= 255\n",
    "    return int(255.*plt.cm.jet(x)[0])\n",
    "\n",
    "def jetG(x):\n",
    "    x /= 255\n",
    "    return int(255.*plt.cm.jet(x)[1])\n",
    "\n",
    "def jetB(x):\n",
    "    x /= 255\n",
    "    return int(255.*plt.cm.jet(x)[2])\n",
    "\n",
    "vr = np.vectorize(jetR, otypes=[np.uint8])\n",
    "vg = np.vectorize(jetG, otypes=[np.uint8])\n",
    "vb = np.vectorize(jetB, otypes=[np.uint8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "    \n",
    "keyword = \"on\"\n",
    "files = [\"c842b5e4_nohash_1\",\"9a356ab9_nohash_0\",\"ab5b211a_nohash_0\",\"b665723d_nohash_0\",\"b21f0fa4_nohash_0\",\"30a09789_nohash_0\",\"8e05039f_nohash_1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fname in files:\n",
    "    activations = {}\n",
    "    input_wav = 'D:/tmp/speech_dataset/{}/{}.wav'.format(keyword,fname)\n",
    "    output_path = \"C:/python/asr/activations/{}/{}\".format(keyword,fname) + \"/{}/{}.png\"\n",
    "\n",
    "    for model in [2,4,8,16,32,64]:\n",
    "        # Load model and get activations\n",
    "        restore = load_model(\"C:/python/asr/model_data/spectrogram_{}f_36000s/speech_commands_train/conv.ckpt-36000\".format(model),\n",
    "                            first_layer_filters=model, mfcc=False)\n",
    "        sess = restore[\"session\"]\n",
    "        mfcc, spec = wav_to_spectogram(input_wav, mode=\"original\")\n",
    "        fingerprint = create_fingerprint(spec)\n",
    "        _, _, first_conv, first_relu, max_pool = predict(fingerprint, restore[\"weights\"])\n",
    "\n",
    "        act_relu = np.asarray(first_relu.eval(session=sess))\n",
    "        input_time_size = spec.shape[1]\n",
    "        input_frequency_size = spec.shape[2] \n",
    "\n",
    "        for i in range(act_relu.shape[3]):\n",
    "            f = act_relu[0,:,:,i].reshape((input_time_size,input_frequency_size))\n",
    "\n",
    "            # Save weight as image for visualization\n",
    "            im = (255.*(f-f.min())/(f.max()-f.min()) ).astype(np.uint8)\n",
    "\n",
    "            # colorize to jet\n",
    "            jet = np.stack([vr(im),vg(im),vb(im)], axis=2)\n",
    "            im = jet\n",
    "\n",
    "            path = \"C:/python/asr/activations/{}/{}\".format(keyword,fname) + \"/{}/\".format(model)\n",
    "            if not os.path.exists(path):\n",
    "                os.makedirs(path)\n",
    "\n",
    "            imwrite(output_path.format(model,i+1), np.rot90(im))     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Image-similarity measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sum of squared differences\n",
    "def SSE(A, B):\n",
    "    return np.square(np.subtract(A, B)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sum of absolute differences\n",
    "def SAE(A, B):\n",
    "    return np.absolute(np.subtract(A, B)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cosine similarity\n",
    "def cosine(A, B):\n",
    "    return (1 - sp.distance.cdist(A, B, 'cosine'))[0,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot a hierarchy graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dot = Digraph()\n",
    "\n",
    "# Similarity measure to use\n",
    "measure = SSE #SSE # requires argmin\n",
    "\n",
    "# Load all images as graph nodes\n",
    "for size in [64,32,16,8,4,2]:\n",
    "    for i in range(size):\n",
    "        dot.node('{}_{}'.format(size,i), image='C:/python/asr/filters/spectrogram/{}/{}.png'.format(size,i), \n",
    "                 label='\\n\\n{}'.format(i), style=\"setlinewidth(0)\")\n",
    "\n",
    "# For each parent-child layer\n",
    "for pair in [(2,4), (4,8), (8,16), (16,32), (32,64)]:\n",
    "    parent_n, child_n = pair\n",
    "    # For each child in the child layer\n",
    "    for child in range(1,child_n):\n",
    "        # Calculate similarity with all parents\n",
    "        similarities = [measure(filters['{}_{}'.format(child_n, child)],\n",
    "                                filters['{}_{}'.format(parent_n, i)]) for i in range(1,parent_n)]\n",
    "        # Choose the best parent\n",
    "        parent = 1 + np.argmin(similarities)\n",
    "        # Add graph edge from parent to child\n",
    "        dot.edge('{}_{}'.format(parent_n, parent),'{}_{}'.format(child_n, child))\n",
    "\n",
    "dot.render('{}_hierarchy_spectrogram.png'.format(measure.__name__), view=True)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Activation hierarchy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fname in files:\n",
    "    for measure in [SSE, SAE, cosine]:\n",
    "        dot = Digraph()\n",
    "\n",
    "        # Similarity measure to use\n",
    "        #measure = cosine #SSE # requires argmin\n",
    "\n",
    "        # Load all images as graph nodes\n",
    "        for size in [64,32,16,8,4,2]:\n",
    "            for i in range(size):\n",
    "                dot.node('{}_{}'.format(size,i), image='C:/python/asr/activations/{}/{}/{}/{}.png'.format(keyword,fname,size,i), \n",
    "                         label='\\n\\n\\t\\t\\t\\t{}'.format(i), style=\"setlinewidth(0)\")\n",
    "\n",
    "        # For each parent-child layer\n",
    "        for pair in [(2,4), (4,8), (8,16), (16,32), (32,64)]:\n",
    "            parent_n, child_n = pair\n",
    "            # For each child in the child layer\n",
    "            for child in range(1,child_n):\n",
    "                # Calculate similarity with all parents\n",
    "                similarities = [measure(filters['{}_{}'.format(child_n, child)],\n",
    "                                        filters['{}_{}'.format(parent_n, i)]) for i in range(1,parent_n)]\n",
    "                # Choose the best parent\n",
    "                parent = 1 + np.argmin(similarities)\n",
    "                # Add graph edge from parent to child\n",
    "                dot.edge('{}_{}'.format(parent_n, parent),'{}_{}'.format(child_n, child))\n",
    "\n",
    "        dot.render('C:/python/asr/activations/{}/{}/{}_activation_hierarchy.png'.format(keyword,fname,measure.__name__), view=False)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}