GMM_HMM_1_iteration_multiple_input_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import string\n",
    "from scipy.misc import logsumexp\n",
    "from sklearn import cluster\n",
    "from sklearn.base import BaseEstimator, _pprint\n",
    "from sklearn.utils import check_array, check_random_state\n",
    "from sklearn.utils.validation import check_is_fitted\n",
    "\n",
    "from sklearn.mixture import(distribute_covar_matrix_to_match_covariance_type, _validate_covars)\n",
    "from sklearn.utils import check_random_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_seq_serial(fname):\n",
    "    # Serial Implementation\n",
    "\n",
    "    df = pd.read_csv(fname)\n",
    "    seq = list(df['Data'])\n",
    "    seq = np.array(seq)\n",
    "\n",
    "    return seq\n",
    "\n",
    "def log_mask_zero(a):\n",
    "    \n",
    "    a = np.asarray(a)\n",
    "    with np.errstate(divide=\"ignore\"):\n",
    "        return np.log(a)\n",
    "\n",
    "def normalize(a, axis=None):\n",
    "    \"\"\"Normalizes the input array so that it sums to 1.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    a : array\n",
    "        Non-normalized input data.\n",
    "\n",
    "    axis : int\n",
    "        Dimension along which normalization is performed.\n",
    "\n",
    "    Notes\n",
    "    -----\n",
    "    Modifies the input **inplace**.\n",
    "    \"\"\"\n",
    "    a_sum = a.sum(axis)\n",
    "    if axis and a.ndim > 1:\n",
    "        # Make sure we don't divide by zero.\n",
    "        a_sum[a_sum == 0] = 1\n",
    "        shape = list(a.shape)\n",
    "        shape[axis] = 1\n",
    "        a_sum.shape = shape\n",
    "\n",
    "    a /= a_sum\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fname = 'no_blink.csv'\n",
    "\n",
    "sequences = np.atleast_2d(get_seq_serial(fname))\n",
    "\n",
    "X_input = sequences.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.614286,  0.528571],\n",
       "       [ 0.457143,  0.528571],\n",
       "       [ 0.6     ,  0.728571],\n",
       "       [ 0.5     ,  0.657143],\n",
       "       [ 0.757143,  0.842857]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = 10\n",
    "\n",
    "X = X_input[i:i+5]\n",
    "# X = check_array(X)\n",
    "\n",
    "X = np.hstack([X,X_input[i+5:i+10]]) # 2 features\n",
    "\n",
    "X = check_array(X)\n",
    "\n",
    "X \n",
    "\n",
    "# X.shape\n",
    "\n",
    "# type(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## _BaseHMM class initialization parameters\n",
    "\n",
    "n_components=2\n",
    "covariance_type=\"diag\"\n",
    "n_iter=1000\n",
    "startprob_prior = 1.0\n",
    "transmat_prior=1.0\n",
    "random_state=None\n",
    "tol=1e-2 \n",
    "verbose=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## GMMHMM class initialization parameters\n",
    "\n",
    "covariance_type = \"diag\"\n",
    "min_covar = 1e-3\n",
    "n_mix = 2\n",
    "weights_prior = 1.0\n",
    "means_prior = 0.0\n",
    "means_weight = 0.0\n",
    "covars_prior = None\n",
    "covars_weight = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# init\n",
    "\n",
    "init = 1. / n_components\n",
    "\n",
    "init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.5,  0.5])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "startprob_ = np.full(n_components, init)   # pi, initial probability\n",
    "\n",
    "startprob_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5,  0.5],\n",
       "       [ 0.5,  0.5]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transmat_ = np.full((n_components, n_components),init)  # Transition matrix (A) or Hidden Layer\n",
    "\n",
    "transmat_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples, n_features = X.shape\n",
    "\n",
    "## Calling _init_covar_priors() function , covariance_type = \"diag\"\n",
    "covars_prior = -1.5  \n",
    "covars_weight = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.,  1.],\n",
       "       [ 1.,  1.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# weights_prior : array, shape (n_mix, ), optional\n",
    "#               Parameters of the Dirichlet prior distribution for\n",
    "#               :attr:`weights_`.\n",
    "\n",
    "weights_prior = np.broadcast_to(weights_prior, (n_components, n_mix)).copy()\n",
    "\n",
    "weights_prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "means_prior : \n",
      "[[[ 0.  0.]\n",
      "  [ 0.  0.]]\n",
      "\n",
      " [[ 0.  0.]\n",
      "  [ 0.  0.]]]\n",
      "means_weight : \n",
      "[[ 0.  0.]\n",
      " [ 0.  0.]]\n",
      "means_ : \n",
      "[[[ 0.  0.]\n",
      "  [ 0.  0.]]\n",
      "\n",
      " [[ 0.  0.]\n",
      "  [ 0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "# means_prior, means_weight : array, shape (n_mix, ), optional\n",
    "#         Mean and precision of the Normal prior distribtion for\n",
    "#         :attr:`means_`.\n",
    "\n",
    "means_prior = np.broadcast_to(means_prior,(n_components, n_mix, n_features)).copy()\n",
    "\n",
    "print \"means_prior : \"\n",
    "print means_prior\n",
    "\n",
    "means_weight = np.broadcast_to(means_weight,(n_components, n_mix)).copy()   # C matrix / Weight matrix\n",
    "\n",
    "print \"means_weight : \"\n",
    "print means_weight\n",
    "\n",
    "means_ = np.zeros((n_components, n_mix,n_features))\n",
    "\n",
    "print \"means_ : \"\n",
    "print means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covars_prior\n",
      "[[[-1.5 -1.5]\n",
      "  [-1.5 -1.5]]\n",
      "\n",
      " [[-1.5 -1.5]\n",
      "  [-1.5 -1.5]]]\n",
      "covars_weight\n",
      "[[[ 0.  0.]\n",
      "  [ 0.  0.]]\n",
      "\n",
      " [[ 0.  0.]\n",
      "  [ 0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "covars_prior = np.broadcast_to(covars_prior,(n_components, n_mix, n_features)).copy()\n",
    "\n",
    "print \"covars_prior\"\n",
    "print covars_prior\n",
    "\n",
    "covars_weight = np.broadcast_to(covars_weight,(n_components, n_mix, n_features)).copy()\n",
    "\n",
    "print \"covars_weight\"\n",
    "print covars_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1], dtype=int32)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Kmeans\n",
    "\n",
    "main_kmeans = cluster.KMeans(n_clusters=n_components, random_state=random_state)\n",
    "labels = main_kmeans.fit_predict(X)\n",
    "\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeanses = []\n",
    "\n",
    "for label in range(n_components):\n",
    "    kmeans = cluster.KMeans(n_clusters = n_mix,random_state = random_state)\n",
    "    kmeans.fit(X[np.where(labels == label)])\n",
    "    kmeanses.append(kmeans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5,  0.5],\n",
       "       [ 0.5,  0.5]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_ = (np.ones((n_components, n_mix)) /(np.ones((n_components, 1)) * n_mix))\n",
    "\n",
    "weights_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.4785715,  0.592857 ],\n",
       "        [ 0.614286 ,  0.528571 ]],\n",
       "\n",
       "       [[ 0.757143 ,  0.842857 ],\n",
       "        [ 0.6      ,  0.728571 ]]])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i, kmeans in enumerate(kmeanses):\n",
    "    means_[i] = kmeans.cluster_centers_\n",
    "\n",
    "means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.01457144,  0.01916329],\n",
       "        [ 0.01457144,  0.01916329]],\n",
       "\n",
       "       [[ 0.01457144,  0.01916329],\n",
       "        [ 0.01457144,  0.01916329]]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = np.cov(X.T) + min_covar*np.eye(n_features)\n",
    "if not cv.shape:\n",
    "    cv.shape = (1, 1)\n",
    "        \n",
    "covars_ = np.zeros((n_components, n_mix,n_features))\n",
    "covars_[:] = np.diag(cv)\n",
    "\n",
    "covars_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Univariate Gaussian\n",
    "\n",
    "### $P(X;\\mu,\\sigma^{2})$ = $ \\frac{1}{\\sqrt{2\\pi}\\sigma}\\exp\\Bigl(-\\frac{1}{2\\sigma^{2}}(x-\\mu)^{2}\\Bigr)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _log_multivariate_normal_density_diag(X, means, covars):\n",
    "    \"\"\"Compute Gaussian log-density at X for a diagonal model.\"\"\"\n",
    "    n_samples, n_dim = X.shape\n",
    "    lpr = -0.5 * (n_dim * np.log(2 * np.pi) + np.sum(np.log(covars), 1)\n",
    "                  + np.sum((means ** 2) / covars, 1)\n",
    "                  - 2 * np.dot(X, (means / covars).T)\n",
    "                  + np.dot(X ** 2, (1.0 / covars).T))\n",
    "    return lpr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gaussian Mixture Model :\n",
    "\n",
    "### $P(X;\\mu,\\sigma^{2})$ = $w_{1}*P(X_{1};\\mu_{1},\\sigma_{1}^{2})$ + $w_{2}*P(X_{2};\\mu_{2},\\sigma_{2}^{2})$\n",
    "### $\\sum_{m=1}^M w_{k} = 1$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _compute_log_weighted_gaussian_densities(X, i_comp):\n",
    "    cur_means = means_[i_comp]\n",
    "    cur_covs = covars_[i_comp]\n",
    "    log_cur_weights = np.log(weights_[i_comp])\n",
    "\n",
    "    return _log_multivariate_normal_density_diag(X, cur_means, cur_covs) + log_cur_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _compute_log_likelihood(X):\n",
    "    n_samples, _ = X.shape\n",
    "    res = np.zeros((n_samples, n_components))\n",
    "    for i in range(n_components):\n",
    "        log_denses = _compute_log_weighted_gaussian_densities(X, i)\n",
    "#         print log_denses.T\n",
    "        res[:, i] = logsumexp(log_denses, axis=1)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Propogation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.95084532,  1.83248274,  1.2357735 ,  1.82209547, -1.40781709],\n",
       "       [ 0.61247574, -0.16362164,  1.82674396,  1.14996083,  1.82674396]])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "framelogprob = _compute_log_likelihood(X)  # B matrix, or output probabilities \n",
    "\n",
    "framelogprob.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "log_startprob = log_mask_zero(startprob_)\n",
    "log_transmat = log_mask_zero(transmat_)\n",
    "\n",
    "n_samples, n_components = framelogprob.shape  # length * 2 pr 5 * 2\n",
    "\n",
    "fwdlattice = np.zeros((n_samples, n_components))  # alpha\n",
    "\n",
    "work_buffer = np.zeros(n_components)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization \n",
    "### $\\alpha_{1}(i)$  = $\\pi_{i}b_{i}(O_{1})$   1<=i<=N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.25769814,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [-0.08067144,  0.        ,  0.        ,  0.        ,  0.        ]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(n_components):\n",
    "    fwdlattice[0, i] = log_startprob[i] + framelogprob[0, i]\n",
    "    \n",
    "fwdlattice.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Induction\n",
    "\n",
    "### $\\alpha_{t+1}(j)$  = $\\Bigl[\\sum_{i=1}^N\\alpha_{t}(i)a_{ij}\\Bigr]b_{j}O_{(t+1)}$   \n",
    "1<=t<=T-1  <br> 1<=j<=N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.25769814  2.62994771  3.29996721  5.4605828   3.77213699]\n",
      " [-0.08067144  0.63384333  3.89093767  4.78844817  7.00669804]]\n"
     ]
    }
   ],
   "source": [
    "for t in range(1, n_samples):\n",
    "    for j in range(n_components):\n",
    "        for i in range(n_components):\n",
    "            work_buffer[i] = fwdlattice[t - 1, i] + log_transmat[i, j]\n",
    "        fwdlattice[t, j] = logsumexp(work_buffer) + framelogprob[t, j]\n",
    "        \n",
    "print fwdlattice.T\n",
    "\n",
    "# print np.exp(fwdlattice).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Termination : Forward Propogation\n",
    "### $P(O|\\lambda)$ = $\\sum_{i=1}^N\\alpha_{T}(i)$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backward Propogation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_samples, n_components = framelogprob.shape\n",
    "\n",
    "bwdlattice = np.zeros((n_samples, n_components))\n",
    "\n",
    "work_buffer = np.zeros(n_components)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization :\n",
    "\n",
    "### $\\beta_{T}(i) = 1 $   \n",
    "\n",
    "$1<=i<=N$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(n_components):\n",
    "    bwdlattice[n_samples - 1, i] = 0.0    ## Log(1) = 0\n",
    "    \n",
    "bwdlattice.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Induction\n",
    "### $\\beta_{t}(i)$ = $\\sum_{j=1}^N a_{ij}b_{j}(O_{t+1})\\beta_{t+1}(j)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 5.55470785,  4.28797911,  2.71368549,  1.17221874,  0.        ],\n",
       "       [ 5.55470785,  4.28797911,  2.71368549,  1.17221874,  0.        ]])"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for t in range(n_samples - 2, -1, -1):\n",
    "    for i in range(n_components):\n",
    "        for j in range(n_components):\n",
    "            work_buffer[j] = (log_transmat[i, j]\n",
    "                              + framelogprob[t + 1, j]\n",
    "                              + bwdlattice[t + 1, j])\n",
    "        bwdlattice[t, i] = logsumexp(work_buffer)\n",
    "        \n",
    "bwdlattice.T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## $Posteriors$ : $\\alpha(t)*\\beta(t)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.79222169,  0.20777831],\n",
       "       [ 0.88038746,  0.11961254],\n",
       "       [ 0.35641222,  0.64358778],\n",
       "       [ 0.66198097,  0.33801903],\n",
       "       [ 0.03788564,  0.96211436]])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# _compute_posteriors\n",
    "\n",
    "log_gamma = fwdlattice + bwdlattice\n",
    "log_normalize(log_gamma, axis=1)  \n",
    "posteriors = np.exp(log_gamma)  \n",
    "\n",
    "posteriors          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stats = {'nobs': 0,\n",
    "         'start': np.zeros(n_components),\n",
    "         'trans': np.zeros((n_components, n_components))}\n",
    "\n",
    "stats['post'] = np.zeros(n_components)\n",
    "stats['obs']  = np.zeros((n_components, n_features))\n",
    "stats['obs**2'] = np.zeros((n_components, n_features))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## $\\gamma_{t}(i)$ = $\\frac{ \\alpha_{t}(i)\\beta_{t}(i)}{\\sum_{i=1}^N \\alpha_{t}(i)\\beta_{t}(i)}$\n",
    "\n",
    "## $\\pi_{i} = \\gamma_{1}(i) $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stats['start'] = posteriors[0]\n",
    "\n",
    "n_samples, n_components = framelogprob.shape\n",
    "log_xi_sum = np.full((n_components, n_components), -np.inf)\n",
    "prob_mix = np.zeros((n_samples, n_components, n_mix))\n",
    "\n",
    "stats['n_samples'] = n_samples  # 5\n",
    "stats['samples'] = X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 2.2724724 ,  4.76215917],\n",
       "        [ 0.17964221,  1.66535126]],\n",
       "\n",
       "       [[ 4.2085438 ,  2.04083923],\n",
       "        [ 0.01649425,  0.83256896]],\n",
       "\n",
       "       [[ 1.77568789,  1.66535126],\n",
       "        [ 1.4514627 ,  4.76215917]],\n",
       "\n",
       "       [[ 4.2085438 ,  1.97626112],\n",
       "        [ 0.2002683 ,  2.95780092]],\n",
       "\n",
       "       [[ 0.06503459,  0.17964221],\n",
       "        [ 4.76215917,  1.4514627 ]]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Real output, B->O per weight value. Not logarithmic value Dimensions T*N*M\n",
    "\n",
    "for p in range(n_components):\n",
    "    log_denses = _compute_log_weighted_gaussian_densities(X, p)   \n",
    "    prob_mix[:,p,:] = np.exp(log_denses) + np.finfo(np.float).eps  \n",
    "    \n",
    "prob_mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.03463156,  1.84499347],\n",
       "       [ 6.24938303,  0.84906321],\n",
       "       [ 3.44103915,  6.21362187],\n",
       "       [ 6.18480492,  3.15806922],\n",
       "       [ 0.24467681,  6.21362187]])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prob_mix_sum = np.sum(prob_mix, axis=2)   # Dimensions T*N\n",
    "\n",
    "prob_mix_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.32304071,  0.67695929],\n",
       "        [ 0.0973674 ,  0.9026326 ]],\n",
       "\n",
       "       [[ 0.67343349,  0.32656651],\n",
       "        [ 0.01942641,  0.98057359]],\n",
       "\n",
       "       [[ 0.51603246,  0.48396754],\n",
       "        [ 0.23359366,  0.76640634]],\n",
       "\n",
       "       [[ 0.68046508,  0.31953492],\n",
       "        [ 0.06341479,  0.93658521]],\n",
       "\n",
       "       [[ 0.26579795,  0.73420205],\n",
       "        [ 0.76640634,  0.23359366]]])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Normalizing prob_mix\n",
    "\n",
    "post_mix = prob_mix / prob_mix_sum[:, :, np.newaxis]\n",
    "post_mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}